From ef504f3775482deb90a363787eba995bff38bcb5 Mon Sep 17 00:00:00 2001 From: kaveh Date: Tue, 31 Dec 2024 22:49:43 +0330 Subject: [PATCH] ADD: TsApiFeatures ADD: Typesense ADD: Index all belongings router, controller & service --- components/belonging/belongingController.js | 10 + components/belonging/belongingRoutes.js | 2 + components/belonging/belongingService.js | 70 +++++++ setup/typesense.js | 15 ++ utils/TsApiFeatures.js | 194 ++++++++++++++++++++ 5 files changed, 291 insertions(+) create mode 100644 setup/typesense.js create mode 100644 utils/TsApiFeatures.js diff --git a/components/belonging/belongingController.js b/components/belonging/belongingController.js index aabe7f0..0e86a51 100644 --- a/components/belonging/belongingController.js +++ b/components/belonging/belongingController.js @@ -1,6 +1,16 @@ const catchAsync = require('../../globalErrorHandler/catchAsync') const belongingService = require('./belongingService') +exports.indexAllBelongings = catchAsync(async (req, res, next) => { + const { indexedCount } = await belongingService.indexAllBelongings() + + res.status(200).json({ + status: 'success', + indexedCount + }) +}) + + exports.getAllBelongings = catchAsync(async (req, res, next) => { const belongings = await belongingService.getAllBelongings() diff --git a/components/belonging/belongingRoutes.js b/components/belonging/belongingRoutes.js index 0e8fc7e..584f9c9 100644 --- a/components/belonging/belongingRoutes.js +++ b/components/belonging/belongingRoutes.js @@ -11,6 +11,8 @@ router.get('/admin/get-all', perm.requires('getallbelongings'), belongingControl router.post('/admin/create', perm.requires('createabelonging'), belongingController.createBelonging) +router.post('/admin/index-all', perm.requires('indexbelongings', belongingController.indexAllBelongings)) + router.get('/admin/get-one/:belongingId', perm.requires('getabelonging'), belongingController.getBelonging) router.put('/admin/update/:belongingId', perm.requires('updateabelonging'), belongingController.updateBelonging) diff --git a/components/belonging/belongingService.js b/components/belonging/belongingService.js index a2643bb..c5311fb 100644 --- a/components/belonging/belongingService.js +++ b/components/belonging/belongingService.js @@ -1,5 +1,6 @@ const Belonging = require('./belongingModel') const AppError = require('../../globalErrorHandler/AppError') +const { client } = require('../../setup/typesense') module.exports.getAllBelongings = async () => { return Belonging.find({}, null, null) @@ -38,3 +39,72 @@ module.exports.deleteBelonging = async (belongingId) => { return null } + +const ensureCollectionExists = async () => { + const collections = await client.collections().retrieve() + const collectionExists = collections.some((collection) => collection.name === 'belongings') + + if (collectionExists) { + await client.collections('belongings').delete() + } + + const schema = getTypeSenseSchema() + + await client.collections().create(schema) + console.log('Collection "belongings" created successfully') +} +const indexAllBelongings = async () => { + const batchSize = 20 + const count = await Product.countDocuments({isTrash: false, isActive: true, isHidden: false}, null) + const rounds = Math.ceil(count / batchSize) + + await ensureCollectionExists() + + for (let i = 1; i <= rounds; i++) { + console.log(`Indexing ${batchSize} products, Round ${i}`) + + const belongings = await Belonging.find({}, null, null) + .populate('category') + .skip((i - 1) * batchSize) + .limit(batchSize) + + let documents = [] + + for (const belonging of belongings) { + + const data = { + } + + for (let feature of product.features) { + if (feature.definition && feature.value) { + data[feature.definition.englishName] = feature.value + } + } + + documents.push(data) + } + + await client.collections('belonging').documents().import(documents, {action: 'emplace'}) + } + + return {totalCount: count} +} + + +const getTypeSenseSchema = () => { + return { + name: 'belongings', + enable_nested_fields: true, + fields: [ + { name: '_id', type: 'string', facet: false, infix: false }, + { name: 'projectCode', type: 'string', facet: false, infix: true }, + { name: 'belongingCode', type: 'string', facet: true }, + { name: 'category', type: 'string[]', facet: true }, + { name: 'nativeCategory', type: 'string[]', facet: false }, + { name: 'documents', type: 'string[]', facet: false }, + { name: 'description', type: 'string', facet: false }, + { name: 'coordinates', type: 'geopoint' }, + ], + default_sorting_field: 'createdAt', + } +} \ No newline at end of file diff --git a/setup/typesense.js b/setup/typesense.js new file mode 100644 index 0000000..ebe7e24 --- /dev/null +++ b/setup/typesense.js @@ -0,0 +1,15 @@ +const TypeSense = require('typesense') + +let client = new TypeSense.Client({ + nodes: [ + { + host: process.env.TYPE_SENSE_URL, + port: '443', + protocol: 'httpS', + }, + ], + apiKey: process.env.TYPE_SENSE_API_KEY, + connectionTimeoutSeconds: 2, +}) + +module.exports = { client } diff --git a/utils/TsApiFeatures.js b/utils/TsApiFeatures.js new file mode 100644 index 0000000..6f81157 --- /dev/null +++ b/utils/TsApiFeatures.js @@ -0,0 +1,194 @@ +const axios = require('axios') +const { fi } = require('@faker-js/faker') +class TsApiFeatures { + constructor(client, collectionName, queryString) { + this.client = client + this.collectionName = collectionName + this.queryString = queryString + this.queryOptions = {} + } + + search() { + // Set `q` to provided search term or default to `*` for all documents + this.queryOptions.q = this.queryString.query || '*' + + // Set `query_by` to specified fields or default to `*` + this.queryOptions.query_by = this.queryString.query_by || 'name' + + return this + } + + filter() { + let filters = [] + let rawKeys = [] + + // Add any additional filter conditions from queryString, if needed + Object.keys(this.queryString).forEach((key) => { + let operatorConditions = [] + let normalValues = [] + if ( + [ + 'page', + 'sort', + 'limit', + 'fields', + 'per_page', + 'filter', + 'query', + 'facet_by', + 'query_by', + 'highlight_full_fields', + ].includes(key) + ) + return + + let value = this.queryString[key] + rawKeys.push(key) + if (!Array.isArray(value)) { + value = [value] + } + if (/\b(gte|gt|lte|lt|ne|eq)\b/.test(value)) { + const operatorMap = { gte: '>=', gt: '>', lte: '<=', lt: '<', ne: '!=', eq: '=' } + const match = value.match(/\b(gte|gt|lte|lt|ne|eq)\b/) + if (match) { + const operator = operatorMap[match[0]] + const cleanValue = value.replace(match[0], '').trim() + operatorConditions.push(`${key}${operator}${cleanValue}`) + } + } else { + normalValues.push(value) + } + + if (operatorConditions.length > 0 && normalValues.length > 0) { + // Add all operator conditions + operatorConditions.forEach((cond) => filters.push(cond)) + + // Add the normal values condition + if (normalValues.length === 1) { + filters.push(`${key}:=${normalValues[0]}`) + } else { + filters.push(`${key}:=[${normalValues.join(',')}]`) + } + } else if (operatorConditions.length > 0) { + // Only operators + operatorConditions.forEach((cond) => filters.push(cond)) + } else { + // Only normal values + if (normalValues.length === 1) { + filters.push(`${key}:=${normalValues[0]}`) + } else if (normalValues.length > 1) { + filters.push(`${key}:=[${normalValues.join(',')}]`) + } + } + }) + + // Join all conditions for TypeSense filter + if (filters.length) { + this.queryOptions.filters = filters.join(' && ') + } + const searchFilters = [] + rawKeys.forEach((value, index) => { + const tempArray = JSON.parse(JSON.stringify(filters)) + tempArray.splice(index, 1) + searchFilters.push(tempArray.join(' && ')) + }) + this.filterKeys = rawKeys + this.searchFilters = searchFilters + return this + } + + sort() { + if (this.queryString.sort) { + this.queryOptions.sort_by = this.queryString.sort.split(',').join(',') + } else { + this.queryOptions.sort_by = 'createdAt:desc' + } + + return this + } + + limitFields() { + if (this.queryString.fields) { + this.queryOptions.include_fields = this.queryString.fields.split(',').join(',') + } else { + this.queryOptions.exclude_fields = '__v' + } + + return this + } + + facet() { + // Set `facet_by` based on the queryString, if available + if (this.queryString.facet_by) { + this.queryOptions.facet_by = this.queryString.facet_by + this.queryOptions.highlight_full_fields = this.queryString.highlight_full_fields + this.queryOptions.exhaustive_search = true + } + + return this + } + + paginate() { + const page = parseInt(this.queryString.page) || 1 + const per_page = parseInt(this.queryString.per_page) || 20 + + this.queryOptions.page = page + this.queryOptions.per_page = per_page + + return this + } + + async execute() { + const searches = [ + { + collection: this.collectionName, + exhaustive_search: this.queryOptions.exhaustive_search, + query_by: this.queryOptions.query_by, + filter_by: this.queryOptions.filters, + highlight_full_fields: this.queryOptions.highlight_full_fields, + q: this.queryOptions.q, + facet_by: this.queryOptions.facet_by, + max_facet_values: 100, + page: this.queryOptions.page, + per_page: this.queryOptions.per_page, + }, + ] + this.filterKeys.forEach((value, index) => { + searches.push({ + collection: this.collectionName, + exhaustive_search: this.queryOptions.exhaustive_search, + query_by: this.queryOptions.query_by, + filter_by: this.searchFilters[index], + highlight_full_fields: this.queryOptions.highlight_full_fields, + q: this.queryOptions.q, + facet_by: value, + max_facet_values: 100, + page: this.queryOptions.page, + per_page: this.queryOptions.per_page, + }) + }) + const config = { + url: 'https://' + process.env.TYPE_SENSE_URL + '/multi_search?x-typesense-api-key=' + process.env.TYPE_SENSE_API_KEY, + method: 'POST', + data: { + searches, + }, + } + + const request = await axios(config) + const results = request.data.results[0] + let facet_counts = results.facet_counts + this.filterKeys.forEach((value, index) => { + const objIndex = facet_counts.findIndex((item) => item.field_name === value) + facet_counts[objIndex] = request.data.results[index + 1].facet_counts[0] + }) + return { + facets: facet_counts, + items: results.hits.map((hit) => hit.document), + totalCount: results.found, + pagesCount: Math.ceil(results.found / this.queryOptions.per_page), + } + } +} + +module.exports = TsApiFeatures