diff --git a/components/belonging/belongingController.js b/components/belonging/belongingController.js index aabe7f0..5c57c31 100644 --- a/components/belonging/belongingController.js +++ b/components/belonging/belongingController.js @@ -1,6 +1,25 @@ const catchAsync = require('../../globalErrorHandler/catchAsync') const belongingService = require('./belongingService') +exports.searchBelongings = catchAsync(async (req, res, next) => { + const {facets, belongings, totalCount, pagesCount} = await belongingService.searchBelongings() + + res.status(200).json({ + status: 'success', + facets, belongings, totalCount, pagesCount + }) +}) + +exports.indexAllBelongings = catchAsync(async (req, res, next) => { + const { indexedCount } = await belongingService.indexAllBelongings() + + res.status(200).json({ + status: 'success', + indexedCount + }) +}) + + exports.getAllBelongings = catchAsync(async (req, res, next) => { const belongings = await belongingService.getAllBelongings() diff --git a/components/belonging/belongingModel.js b/components/belonging/belongingModel.js index b8d9a8c..9214a71 100644 --- a/components/belonging/belongingModel.js +++ b/components/belonging/belongingModel.js @@ -9,15 +9,11 @@ const belongingSchema = new mongoose.Schema( required: true, unique: true, }, - belongingCategory: { + category: { type: ObjectID, ref: 'Category', }, - projectCode: { - type: String, - required: true, - default: false, - }, + description: String, relatedProjects: [String], location: { type: { diff --git a/components/belonging/belongingRoutes.js b/components/belonging/belongingRoutes.js index e3715e1..d2d306c 100644 --- a/components/belonging/belongingRoutes.js +++ b/components/belonging/belongingRoutes.js @@ -11,6 +11,8 @@ router.get('/admin/get-all', perm.requires('getallbelongings'), belongingControl router.post('/admin/create', perm.requires('createabelonging'), belongingController.createBelonging) +router.post('/admin/index-all', perm.requires('indexbelongings', belongingController.indexAllBelongings)) + router.get('/admin/get-one/:belongingId', perm.requires('getabelonging'), belongingController.getBelonging) router.patch('/admin/update/:belongingId', perm.requires('updateabelonging'), belongingController.updateBelonging) diff --git a/components/belonging/belongingService.js b/components/belonging/belongingService.js index a2643bb..6d576f1 100644 --- a/components/belonging/belongingService.js +++ b/components/belonging/belongingService.js @@ -1,15 +1,17 @@ const Belonging = require('./belongingModel') const AppError = require('../../globalErrorHandler/AppError') +const { client } = require('../../setup/typesense') +const TsApiFeatures = require("../../utils/TsApiFeatures"); -module.exports.getAllBelongings = async () => { +const getAllBelongings = async () => { return Belonging.find({}, null, null) } -module.exports.createBelonging = async (data) => { +const createBelonging = async (data) => { return Belonging.create(data, null) } -module.exports.getBelonging = async (belongingId) => { +const getBelonging = async (belongingId) => { const belonging = await Belonging.findById(belongingId, null, null) if (!belonging) { @@ -19,7 +21,7 @@ module.exports.getBelonging = async (belongingId) => { return belonging } -module.exports.updateBelonging = async (belongingId, data) => { +const updateBelonging = async (belongingId, data) => { const belonging = await Belonging.findOneAndUpdate({ _id: belongingId }, data, { new: true }) if (!belonging) { @@ -29,7 +31,7 @@ module.exports.updateBelonging = async (belongingId, data) => { return belonging } -module.exports.deleteBelonging = async (belongingId) => { +const deleteBelonging = async (belongingId) => { const belonging = await Belonging.findByIdAndUpdate(belongingId, { isActive: false }, { new: true }) if (!belonging) { @@ -38,3 +40,84 @@ module.exports.deleteBelonging = async (belongingId) => { return null } + +const ensureCollectionExists = async () => { + const collections = await client.collections().retrieve() + const collectionExists = collections.some((collection) => collection.name === 'belongings') + + if (collectionExists) { + await client.collections('belongings').delete() + } + + const schema = getTypeSenseSchema() + + await client.collections().create(schema) + console.log('Collection "belongings" created successfully') +} + +const searchBelongings = async (inputQuery) => { + const tsFeatures = new TsApiFeatures(client, 'belongings', inputQuery).filter().facet().sort().limitFields().paginate().search() + + let {facets, items, totalCount, pagesCount} = await tsFeatures.execute() + + return {facets, belongings: items, totalCount, pagesCount} +} +const indexAllBelongings = async () => { + const batchSize = 20 + const count = await Belonging.countDocuments({}, null) + const rounds = Math.ceil(count / batchSize) + + await ensureCollectionExists() + + for (let i = 1; i <= rounds; i++) { + console.log(`Indexing ${batchSize} belongings, Round ${i}`) + + const belongings = await Belonging.find({}, null, null) + .populate('category') + .skip((i - 1) * batchSize) + .limit(batchSize) + + let documents = [] + + for (const belonging of belongings) { + + const data = { + _id: belonging._id, + isActive: belonging.isActive, + belongingCode: belonging.belongingCode, + category: belonging.category.map( x=> x.englishName), + nativeCategory: belonging.category.map( x=> x.nativeName), + documents: belonging.documents, + description: belonging.description, + coordinates: belonging.location.coordinates + } + + documents.push(data) + } + + await client.collections('belonging').documents().import(documents, {action: 'emplace'}) + } + + return {totalCount: count} +} + + +const getTypeSenseSchema = () => { + return { + name: 'belongings', + enable_nested_fields: true, + fields: [ + { name: '_id', type: 'string', facet: false, infix: false }, + { name: 'isActive', type: 'bool', facet: false, infix: true }, + { name: 'belongingCode', type: 'string', facet: true }, + { name: 'category', type: 'string[]', facet: true }, + { name: 'nativeCategory', type: 'string[]', facet: false }, + { name: 'documents', type: 'string[]', facet: false }, + { name: 'description', type: 'string', facet: false }, + { name: 'coordinates', type: 'geopoint' }, + ], + default_sorting_field: 'createdAt', + } +} + +module.exports = {indexAllBelongings, getBelonging, getAllBelongings, createBelonging, deleteBelonging, searchBelongings} \ No newline at end of file diff --git a/config/.env b/config/.env index 4642e61..e76abc7 100644 --- a/config/.env +++ b/config/.env @@ -1,7 +1,9 @@ -MONGODB_URL=mongodb://admin:password@mongodb:27018/marbm?authSource=admin +MONGODB_URL=mongodb://admin:*135792468*@localhost:27018/marbm?authSource=admin NODE_ENV=development PORT=3000 JWT_SECRET=Y@RJ$WT!@K0NS@MPL! -REDIS_URL= redis://:135792468@redis:6379/0 +REDIS_URL= redis://:135792468@localhost:6379/0 LOGIN_ATTEMPT_LIMIT=5 -SUPER_ADMIN_ROLE_ID=676afd6d076b286f9395b05f \ No newline at end of file +SUPER_ADMIN_ROLE_ID=676afd6d076b286f9395b05f +TYPE_SENSE_URL=localhost:8108 +TYPE_SENSE_API_KEY=vYQq7pTgl6ACoRy \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bf85114..3c87d83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,14 @@ services: - api-server: - build: ./ - ports: - - 3000:3000 - environment: - - MONGODB_URL=mongodb://admin:*135792468*@mongo:27017/marbm?authSource=admin - - NODE_ENV=development - - PORT=3000 - - REDIS_URL=redis://:135792468@redis:6379/0 - minio-server: +# api-server: +# build: ./ +# ports: +# - 3000:3000 +# environment: +# - MONGODB_URL=mongodb://admin:*135792468*@mongo:27017/marbm?authSource=admin +# - NODE_ENV=development +# - PORT=3000 +# - REDIS_URL=redis://:135792468@redis:6379/0 + bm_minio: image: minio/minio ports: - 9000:9000 @@ -19,7 +19,7 @@ services: volumes: - bm_minio:/upload/ command: server /data --console-address ":9001" - mongo: + bm_mongo: image: mongo:8.0 ports: - 27018:27017 @@ -30,7 +30,7 @@ services: volumes: - bm_mongo:/data/db - redis: + bm_redis: image: redis command: redis-server --save 20 1 --loglevel warning --requirepass 135792468 volumes: @@ -38,7 +38,19 @@ services: ports: - 6379:6379 + bm_typesense: + image: typesense/typesense:27.1 + command: ./typesense-server + ports: + - "8108:8108" + environment: + TYPESENSE_API_KEY: vYQq7pTgl6ACoRy + TYPESENSE_DATA_DIR: /data + volumes: + - bm_typesense:/data + volumes: bm_mongo: bm_redis: bm_minio: + bm_typesense: diff --git a/package.json b/package.json index 413bb73..92a8be4 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "license": "ISC", "description": "", "dependencies": { + "axios": "^1.7.9", "bcryptjs": "^2.4.3", "body-parser": "^1.20.3", "chalk": "^4.1.2", @@ -24,6 +25,7 @@ "mongoose": "^8.9.0", "mongoose-sequence": "^6.0.1", "redis": "^4.7.0", + "typesense": "^1.8.2", "validator": "^13.12.0" }, "devDependencies": { diff --git a/setup/typesense.js b/setup/typesense.js new file mode 100644 index 0000000..ebe7e24 --- /dev/null +++ b/setup/typesense.js @@ -0,0 +1,15 @@ +const TypeSense = require('typesense') + +let client = new TypeSense.Client({ + nodes: [ + { + host: process.env.TYPE_SENSE_URL, + port: '443', + protocol: 'httpS', + }, + ], + apiKey: process.env.TYPE_SENSE_API_KEY, + connectionTimeoutSeconds: 2, +}) + +module.exports = { client } diff --git a/utils/TsApiFeatures.js b/utils/TsApiFeatures.js new file mode 100644 index 0000000..222f499 --- /dev/null +++ b/utils/TsApiFeatures.js @@ -0,0 +1,193 @@ +const axios = require('axios') +class TsApiFeatures { + constructor(client, collectionName, queryString) { + this.client = client + this.collectionName = collectionName + this.queryString = queryString + this.queryOptions = {} + } + + search() { + // Set `q` to provided search term or default to `*` for all documents + this.queryOptions.q = this.queryString.query || '*' + + // Set `query_by` to specified fields or default to `*` + this.queryOptions.query_by = this.queryString.query_by || 'name' + + return this + } + + filter() { + let filters = [] + let rawKeys = [] + + // Add any additional filter conditions from queryString, if needed + Object.keys(this.queryString).forEach((key) => { + let operatorConditions = [] + let normalValues = [] + if ( + [ + 'page', + 'sort', + 'limit', + 'fields', + 'per_page', + 'filter', + 'query', + 'facet_by', + 'query_by', + 'highlight_full_fields', + ].includes(key) + ) + return + + let value = this.queryString[key] + rawKeys.push(key) + if (!Array.isArray(value)) { + value = [value] + } + if (/\b(gte|gt|lte|lt|ne|eq)\b/.test(value)) { + const operatorMap = { gte: '>=', gt: '>', lte: '<=', lt: '<', ne: '!=', eq: '=' } + const match = value.match(/\b(gte|gt|lte|lt|ne|eq)\b/) + if (match) { + const operator = operatorMap[match[0]] + const cleanValue = value.replace(match[0], '').trim() + operatorConditions.push(`${key}${operator}${cleanValue}`) + } + } else { + normalValues.push(value) + } + + if (operatorConditions.length > 0 && normalValues.length > 0) { + // Add all operator conditions + operatorConditions.forEach((cond) => filters.push(cond)) + + // Add the normal values condition + if (normalValues.length === 1) { + filters.push(`${key}:=${normalValues[0]}`) + } else { + filters.push(`${key}:=[${normalValues.join(',')}]`) + } + } else if (operatorConditions.length > 0) { + // Only operators + operatorConditions.forEach((cond) => filters.push(cond)) + } else { + // Only normal values + if (normalValues.length === 1) { + filters.push(`${key}:=${normalValues[0]}`) + } else if (normalValues.length > 1) { + filters.push(`${key}:=[${normalValues.join(',')}]`) + } + } + }) + + // Join all conditions for TypeSense filter + if (filters.length) { + this.queryOptions.filters = filters.join(' && ') + } + const searchFilters = [] + rawKeys.forEach((value, index) => { + const tempArray = JSON.parse(JSON.stringify(filters)) + tempArray.splice(index, 1) + searchFilters.push(tempArray.join(' && ')) + }) + this.filterKeys = rawKeys + this.searchFilters = searchFilters + return this + } + + sort() { + if (this.queryString.sort) { + this.queryOptions.sort_by = this.queryString.sort.split(',').join(',') + } else { + this.queryOptions.sort_by = 'createdAt:desc' + } + + return this + } + + limitFields() { + if (this.queryString.fields) { + this.queryOptions.include_fields = this.queryString.fields.split(',').join(',') + } else { + this.queryOptions.exclude_fields = '__v' + } + + return this + } + + facet() { + // Set `facet_by` based on the queryString, if available + if (this.queryString.facet_by) { + this.queryOptions.facet_by = this.queryString.facet_by + this.queryOptions.highlight_full_fields = this.queryString.highlight_full_fields + this.queryOptions.exhaustive_search = true + } + + return this + } + + paginate() { + const page = parseInt(this.queryString.page) || 1 + const per_page = parseInt(this.queryString.per_page) || 20 + + this.queryOptions.page = page + this.queryOptions.per_page = per_page + + return this + } + + async execute() { + const searches = [ + { + collection: this.collectionName, + exhaustive_search: this.queryOptions.exhaustive_search, + query_by: this.queryOptions.query_by, + filter_by: this.queryOptions.filters, + highlight_full_fields: this.queryOptions.highlight_full_fields, + q: this.queryOptions.q, + facet_by: this.queryOptions.facet_by, + max_facet_values: 100, + page: this.queryOptions.page, + per_page: this.queryOptions.per_page, + }, + ] + this.filterKeys.forEach((value, index) => { + searches.push({ + collection: this.collectionName, + exhaustive_search: this.queryOptions.exhaustive_search, + query_by: this.queryOptions.query_by, + filter_by: this.searchFilters[index], + highlight_full_fields: this.queryOptions.highlight_full_fields, + q: this.queryOptions.q, + facet_by: value, + max_facet_values: 100, + page: this.queryOptions.page, + per_page: this.queryOptions.per_page, + }) + }) + const config = { + url: 'https://' + process.env.TYPE_SENSE_URL + '/multi_search?x-typesense-api-key=' + process.env.TYPE_SENSE_API_KEY, + method: 'POST', + data: { + searches, + }, + } + + const request = await axios(config) + const results = request.data.results[0] + let facet_counts = results.facet_counts + this.filterKeys.forEach((value, index) => { + const objIndex = facet_counts.findIndex((item) => item.field_name === value) + facet_counts[objIndex] = request.data.results[index + 1].facet_counts[0] + }) + return { + facets: facet_counts, + items: results.hits.map((hit) => hit.document), + totalCount: results.found, + pagesCount: Math.ceil(results.found / this.queryOptions.per_page), + } + } +} + +module.exports = TsApiFeatures