FIX: Conflict

components/belonging/belongingController.js

@@ -1,6 +1,25 @@
 const catchAsync = require('../../globalErrorHandler/catchAsync')
 const belongingService = require('./belongingService')
 
+exports.searchBelongings = catchAsync(async (req, res, next) => {
+    const {facets, belongings, totalCount, pagesCount} = await belongingService.searchBelongings()
+
+    res.status(200).json({
+        status: 'success',
+        facets, belongings, totalCount, pagesCount
+    })
+})
+
+exports.indexAllBelongings = catchAsync(async (req, res, next) => {
+    const { indexedCount } = await belongingService.indexAllBelongings()
+
+    res.status(200).json({
+        status: 'success',
+        indexedCount
+    })
+})
+
+
 exports.getAllBelongings = catchAsync(async (req, res, next) => {
     const belongings = await belongingService.getAllBelongings()
 

components/belonging/belongingModel.js

@@ -9,15 +9,11 @@ const belongingSchema = new mongoose.Schema(
             required: true,
             unique: true,
         },
-        belongingCategory: {
+        category: {
             type: ObjectID,
             ref: 'Category',
         },
-        projectCode: {
-            type: String,
-            required: true,
-            default: false,
-        },
+        description: String,
         relatedProjects: [String],
         location: {
             type: {

components/belonging/belongingRoutes.js

@@ -11,6 +11,8 @@ router.get('/admin/get-all', perm.requires('getallbelongings'), belongingControl
 
 router.post('/admin/create', perm.requires('createabelonging'), belongingController.createBelonging)
 
+router.post('/admin/index-all', perm.requires('indexbelongings', belongingController.indexAllBelongings))
+
 router.get('/admin/get-one/:belongingId', perm.requires('getabelonging'), belongingController.getBelonging)
 
 router.patch('/admin/update/:belongingId', perm.requires('updateabelonging'), belongingController.updateBelonging)

components/belonging/belongingService.js

@@ -1,15 +1,17 @@
 const Belonging = require('./belongingModel')
 const AppError = require('../../globalErrorHandler/AppError')
+const { client } = require('../../setup/typesense')
+const TsApiFeatures = require("../../utils/TsApiFeatures");
 
-module.exports.getAllBelongings = async () => {
+const getAllBelongings = async () => {
     return Belonging.find({}, null, null)
 }
 
-module.exports.createBelonging = async (data) => {
+const createBelonging = async (data) => {
     return Belonging.create(data, null)
 }
 
-module.exports.getBelonging = async (belongingId) => {
+const getBelonging = async (belongingId) => {
     const belonging = await Belonging.findById(belongingId, null, null)
 
     if (!belonging) {
@@ -19,7 +21,7 @@ module.exports.getBelonging = async (belongingId) => {
     return belonging
 }
 
-module.exports.updateBelonging = async (belongingId, data) => {
+const updateBelonging = async (belongingId, data) => {
     const belonging = await Belonging.findOneAndUpdate({ _id: belongingId }, data, { new: true })
 
     if (!belonging) {
@@ -29,7 +31,7 @@ module.exports.updateBelonging = async (belongingId, data) => {
     return belonging
 }
 
-module.exports.deleteBelonging = async (belongingId) => {
+const deleteBelonging = async (belongingId) => {
     const belonging = await Belonging.findByIdAndUpdate(belongingId, { isActive: false }, { new: true })
 
     if (!belonging) {
@@ -38,3 +40,84 @@ module.exports.deleteBelonging = async (belongingId) => {
 
     return null
 }
+
+const ensureCollectionExists = async () => {
+    const collections = await client.collections().retrieve()
+    const collectionExists = collections.some((collection) => collection.name === 'belongings')
+
+    if (collectionExists) {
+        await client.collections('belongings').delete()
+    }
+
+    const schema = getTypeSenseSchema()
+
+    await client.collections().create(schema)
+    console.log('Collection "belongings" created successfully')
+}
+
+const searchBelongings = async (inputQuery) => {
+    const tsFeatures = new TsApiFeatures(client, 'belongings', inputQuery).filter().facet().sort().limitFields().paginate().search()
+
+    let {facets, items, totalCount, pagesCount} = await tsFeatures.execute()
+
+    return {facets, belongings: items, totalCount, pagesCount}
+}
+const indexAllBelongings = async () => {
+    const batchSize = 20
+    const count = await Belonging.countDocuments({}, null)
+    const rounds = Math.ceil(count / batchSize)
+
+    await ensureCollectionExists()
+
+    for (let i = 1; i <= rounds; i++) {
+        console.log(`Indexing ${batchSize} belongings, Round ${i}`)
+
+        const belongings = await Belonging.find({}, null, null)
+            .populate('category')
+            .skip((i - 1) * batchSize)
+            .limit(batchSize)
+
+        let documents = []
+
+        for (const belonging of belongings) {
+
+            const data = {
+                _id: belonging._id,
+                isActive: belonging.isActive,
+                belongingCode: belonging.belongingCode,
+                category: belonging.category.map( x=> x.englishName),
+                nativeCategory: belonging.category.map( x=> x.nativeName),
+                documents: belonging.documents,
+                description: belonging.description,
+                coordinates: belonging.location.coordinates
+            }
+
+            documents.push(data)
+        }
+
+        await client.collections('belonging').documents().import(documents, {action: 'emplace'})
+    }
+
+    return {totalCount: count}
+}
+
+
+const getTypeSenseSchema = () => {
+    return {
+        name: 'belongings',
+        enable_nested_fields: true,
+        fields: [
+            { name: '_id', type: 'string', facet: false, infix: false },
+            { name: 'isActive', type: 'bool', facet: false, infix: true },
+            { name: 'belongingCode', type: 'string', facet: true },
+            { name: 'category', type: 'string[]', facet: true },
+            { name: 'nativeCategory', type: 'string[]', facet: false },
+            { name: 'documents', type: 'string[]', facet: false },
+            { name: 'description', type: 'string', facet: false },
+            { name: 'coordinates', type: 'geopoint' },
+        ],
+        default_sorting_field: 'createdAt',
+    }
+}
+
+module.exports = {indexAllBelongings, getBelonging, getAllBelongings, createBelonging, deleteBelonging, searchBelongings}

config/.env

@@ -1,7 +1,9 @@
-MONGODB_URL=mongodb://admin:password@mongodb:27018/marbm?authSource=admin
+MONGODB_URL=mongodb://admin:*135792468*@localhost:27018/marbm?authSource=admin
 NODE_ENV=development
 PORT=3000
 JWT_SECRET=Y@RJ$WT!@K0NS@MPL!
-REDIS_URL= redis://:135792468@redis:6379/0
+REDIS_URL= redis://:135792468@localhost:6379/0
 LOGIN_ATTEMPT_LIMIT=5
-SUPER_ADMIN_ROLE_ID=676afd6d076b286f9395b05f
+SUPER_ADMIN_ROLE_ID=676afd6d076b286f9395b05f
+TYPE_SENSE_URL=localhost:8108
+TYPE_SENSE_API_KEY=vYQq7pTgl6ACoRy

docker-compose.yml

@@ -1,14 +1,14 @@
 services:
-    api-server:
-        build: ./
-        ports:
-            - 3000:3000
-        environment:
-            - MONGODB_URL=mongodb://admin:*135792468*@mongo:27017/marbm?authSource=admin
-            - NODE_ENV=development
-            - PORT=3000
-            - REDIS_URL=redis://:135792468@redis:6379/0
-    minio-server:
+#    api-server:
+#        build: ./
+#        ports:
+#            - 3000:3000
+#        environment:
+#            - MONGODB_URL=mongodb://admin:*135792468*@mongo:27017/marbm?authSource=admin
+#            - NODE_ENV=development
+#            - PORT=3000
+#            - REDIS_URL=redis://:135792468@redis:6379/0
+    bm_minio:
         image: minio/minio
         ports:
             - 9000:9000
@@ -19,7 +19,7 @@ services:
         volumes:
             - bm_minio:/upload/
         command: server /data --console-address ":9001"
-    mongo:
+    bm_mongo:
         image: mongo:8.0
         ports:
             - 27018:27017
@@ -30,7 +30,7 @@ services:
         volumes:
             - bm_mongo:/data/db
 
-    redis:
+    bm_redis:
         image: redis
         command: redis-server --save 20 1 --loglevel warning --requirepass 135792468
         volumes:
@@ -38,7 +38,19 @@ services:
         ports:
             - 6379:6379
 
+    bm_typesense:
+        image: typesense/typesense:27.1
+        command: ./typesense-server
+        ports:
+            - "8108:8108"
+        environment:
+            TYPESENSE_API_KEY: vYQq7pTgl6ACoRy
+            TYPESENSE_DATA_DIR: /data
+        volumes:
+            - bm_typesense:/data
+
 volumes:
     bm_mongo:
     bm_redis:
     bm_minio:
+    bm_typesense:

package.json

@@ -12,6 +12,7 @@
   "license": "ISC",
   "description": "",
   "dependencies": {
+    "axios": "^1.7.9",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.20.3",
     "chalk": "^4.1.2",
@@ -24,6 +25,7 @@
     "mongoose": "^8.9.0",
     "mongoose-sequence": "^6.0.1",
     "redis": "^4.7.0",
+    "typesense": "^1.8.2",
     "validator": "^13.12.0"
   },
   "devDependencies": {

setup/typesense.js

@@ -0,0 +1,15 @@
+const TypeSense = require('typesense')
+
+let client = new TypeSense.Client({
+    nodes: [
+        {
+            host: process.env.TYPE_SENSE_URL,
+            port: '443',
+            protocol: 'httpS',
+        },
+    ],
+    apiKey: process.env.TYPE_SENSE_API_KEY,
+    connectionTimeoutSeconds: 2,
+})
+
+module.exports = { client }

utils/TsApiFeatures.js

@@ -0,0 +1,193 @@
+const axios = require('axios')
+class TsApiFeatures {
+    constructor(client, collectionName, queryString) {
+        this.client = client
+        this.collectionName = collectionName
+        this.queryString = queryString
+        this.queryOptions = {}
+    }
+
+    search() {
+        // Set `q` to provided search term or default to `*` for all documents
+        this.queryOptions.q = this.queryString.query || '*'
+
+        // Set `query_by` to specified fields or default to `*`
+        this.queryOptions.query_by = this.queryString.query_by || 'name'
+
+        return this
+    }
+
+    filter() {
+        let filters = []
+        let rawKeys = []
+
+        // Add any additional filter conditions from queryString, if needed
+        Object.keys(this.queryString).forEach((key) => {
+            let operatorConditions = []
+            let normalValues = []
+            if (
+                [
+                    'page',
+                    'sort',
+                    'limit',
+                    'fields',
+                    'per_page',
+                    'filter',
+                    'query',
+                    'facet_by',
+                    'query_by',
+                    'highlight_full_fields',
+                ].includes(key)
+            )
+                return
+
+            let value = this.queryString[key]
+            rawKeys.push(key)
+            if (!Array.isArray(value)) {
+                value = [value]
+            }
+            if (/\b(gte|gt|lte|lt|ne|eq)\b/.test(value)) {
+                const operatorMap = { gte: '>=', gt: '>', lte: '<=', lt: '<', ne: '!=', eq: '=' }
+                const match = value.match(/\b(gte|gt|lte|lt|ne|eq)\b/)
+                if (match) {
+                    const operator = operatorMap[match[0]]
+                    const cleanValue = value.replace(match[0], '').trim()
+                    operatorConditions.push(`${key}${operator}${cleanValue}`)
+                }
+            } else {
+                normalValues.push(value)
+            }
+
+            if (operatorConditions.length > 0 && normalValues.length > 0) {
+                // Add all operator conditions
+                operatorConditions.forEach((cond) => filters.push(cond))
+
+                // Add the normal values condition
+                if (normalValues.length === 1) {
+                    filters.push(`${key}:=${normalValues[0]}`)
+                } else {
+                    filters.push(`${key}:=[${normalValues.join(',')}]`)
+                }
+            } else if (operatorConditions.length > 0) {
+                // Only operators
+                operatorConditions.forEach((cond) => filters.push(cond))
+            } else {
+                // Only normal values
+                if (normalValues.length === 1) {
+                    filters.push(`${key}:=${normalValues[0]}`)
+                } else if (normalValues.length > 1) {
+                    filters.push(`${key}:=[${normalValues.join(',')}]`)
+                }
+            }
+        })
+
+        // Join all conditions for TypeSense filter
+        if (filters.length) {
+            this.queryOptions.filters = filters.join(' && ')
+        }
+        const searchFilters = []
+        rawKeys.forEach((value, index) => {
+            const tempArray = JSON.parse(JSON.stringify(filters))
+            tempArray.splice(index, 1)
+            searchFilters.push(tempArray.join(' && '))
+        })
+        this.filterKeys = rawKeys
+        this.searchFilters = searchFilters
+        return this
+    }
+
+    sort() {
+        if (this.queryString.sort) {
+            this.queryOptions.sort_by = this.queryString.sort.split(',').join(',')
+        } else {
+            this.queryOptions.sort_by = 'createdAt:desc'
+        }
+
+        return this
+    }
+
+    limitFields() {
+        if (this.queryString.fields) {
+            this.queryOptions.include_fields = this.queryString.fields.split(',').join(',')
+        } else {
+            this.queryOptions.exclude_fields = '__v'
+        }
+
+        return this
+    }
+
+    facet() {
+        // Set `facet_by` based on the queryString, if available
+        if (this.queryString.facet_by) {
+            this.queryOptions.facet_by = this.queryString.facet_by
+            this.queryOptions.highlight_full_fields = this.queryString.highlight_full_fields
+            this.queryOptions.exhaustive_search = true
+        }
+
+        return this
+    }
+
+    paginate() {
+        const page = parseInt(this.queryString.page) || 1
+        const per_page = parseInt(this.queryString.per_page) || 20
+
+        this.queryOptions.page = page
+        this.queryOptions.per_page = per_page
+
+        return this
+    }
+
+    async execute() {
+        const searches = [
+            {
+                collection: this.collectionName,
+                exhaustive_search: this.queryOptions.exhaustive_search,
+                query_by: this.queryOptions.query_by,
+                filter_by: this.queryOptions.filters,
+                highlight_full_fields: this.queryOptions.highlight_full_fields,
+                q: this.queryOptions.q,
+                facet_by: this.queryOptions.facet_by,
+                max_facet_values: 100,
+                page: this.queryOptions.page,
+                per_page: this.queryOptions.per_page,
+            },
+        ]
+        this.filterKeys.forEach((value, index) => {
+            searches.push({
+                collection: this.collectionName,
+                exhaustive_search: this.queryOptions.exhaustive_search,
+                query_by: this.queryOptions.query_by,
+                filter_by: this.searchFilters[index],
+                highlight_full_fields: this.queryOptions.highlight_full_fields,
+                q: this.queryOptions.q,
+                facet_by: value,
+                max_facet_values: 100,
+                page: this.queryOptions.page,
+                per_page: this.queryOptions.per_page,
+            })
+        })
+        const config = {
+            url: 'https://' + process.env.TYPE_SENSE_URL + '/multi_search?x-typesense-api-key=' + process.env.TYPE_SENSE_API_KEY,
+            method: 'POST',
+            data: {
+                searches,
+            },
+        }
+
+        const request = await axios(config)
+        const results = request.data.results[0]
+        let facet_counts = results.facet_counts
+        this.filterKeys.forEach((value, index) => {
+            const objIndex = facet_counts.findIndex((item) => item.field_name === value)
+            facet_counts[objIndex] = request.data.results[index + 1].facet_counts[0]
+        })
+        return {
+            facets: facet_counts,
+            items: results.hits.map((hit) => hit.document),
+            totalCount: results.found,
+            pagesCount: Math.ceil(results.found / this.queryOptions.per_page),
+        }
+    }
+}
+
+module.exports = TsApiFeatures