• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mia-platform / crud-service / 13717074317

07 Mar 2025 08:50AM UTC coverage: 97.386% (+0.2%) from 97.209%
13717074317

push

github

web-flow
fix: overhaul /export endpoint to prevent memory leak (#467)

* refactor: review code

* refactor: review list and export endpoint logic

* fix: prevent cursors memory leak when request cannot be served

* 7.2.3-rc.2

* refactor: replace JSON serialization with object conversion

* refactor: cleanup configuration option and update CHANGELOG.md

* 7.2.3-rc.3

* chore(deps): update mongodb to fix kms error

* 7.2.3-rc.4

1448 of 1556 branches covered (93.06%)

Branch coverage included in aggregate %.

241 of 260 new or added lines in 9 files covered. (92.69%)

2 existing lines in 1 file now uncovered.

9392 of 9575 relevant lines covered (98.09%)

7883.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.62
/lib/loadModels.js
1
/*
96✔
2
 * Copyright 2023 Mia s.r.l.
96✔
3
 *
96✔
4
 * Licensed under the Apache License, Version 2.0 (the "License");
96✔
5
 * you may not use this file except in compliance with the License.
96✔
6
 * You may obtain a copy of the License at
96✔
7
 *
96✔
8
 *     http://www.apache.org/licenses/LICENSE-2.0
96✔
9
 *
96✔
10
 * Unless required by applicable law or agreed to in writing, software
96✔
11
 * distributed under the License is distributed on an "AS IS" BASIS,
96✔
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
96✔
13
 * See the License for the specific language governing permissions and
96✔
14
 * limitations under the License.
96✔
15
 */
96✔
16

96✔
17
'use strict'
96✔
18

96✔
19
const Ajv = require('ajv')
96✔
20
const ajvFormats = require('ajv-formats')
96✔
21
const ajvKeywords = require('ajv-keywords')
96✔
22
const { omit: lomit } = require('lodash')
96✔
23

96✔
24
const mergeViewsInCollections = require('./mergeViewsInCollections')
96✔
25
const { compatibilityModelJsonSchema, modelJsonSchema } = require('./model.jsonschema')
96✔
26
const createIndexes = require('./createIndexes')
96✔
27
const { OBJECTID, aggregationConversion } = require('./consts')
96✔
28
const JSONSchemaGenerator = require('./JSONSchemaGenerator')
96✔
29
const QueryParser = require('./QueryParser')
96✔
30
const CrudService = require('./CrudService')
96✔
31
const generatePathFieldsForRawSchema = require('./generatePathFieldsForRawSchema')
96✔
32
const { getIdType } = require('./mongo/mongo-plugin')
96✔
33
const { getDatabaseNameByType, getPrefixedDatabaseName } = require('./pkFactories')
96✔
34

96✔
35
const ajv = new Ajv({ useDefaults: true })
96✔
36
ajvFormats(ajv)
96✔
37
ajvKeywords(ajv)
96✔
38

96✔
39
const compatibilityValidate = ajv.compile(compatibilityModelJsonSchema)
96✔
40
const validate = ajv.compile(modelJsonSchema)
96✔
41

96✔
42
const PREFIX_OF_INDEX_NAMES_TO_PRESERVE = 'preserve_'
96✔
43
const VIEW_TYPE = 'view'
96✔
44

96✔
45
async function loadModels(fastify) {
228✔
46
  const { default: plimit } = await import('p-limit')
228✔
47
  const limiter = plimit(5)
228✔
48

228✔
49
  const { collections, views = [] } = fastify
228✔
50
  const mergedCollections = mergeViewsInCollections(collections, views)
228✔
51

228✔
52
  fastify.log.trace({ collectionNames: mergedCollections.map(coll => coll.name) }, 'Registering CRUDs and Views')
228✔
53

228✔
54
  // A side-effect from the collectionModelMapper updates the following data models.
228✔
55
  const models = {}
228✔
56
  const existingStringCollection = []
228✔
57
  const existingObjectIdCollection = []
228✔
58

228✔
59
  const promises = mergedCollections.map(
228✔
60
    (collectionDefinition) => limiter(() =>
228✔
61
      collectionModelMapper(
4,407✔
62
        fastify,
4,407✔
63
        mergedCollections,
4,407✔
64
        { models, existingStringCollection, existingObjectIdCollection }
4,407✔
65
      )(collectionDefinition)
4,407✔
66
    )
4,407✔
67
  )
228✔
68

228✔
69
  try {
228✔
70
    await Promise.all(promises)
228✔
71
    fastify.decorate('models', models)
228✔
72
  } catch (error) {
228!
73
    fastify.log.error({ cause: error }, 'failed to load models')
×
74
    throw error
×
75
  }
×
76
}
228✔
77

96✔
78
function collectionModelMapper(
4,407✔
79
  fastify,
4,407✔
80
  mergedCollections,
4,407✔
81
  // The previous implementation of the mapper was with a closure function that applied a
4,407✔
82
  // side-effect on these values. During the refactor to apply p-limit I've decided not to
4,407✔
83
  // change the implementation, therefore I've wrapped the params to isolate them.
4,407✔
84
  { models, existingStringCollection, existingObjectIdCollection }
4,407✔
85
) {
4,407✔
86
  // eslint-disable-next-line max-statements
4,407✔
87
  return async(collectionDefinition) => {
4,407✔
88
    // avoid validating the collection definition twice, since it would only
4,407✔
89
    // match one of the two, depending on the existence of schema property
4,407✔
90
    if (!collectionDefinition.schema) {
4,407✔
91
      if (!compatibilityValidate(collectionDefinition)) {
4,383!
92
        fastify.log.error({ collection: collectionDefinition.name }, compatibilityValidate.errors)
×
93
        throw new Error(`invalid collection definition: ${JSON.stringify(compatibilityValidate.errors)}`)
×
94
      }
×
95
    } else if (!validate(collectionDefinition)) {
4,407!
96
      fastify.log.error(validate.errors)
×
97
      throw new Error(`invalid collection definition: ${JSON.stringify(validate.errors)}`)
×
98
    }
×
99

4,407✔
100
    const {
4,407✔
101
      source: viewSourceCollectionName,
4,407✔
102
      name: collectionName,
4,407✔
103
      endpointBasePath: collectionEndpoint,
4,407✔
104
      type,
4,407✔
105
      indexes = [],
4,407✔
106
      enableLookup,
4,407✔
107
      pipeline,
4,407✔
108
    } = collectionDefinition ?? {}
4,407!
109
    const isView = type === VIEW_TYPE
4,407✔
110
    const viewLookupsEnabled = isView && enableLookup
4,407✔
111

4,407✔
112
    fastify.log.trace({ collectionEndpoint, collectionName }, 'Registering CRUD')
4,407✔
113

4,407✔
114
    const collectionIdType = getIdType(collectionDefinition)
4,407✔
115
    const collection = await fastify
4,407✔
116
      .mongo[getDatabaseNameByType(collectionIdType)]
4,407✔
117
      .db
4,407✔
118
      .collection(collectionName)
4,407✔
119
    const modelDependencies = buildModelDependencies(fastify, collectionDefinition, collection)
4,407✔
120

4,407✔
121
    let viewDependencies = {}
4,407✔
122
    if (viewLookupsEnabled) {
4,407✔
123
      const sourceCollection = mergedCollections.find(mod => mod.name === viewSourceCollectionName)
18✔
124
      const sourceCollectionDependencies = buildModelDependencies(fastify, sourceCollection)
18✔
125

18✔
126
      const viewIdType = getIdType(sourceCollection)
18✔
127
      const sourceCollectionMongo = await fastify
18✔
128
        .mongo[getDatabaseNameByType(viewIdType)]
18✔
129
        .db
18✔
130
        .collection(viewSourceCollectionName)
18✔
131
      viewDependencies = buildModelDependencies(fastify, collectionDefinition, sourceCollectionMongo)
18✔
132
      viewDependencies.queryParser = sourceCollectionDependencies.queryParser
18✔
133
      viewDependencies.allFieldNames = sourceCollectionDependencies.allFieldNames
18✔
134
      viewDependencies.lookupsModels = createLookupModel(fastify, collectionDefinition, mergedCollections)
18✔
135
    }
18✔
136

4,407✔
137
    models[getCollectionNameFromEndpoint(collectionEndpoint)] = {
4,407✔
138
      definition: collectionDefinition,
4,407✔
139
      ...modelDependencies,
4,407✔
140
      viewDependencies,
4,407✔
141
      isView,
4,407✔
142
      viewLookupsEnabled,
4,407✔
143
    }
4,407✔
144

4,407✔
145
    if (isView) {
4,407✔
146
      // check whether it exists a Mongo instance dedicated to creating Mongo Views before using the standard one
627✔
147
      const mongoInstance = fastify
627✔
148
        .mongo[getPrefixedDatabaseName(collectionIdType)] ?? fastify
627✔
149
        .mongo[getDatabaseNameByType(collectionIdType)]
624✔
150

627✔
151
      const existingCollections = collectionIdType === OBJECTID ? existingObjectIdCollection : existingStringCollection
627✔
152
      if (existingCollections.length === 0) {
627✔
153
        existingCollections.push(
622✔
154
          ...(
622✔
155
            await mongoInstance
622✔
156
              .db
622✔
157
              .listCollections({}, { nameOnly: true })
622✔
158
              .toArray()
622✔
159
          )
622✔
160
            .filter(({ type: collectionType }) => collectionType === VIEW_TYPE)
622✔
161
            .map(({ name }) => name)
622✔
162
        )
622✔
163
      }
622✔
164

627✔
165
      // in case a view with selected name already exists, update it with the newer configuration
627✔
166
      if (existingCollections.includes(collectionName)) {
627!
167
        return mongoInstance.db.command({
×
168
          collMod: collectionName,
×
169
          viewOn: viewSourceCollectionName,
×
170
          pipeline,
×
171
        })
×
172
      }
×
173

627✔
174
      return mongoInstance
627✔
175
        .db
627✔
176
        .createCollection(
627✔
177
          collectionName,
627✔
178
          {
627✔
179
            viewOn: viewSourceCollectionName,
627✔
180
            pipeline,
627✔
181
          }
627✔
182
        )
627✔
183
    }
627✔
184

3,780✔
185
    if (!fastify.config.DISABLE_INDEX_MANAGEMENT) {
3,780✔
186
      try {
3,780✔
187
        await createIndexes(collection, indexes, PREFIX_OF_INDEX_NAMES_TO_PRESERVE)
3,780✔
188
      } catch (error) {
3,780!
NEW
189
        fastify.log.error({ cause: error, collectionName }, 'failed to create/update/delete an index on collection')
×
NEW
190
        throw new Error('failed setting up an index for collection', { cause: error })
×
NEW
191
      }
×
192
    }
3,780✔
193
  }
4,407✔
194
}
4,407✔
195

96✔
196
function getCollectionNameFromEndpoint(endpointBasePath) {
4,407✔
197
  return endpointBasePath.replace('/', '').replace(/\//g, '-')
4,407✔
198
}
4,407✔
199

96✔
200
function createLookupModel(fastify, viewDefinition, mergedCollections) {
18✔
201
  const lookupModels = []
18✔
202
  const viewLookups = viewDefinition.pipeline
18✔
203
    .filter(pipeline => '$lookup' in pipeline)
18✔
204
    .map(lookup => Object.values(lookup).shift())
18✔
205

18✔
206
  for (const lookup of viewLookups) {
18✔
207
    const { from, pipeline } = lookup
18✔
208
    const lookupCollection = mergedCollections.find(({ name }) => name === from)
18✔
209
    const lookupIdType = getIdType(lookupCollection)
18✔
210
    const lookupCollectionMongo = fastify.mongo[getDatabaseNameByType(lookupIdType)].db.collection(from)
18✔
211

18✔
212
    const lookupProjection = pipeline?.find(({ $project }) => $project)?.$project ?? {}
18!
213
    const parsedLookupProjection = []
18✔
214
    const lookupCollectionDefinition = {
18✔
215
      ...lomit(viewDefinition, ['fields']),
18✔
216
      schema: {
18✔
217
        type: 'object',
18✔
218
        properties: {},
18✔
219
        required: [],
18✔
220
      },
18✔
221
    }
18✔
222

18✔
223
    Object.entries(lookupProjection)
18✔
224
      .forEach(([fieldName, schema]) => {
18✔
225
        parsedLookupProjection.push({ [fieldName]: schema })
54✔
226
        const conversion = Object.keys(schema).shift()
54✔
227
        if (schema !== 0) {
54✔
228
          if (!aggregationConversion[conversion]) {
36!
229
            throw new Error(`Invalid view lookup definition: no explicit type found in ${JSON.stringify({ [fieldName]: schema })}`)
×
230
          }
×
231
          lookupCollectionDefinition.schema.properties[fieldName] = {
36✔
232
            type: aggregationConversion[conversion],
36✔
233
          }
36✔
234
        }
36✔
235
      })
18✔
236

18✔
237
    const lookupModel = {
18✔
238
      ...buildModelDependencies(fastify, lookupCollectionDefinition, lookupCollectionMongo),
18✔
239
      definition: lookupCollectionDefinition,
18✔
240
      lookup,
18✔
241
      parsedLookupProjection,
18✔
242
    }
18✔
243
    lookupModels.push(lookupModel)
18✔
244
  }
18✔
245
  return lookupModels
18✔
246
}
18✔
247

96✔
248
function buildModelDependencies(fastify, collectionDefinition, collection) {
4,461✔
249
  const {
4,461✔
250
    defaultState,
4,461✔
251
    defaultSorting,
4,461✔
252
  } = collectionDefinition
4,461✔
253

4,461✔
254
  const allFieldNames = collectionDefinition.fields
4,461✔
255
    ? collectionDefinition.fields.map(field => field.name)
4,461✔
256
    : Object.keys(collectionDefinition.schema.properties)
4,461✔
257
  const pathsForRawSchema = generatePathFieldsForRawSchema(fastify.log, collectionDefinition)
4,461✔
258

4,461✔
259
  // TODO: make this configurable
4,461✔
260
  const crudService = new CrudService(
4,461✔
261
    collection,
4,461✔
262
    defaultState,
4,461✔
263
    defaultSorting,
4,461✔
264
    { allowDiskUse: fastify.config.ALLOW_DISK_USE_IN_QUERIES },
4,461✔
265
  )
4,461✔
266
  const queryParser = new QueryParser(collectionDefinition, pathsForRawSchema)
4,461✔
267

4,461✔
268
  const jsonSchemaGenerator = new JSONSchemaGenerator(
4,461✔
269
    collectionDefinition,
4,461✔
270
    {},
4,461✔
271
    fastify.config.CRUD_LIMIT_CONSTRAINT_ENABLED,
4,461✔
272
    fastify.config.CRUD_MAX_LIMIT,
4,461✔
273
    fastify.config.ENABLE_STRICT_OUTPUT_VALIDATION,
4,461✔
274
    fastify.config.OPEN_API_SPECIFICATION,
4,461✔
275
  )
4,461✔
276
  const jsonSchemaGeneratorWithNested = new JSONSchemaGenerator(
4,461✔
277
    collectionDefinition,
4,461✔
278
    pathsForRawSchema,
4,461✔
279
    fastify.config.CRUD_LIMIT_CONSTRAINT_ENABLED,
4,461✔
280
    fastify.config.CRUD_MAX_LIMIT,
4,461✔
281
    fastify.config.ENABLE_STRICT_OUTPUT_VALIDATION,
4,461✔
282
    fastify.config.OPEN_API_SPECIFICATION,
4,461✔
283
  )
4,461✔
284

4,461✔
285
  return {
4,461✔
286
    crudService,
4,461✔
287
    queryParser,
4,461✔
288
    allFieldNames,
4,461✔
289
    jsonSchemaGenerator,
4,461✔
290
    jsonSchemaGeneratorWithNested,
4,461✔
291
  }
4,461✔
292
}
4,461✔
293

96✔
294
module.exports = loadModels
96✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc