Merge pull request #416 from twinlogix/feature/mongodb-indexes

feat: partially closes #415

src/dal/dao/indexes/indexes.types.ts

@@ -0,0 +1,35 @@
+import { Update } from '../schemas/ast.types'
+import { AbstractSyntaxTree, AbstractScalars } from '../schemas/ast.types'
+import { CreateIndexesOptions, IndexDirection } from 'mongodb'
+
+type MongoDBIndex<AST extends AbstractSyntaxTree, Scalars extends AbstractScalars, K extends keyof AST> = {
+  name: string
+  specs: { [F in keyof (K extends string ? Update<K, AST, Scalars> : never)]?: IndexDirection }
+  opts?: Omit<CreateIndexesOptions, 'name'>
+}
+
+type MongoEntities<AST extends AbstractSyntaxTree> = {
+  [K in keyof AST]: AST[K]['driverSpecification']['type'] extends 'mongodb' ? K : never
+}[keyof AST]
+
+export type MongoDBIndexes<AST extends AbstractSyntaxTree, Scalars extends AbstractScalars> = { [K in MongoEntities<AST>]: MongoDBIndex<AST, Scalars, K>[] }
+export type Indexes<AST extends AbstractSyntaxTree = AbstractSyntaxTree, Scalars extends AbstractScalars = AbstractScalars> = {
+  mongodb?: Partial<MongoDBIndexes<AST, Scalars>>
+}
+
+export type IndexesPlanResults = {
+  mongodb: {
+    toDelete: { collection: string; name: string }[]
+    toCreate: { collection: string; name: string; specs: { [K in string]: IndexDirection }; opts?: Omit<CreateIndexesOptions, 'name'> }[]
+  }
+  //knexjs: never
+}
+
+export type IndexesApplyResults = {
+  mongodb: {
+    deleted: { collection: string; name: string }[]
+    created: { collection: string; name: string; specs: { [K in string]: IndexDirection }; opts?: Omit<CreateIndexesOptions, 'name'> }[]
+    failed: { collection: string; name: string; specs: { [K in string]: IndexDirection }; opts?: Omit<CreateIndexesOptions, 'name'>; error: unknown }[]
+  }
+  //knexjs: never
+}

src/dal/dao/schemas/ast.types.ts

@@ -27,6 +27,7 @@ export type AbstractSyntaxTree<Entities extends string = string, Scalars extends
   [K in Entities]: {
     fields: AbstractSyntaxTreeFields<Entities, Scalars>
     driverSpecification: {
+      type: 'mongodb' | 'knexjs' | 'memory'
       rawFilter: unknown
       rawUpdate: unknown
       rawSorts: unknown

src/dal/dao/schemas/schemas.types.ts

@@ -46,4 +46,4 @@ export type SchemaField<Scalars extends AbstractScalars> = Readonly<
     Decorators
 >
 
-export type Schema<Scalars extends AbstractScalars> = Readonly<{ [key: string]: SchemaField<Scalars> }>
+export type Schema<Scalars extends AbstractScalars = AbstractScalars> = Readonly<{ [key: string]: SchemaField<Scalars> }>

src/dal/entity-manager.ts

@@ -1,4 +1,5 @@
-import { AbstractScalars } from '..'
+import { AbstractScalars, equals, Indexes, IndexesApplyResults, IndexesPlanResults, Schema } from '..'
+import { toFirstLower } from '../generation/utils'
 import { AbstractDAO } from './dao/dao'
 import { DAOGenerics, TransactionData } from './dao/dao.types'
 import { DriverDataTypeAdapterMap } from './drivers/drivers.adapters'
@@ -7,7 +8,7 @@ import { inMemoryAdapters } from './drivers/in-memory/adapters.memory'
 import { mongoDbAdapters } from './drivers/no-sql/mongodb/adapters.mongodb'
 import { knexJsAdapters } from './drivers/sql/knexjs/adapters.knexjs'
 import { Knex } from 'knex'
-import { ClientSession } from 'mongodb'
+import { ClientSession, Collection, CreateIndexesOptions, IndexDirection } from 'mongodb'
 
 export abstract class AbstractEntityManager<
   MongoDBDatasources extends string,
@@ -40,6 +41,103 @@ export abstract class AbstractEntityManager<
     return this.transactionData !== null
   }
 
+  public abstract execQuery<T>(run: (dbs: Record<string, unknown>, entities: Record<string, unknown>) => Promise<T>): Promise<T>
+
+  protected async planIndexes(args: { indexes: Indexes }, schemas?: Record<string, () => Schema>): Promise<IndexesPlanResults> {
+    function getDbName(field: string, schema: Schema): string {
+      const [head, ...tail] = field.split('.')
+      const schemaField = schema[head]
+      const name = schemaField.alias ?? head
+      return tail.length > 0 && schemaField.type === 'embedded' ? `${name}.${getDbName(tail.join('.'), schemaField.schema())}` : name
+    }
+    if (!schemas) {
+      throw new Error('schemas is required')
+    }
+    const result: IndexesPlanResults = { mongodb: { toCreate: [], toDelete: [] } }
+    await this.execQuery(async (dbs, collections) => {
+      for (const [name, indexes] of Object.entries(args.indexes.mongodb ?? {})) {
+        if (!indexes) continue
+        const key = toFirstLower(name)
+        const collection = collections[key] as Collection
+        if (collection) {
+          // tricky way to create collection
+          const inserted = await collection.insertOne({ empty: true })
+          await collection.deleteOne({ _id: inserted.insertedId })
+          const schema = schemas[name]()
+          const loadedIndexes = await collection.indexes()
+          const indexesThatAreEquals = loadedIndexes.filter((li) => {
+            const index = (indexes as Record<string, unknown>[]).find((i) => li.name === i.name)
+            if (!index) {
+              return false
+            }
+            const li2 = { ...li }
+            delete li2.v
+            delete li2.background
+            delete li2.key
+            delete li2.name
+            const specs = Object.fromEntries(
+              Object.entries(index.specs as Record<string, unknown>).map(([field, value]) => {
+                const dbName = getDbName(field, schema)
+                return [dbName, value]
+              }),
+            )
+            if (equals(specs, li.key) && equals(index.opts ?? {}, li2)) {
+              return true
+            }
+            return false
+          })
+          //same name but not equals
+          const indexesToDrop = loadedIndexes.filter((li) => li.name !== '_id_' && !indexesThatAreEquals.find((i) => li.name === i.name))
+          const indexesToCreate = (indexes as Record<string, unknown>[]).filter((i) => !indexesThatAreEquals.find((li) => li.name === i.name))
+          for (const index of indexesToDrop) {
+            result.mongodb.toDelete.push({ collection: key, name: index.name })
+          }
+          for (const index of indexesToCreate) {
+            const specs = Object.fromEntries(
+              Object.entries(index.specs as Record<string, unknown>).map(([field, value]) => {
+                const dbName = getDbName(field, schema)
+                return [dbName, value]
+              }),
+            )
+            result.mongodb.toCreate.push({
+              collection: key,
+              name: index.name as string,
+              specs: specs as Record<string, IndexDirection>,
+              opts: index.opts as Omit<CreateIndexesOptions, 'name'> | undefined,
+            })
+          }
+        }
+      }
+    })
+    return result
+  }
+
+  protected async applyIndexes(args: ({ plan: IndexesPlanResults } | { indexes: Indexes }) & { logs?: boolean }, schemas?: Record<string, () => Schema>): Promise<IndexesApplyResults> {
+    const plan: IndexesPlanResults = 'plan' in args ? args.plan : await this.planIndexes(args, schemas)
+    const results: IndexesApplyResults = { mongodb: { created: [], failed: [], deleted: [] } }
+    await this.execQuery(async (dbs, collections) => {
+      for (const index of plan.mongodb.toDelete) {
+        const collection = collections[index.collection] as Collection
+        if (!collection) continue
+        if (args.logs) console.log(`Dropping ${index.name} index of collection ${index.collection}...`)
+        await collection.dropIndex(index.name)
+        results.mongodb.deleted.push({ ...index })
+      }
+      for (const index of plan.mongodb.toCreate) {
+        try {
+          const collection = collections[index.collection] as Collection
+          if (!collection) continue
+          if (args.logs) console.log(`Creating ${index.name} index of collection ${index.collection}...`)
+          await collection.createIndex(index.specs, { ...(index.opts ?? {}), name: index.name })
+          results.mongodb.created.push({ ...index })
+        } catch (error) {
+          results.mongodb.failed.push({ ...index, error })
+        }
+      }
+    })
+    return results
+  }
+
   public getMongoSession(datasource: string): ClientSession | undefined {
     return ((this.transactionData?.mongodb ?? {}) as Record<string, ClientSession>)[datasource]
   }

src/generation/generators/dao-generator.ts

@@ -252,6 +252,7 @@ export class TsTypettaGenerator extends TypettaGenerator {
         return `${node.name}: {
           fields: { ${node.fields.map((f) => generateASTNodes(f)).join(',\n')} }, 
           driverSpecification: { 
+            type: '${node.entity?.type === 'mongo' ? 'mongodb' : node.entity?.type === 'sql' ? 'knexjs' : 'memory'}',
             rawFilter: ${
               node.entity?.type === 'mongo' ? '() => M.Filter<M.Document>' : node.entity?.type === 'sql' ? '(builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>' : 'never'
             }, 
@@ -326,6 +327,14 @@ export class TsTypettaGenerator extends TypettaGenerator {
       .join(', ')
     const execQueryF =
       `public async execQuery<T>(run: (dbs: { ${dbsInputParam} }, entities: { ${entitiesInputParam} }) => Promise<T>): Promise<T> {\n` + `  return run({ ${dbsParam} }, { ${entitiesParam} })\n` + `}`
+    const applyIndexesF = `
+      public planIndexes(args: { indexes: Partial<T.Indexes<AST, ScalarsSpecification>> }): Promise<T.IndexesPlanResults> {
+        return super.planIndexes(args, schemas)
+      }
+    
+      public applyIndexes(args: ({ plan: T.IndexesPlanResults } | { indexes: Partial<T.Indexes<AST, ScalarsSpecification>> }) & { logs?: boolean }): Promise<T.IndexesApplyResults> {
+        return super.applyIndexes(args, schemas)
+      }`
     const cloneF = `public clone(): this {\n  return new EntityManager<MetadataType, OperationMetadataType, Permissions, SecurityDomain>(this.params) as this\n}`
     const createTableBody = Array.from(typesMap.values())
       .flatMap((node) => {
@@ -412,6 +421,7 @@ export type EntityManagerParams<MetadataType, OperationMetadataType, Permissions
       entityManagerGetters,
       entityManagerConstructor,
       execQueryF,
+      applyIndexesF,
       cloneF,
       createTableF,
     ].join('\n\n')
@@ -499,6 +509,7 @@ export type EntityManagerParams<MetadataType, OperationMetadataType, Permissions
           domain: SecurityDomain
         }
         daoGenericsMap: DAOGenericsMap<MetadataType, OperationMetadataType>
+        mongodbIndexes: T.MongoDBIndexes<AST, ScalarsSpecification>
       }`
 
     return [[entityManagerParamsExport, entityManagerMiddleware, entityManagerExport, utilsCode, utilsType].join('\n')].join('\n')

src/index.ts

@@ -41,3 +41,4 @@ export * from './dal/drivers/in-memory/utils.memory'
 export * from './dal/drivers/in-memory/dao.memory'
 export * from './dal/drivers/in-memory/adapters.memory'
 export * from './dal/drivers/in-memory/dao.memory.types'
+export * from './dal/dao/indexes/indexes.types'

tests/id-generator/dao.mock.ts

@@ -22,6 +22,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'mongodb'
       rawFilter: () => M.Filter<M.Document>
       rawUpdate: () => M.UpdateFilter<M.Document>
       rawSorts: () => M.Sort
@@ -33,6 +34,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'mongodb'
       rawFilter: () => M.Filter<M.Document>
       rawUpdate: () => M.UpdateFilter<M.Document>
       rawSorts: () => M.Sort
@@ -44,6 +46,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'mongodb'
       rawFilter: () => M.Filter<M.Document>
       rawUpdate: () => M.UpdateFilter<M.Document>
       rawSorts: () => M.Sort
@@ -55,6 +58,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'knexjs'
       rawFilter: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawUpdate: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawSorts: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
@@ -66,6 +70,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'knexjs'
       rawFilter: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawUpdate: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawSorts: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
@@ -77,6 +82,7 @@ export type AST = {
       value: { type: 'scalar'; isList: false; astName: 'Int'; isRequired: true; isListElementRequired: false; isExcluded: false; isId: false; generationStrategy: 'undefined' }
     }
     driverSpecification: {
+      type: 'knexjs'
       rawFilter: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawUpdate: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
       rawSorts: (builder: Knex.QueryBuilder<any, any>) => Knex.QueryBuilder<any, any>
@@ -827,6 +833,14 @@ export class EntityManager<
     )
   }
 
+  public planIndexes(args: { indexes: Partial<T.Indexes<AST, ScalarsSpecification>> }): Promise<T.IndexesPlanResults> {
+    return super.planIndexes(args, schemas)
+  }
+
+  public applyIndexes(args: ({ plan: T.IndexesPlanResults } | { indexes: Partial<T.Indexes<AST, ScalarsSpecification>> }) & { logs?: boolean }): Promise<T.IndexesApplyResults> {
+    return super.applyIndexes(args, schemas)
+  }
+
   public clone(): this {
     return new EntityManager<MetadataType, OperationMetadataType, Permissions, SecurityDomain>(this.params) as this
   }
@@ -920,4 +934,5 @@ export type EntityManagerTypes<MetadataType = never, OperationMetadataType = nev
     domain: SecurityDomain
   }
   daoGenericsMap: DAOGenericsMap<MetadataType, OperationMetadataType>
+  mongodbIndexes: T.MongoDBIndexes<AST, ScalarsSpecification>
 }