From 314cdb7ddfd8ee619e4af70c795b9a9d4c502a7a Mon Sep 17 00:00:00 2001 From: silto Date: Sat, 1 Jun 2024 01:53:52 +0200 Subject: [PATCH 1/8] add basis for partial dump script --- libs/back/partial-backup/.eslintrc.json | 18 ++ .../libs/back/partial-backup/package.json | 6 + libs/back/partial-backup/project.json | 53 ++++++ libs/back/partial-backup/src/main.ts | 167 ++++++++++++++++++ libs/back/partial-backup/tsconfig.app.json | 9 + libs/back/partial-backup/tsconfig.json | 13 ++ 6 files changed, 266 insertions(+) create mode 100644 libs/back/partial-backup/.eslintrc.json create mode 100644 libs/back/partial-backup/dist/libs/back/partial-backup/package.json create mode 100644 libs/back/partial-backup/project.json create mode 100644 libs/back/partial-backup/src/main.ts create mode 100644 libs/back/partial-backup/tsconfig.app.json create mode 100644 libs/back/partial-backup/tsconfig.json diff --git a/libs/back/partial-backup/.eslintrc.json b/libs/back/partial-backup/.eslintrc.json new file mode 100644 index 0000000000..3456be9b90 --- /dev/null +++ b/libs/back/partial-backup/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/back/partial-backup/dist/libs/back/partial-backup/package.json b/libs/back/partial-backup/dist/libs/back/partial-backup/package.json new file mode 100644 index 0000000000..856e84d80f --- /dev/null +++ b/libs/back/partial-backup/dist/libs/back/partial-backup/package.json @@ -0,0 +1,6 @@ +{ + "name": "partial-backup", + "version": "0.0.1", + "main": "./main.js", + "type": "commonjs" +} diff --git a/libs/back/partial-backup/project.json b/libs/back/partial-backup/project.json new file mode 100644 index 0000000000..4ae10abfdb --- /dev/null +++ b/libs/back/partial-backup/project.json @@ -0,0 +1,53 @@ +{ + "name": "partial-backup", + "$schema": "../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/back/partial-backup/src", + "projectType": "application", + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "production", + "options": { + "platform": "node", + "outputPath": "dist/libs/back/partial-backup", + "format": ["cjs"], + "bundle": false, + "main": "libs/back/partial-backup/src/main.ts", + "tsConfig": "libs/back/partial-backup/tsconfig.app.json", + "assets": ["libs/back/partial-backup/src/assets"], + "generatePackageJson": true, + "esbuildOptions": { + "sourcemap": true, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": {}, + "production": { + "esbuildOptions": { + "sourcemap": false, + "outExtension": { + ".js": ".js" + } + } + } + } + }, + "run": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "options": { + "buildTarget": "partial-backup:build" + }, + "configurations": { + "development": { + "buildTarget": "partial-backup:build:development" + } + } + } + }, + "tags": [] +} diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts new file mode 100644 index 0000000000..06b2b7175f --- /dev/null +++ b/libs/back/partial-backup/src/main.ts @@ -0,0 +1,167 @@ +import { unescape } from "node:querystring"; +import { Company, Form, Prisma, PrismaClient } from "@prisma/client"; + +enum BsdType { + Bsda = "BSDA", + Bsdasri = "BSDASRI", + Bsdd = "BSDD", + Bsff = "BSFF", + Bspaoh = "BSPAOH", + Bsvhu = "BSVHU" +} + +const { DATABASE_URL, TUNNELED_DB, DUMP_OBJ } = process.env; + +if (!DATABASE_URL) { + throw new Error("DATABASE_URL is not defined"); +} + +if (!TUNNELED_DB) { + throw new Error("TUNNELED_DB is not defined"); +} + +if (!DUMP_OBJ) { + throw new Error( + "DUMP_OBJ is not defined, please specify an object t act as the dump starting point" + ); +} + +function getDbUrlWithSchema(rawDatabaseUrl: string) { + try { + const dbUrl = new URL(rawDatabaseUrl); + dbUrl.searchParams.set("schema", "default$default"); + + return unescape(dbUrl.href); // unescape needed because of the `$` + } catch (err) { + return ""; + } +} + +const prismaLocal = new PrismaClient({ + datasources: { + db: { url: getDbUrlWithSchema(DATABASE_URL) } + }, + log: ["query", "info", "warn", "error"] +}); + +const prismaRemote = new PrismaClient({ + datasources: { + db: { url: getDbUrlWithSchema(TUNNELED_DB) } + }, + log: ["query", "info", "warn", "error"] +}); + +const pipelines = { + Form: { + getter: async (bsdId: string) => + bsdId && prismaRemote.form.findFirst({ where: { readableId: bsdId } }), + setter: async (bsd?: Form) => + bsd && + prismaLocal.form.create({ + data: { + ...bsd, + wasteDetailsPackagingInfos: + bsd.wasteDetailsPackagingInfos ?? Prisma.JsonNull, + wasteDetailsParcelNumbers: + bsd.wasteDetailsParcelNumbers ?? Prisma.JsonNull + } + }) + }, + Company: { + getter: async (siret?: string) => + siret && prismaRemote.company.findFirst({ where: { orgId: siret } }), + setter: async (company?: Company) => + company && prismaLocal.company.create({ data: company }) + } +}; + +const origins = { + [BsdType.Bsdd]: { + type: "Form", + getter: async (bsdId: string) => + bsdId && prismaRemote.form.findFirst({ where: { readableId: bsdId } }), + setter: async (bsd?: Form) => + bsd && + prismaLocal.form.create({ + data: { + ...bsd, + wasteDetailsPackagingInfos: + bsd.wasteDetailsPackagingInfos ?? Prisma.JsonNull, + wasteDetailsParcelNumbers: + bsd.wasteDetailsParcelNumbers ?? Prisma.JsonNull + } + }) + } +}; + +const traversals = { + Form: [ + { + type: "Company", + localKey: "emitterCompanySiret" + }, + { + type: "Company", + localKey: "recipientCompanySiret" + }, + { + type: "Company", + localKey: "traderCompanySiret" + }, + { + type: "Company", + localKey: "ecoOrganismeSiret" + }, + { + type: "Company", + localKey: "brokerCompanySiret" + }, + { + type: "Company", + localKey: "nextDestinationCompanySiret" + } + ] +}; + +const run = async () => { + let objectType: BsdType; + + const objType = DUMP_OBJ.split("-")?.[0]; + + if (!objType) { + throw new Error("DUMP_OBJ is not a valid BSD id"); + } + + switch (objType) { + case "BSD": + objectType = BsdType.Bsdd; + break; + case "DASRI": + objectType = BsdType.Bsdasri; + break; + case "BSDA": + objectType = BsdType.Bsda; + break; + case "FF": + objectType = BsdType.Bsff; + break; + case "PAOH": + objectType = BsdType.Bspaoh; + break; + case "VHU": + objectType = BsdType.Bsvhu; + break; + default: + throw new Error("DUMP_OBJ is not a valid BSD id"); + } + const bsd = await pipelines[origins[objectType].type].getter(DUMP_OBJ); + await pipelines[origins[objectType].type].setter(bsd); + for (const item of traversals[origins[objectType].type]) { + const getter = pipelines[item.type]?.getter; + const setter = pipelines[item.type]?.setter; + const obj = await getter?.(bsd[item.localKey]); + await setter?.(obj); + } +}; + +run(); diff --git a/libs/back/partial-backup/tsconfig.app.json b/libs/back/partial-backup/tsconfig.app.json new file mode 100644 index 0000000000..762205a8de --- /dev/null +++ b/libs/back/partial-backup/tsconfig.app.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../dist/out-tsc", + "module": "commonjs", + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/libs/back/partial-backup/tsconfig.json b/libs/back/partial-backup/tsconfig.json new file mode 100644 index 0000000000..92de93e5cb --- /dev/null +++ b/libs/back/partial-backup/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} From c4f22fdc7606b2b117f59f7175f9604ced63b6de Mon Sep 17 00:00:00 2001 From: silto Date: Sun, 2 Jun 2024 03:22:27 +0200 Subject: [PATCH 2/8] feat(scripts): create partial dump script --- libs/back/partial-backup/src/main.ts | 1468 +++++++++++++++++++++++++- 1 file changed, 1417 insertions(+), 51 deletions(-) diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts index 06b2b7175f..dc751c9224 100644 --- a/libs/back/partial-backup/src/main.ts +++ b/libs/back/partial-backup/src/main.ts @@ -1,14 +1,53 @@ import { unescape } from "node:querystring"; -import { Company, Form, Prisma, PrismaClient } from "@prisma/client"; - -enum BsdType { - Bsda = "BSDA", - Bsdasri = "BSDASRI", - Bsdd = "BSDD", - Bsff = "BSFF", - Bspaoh = "BSPAOH", - Bsvhu = "BSVHU" -} +import { + AccessToken, + AnonymousCompany, + Application, + BrokerReceipt, + Bsda, + BsdaFinalOperation, + BsdaRevisionRequest, + BsdaRevisionRequestApproval, + BsdaTransporter, + Bsdasri, + BsdasriFinalOperation, + BsdasriRevisionRequest, + BsdasriRevisionRequestApproval, + BsddFinalOperation, + BsddRevisionRequest, + BsddRevisionRequestApproval, + BsddTransporter, + Bsff, + BsffFicheIntervention, + BsffPackaging, + BsffPackagingFinalOperation, + BsffTransporter, + Bspaoh, + BspaohTransporter, + Company, + CompanyAssociation, + EcoOrganisme, + FeatureFlag, + Form, + FormGroupement, + GovernmentAccount, + Grant, + IntermediaryBsdaAssociation, + IntermediaryFormAssociation, + MembershipRequest, + Prisma, + PrismaClient, + SignatureAutomation, + StatusLog, + TraderReceipt, + TransporterReceipt, + User, + UserActivationHash, + UserResetPasswordHash, + VhuAgrement, + WebhookSetting, + WorkerCertification +} from "@prisma/client"; const { DATABASE_URL, TUNNELED_DB, DUMP_OBJ } = process.env; @@ -41,20 +80,20 @@ const prismaLocal = new PrismaClient({ datasources: { db: { url: getDbUrlWithSchema(DATABASE_URL) } }, - log: ["query", "info", "warn", "error"] + log: ["info", "warn", "error"] }); const prismaRemote = new PrismaClient({ datasources: { db: { url: getDbUrlWithSchema(TUNNELED_DB) } }, - log: ["query", "info", "warn", "error"] + log: ["info", "warn", "error"] }); const pipelines = { Form: { - getter: async (bsdId: string) => - bsdId && prismaRemote.form.findFirst({ where: { readableId: bsdId } }), + getter: async (key: string, value?: string) => + value && prismaRemote.form.findMany({ where: { [key]: value } }), setter: async (bsd?: Form) => bsd && prismaLocal.form.create({ @@ -67,30 +106,438 @@ const pipelines = { } }) }, + Bsdasri: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsdasri.findMany({ where: { [key]: value } }), + setter: async (bsdasri?: Bsdasri) => + bsdasri && + prismaLocal.bsdasri.create({ + data: { + ...bsdasri, + emitterWastePackagings: + bsdasri.emitterWastePackagings ?? Prisma.JsonNull, + transporterWastePackagings: + bsdasri.transporterWastePackagings ?? Prisma.JsonNull, + destinationWastePackagings: + bsdasri.destinationWastePackagings ?? Prisma.JsonNull + } + }) + }, + Bsda: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsda.findMany({ where: { [key]: value } }), + setter: async (bsda?: Bsda) => + bsda && + prismaLocal.bsda.create({ + data: { + ...bsda, + packagings: bsda.packagings ?? Prisma.JsonNull + } + }) + }, + Bsff: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsff.findMany({ where: { [key]: value } }), + setter: async (bsff?: Bsff) => + bsff && prismaLocal.bsff.create({ data: bsff }) + }, + Bspaoh: { + getter: async (key: string, value?: string) => + value && prismaRemote.bspaoh.findMany({ where: { [key]: value } }), + setter: async (bspaoh?: Bspaoh) => + bspaoh && + prismaLocal.bspaoh.create({ + data: { + ...bspaoh, + wastePackagings: bspaoh.wastePackagings ?? Prisma.JsonNull, + destinationReceptionWastePackagingsAcceptation: + bspaoh.destinationReceptionWastePackagingsAcceptation ?? + Prisma.JsonNull + } + }) + }, Company: { - getter: async (siret?: string) => - siret && prismaRemote.company.findFirst({ where: { orgId: siret } }), + getter: async (key: string, value?: string) => + value && prismaRemote.company.findMany({ where: { [key]: value } }), setter: async (company?: Company) => company && prismaLocal.company.create({ data: company }) - } -}; - -const origins = { - [BsdType.Bsdd]: { - type: "Form", - getter: async (bsdId: string) => - bsdId && prismaRemote.form.findFirst({ where: { readableId: bsdId } }), - setter: async (bsd?: Form) => - bsd && - prismaLocal.form.create({ + }, + AnonymousCompany: { + getter: async (key: string, value?: string) => + value && + prismaRemote.anonymousCompany.findMany({ where: { [key]: value } }), + setter: async (anonymousCompany?: AnonymousCompany) => + anonymousCompany && + prismaLocal.anonymousCompany.create({ data: anonymousCompany }) + }, + EcoOrganisme: { + getter: async (key: string, value?: string) => + value && prismaRemote.ecoOrganisme.findMany({ where: { [key]: value } }), + setter: async (ecoOrganisme?: EcoOrganisme) => + ecoOrganisme && prismaLocal.ecoOrganisme.create({ data: ecoOrganisme }) + }, + BsddTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddTransporter.findMany({ where: { [key]: value } }), + setter: async (transporter?: BsddTransporter) => + transporter && prismaLocal.bsddTransporter.create({ data: transporter }) + }, + FormGroupement: { + getter: async (key: string, value?: string) => + value && + prismaRemote.formGroupement.findMany({ where: { [key]: value } }), + setter: async (groupement?: FormGroupement) => + groupement && prismaLocal.formGroupement.create({ data: groupement }) + }, + User: { + getter: async (key: string, value?: string) => + value && prismaRemote.user.findMany({ where: { [key]: value } }), + setter: async (user?: User) => + user && prismaLocal.user.create({ data: user }) + }, + StatusLog: { + getter: async (key: string, value?: string) => + value && prismaRemote.statusLog.findMany({ where: { [key]: value } }), + setter: async (statusLog?: StatusLog) => + statusLog && + prismaLocal.statusLog.create({ data: { - ...bsd, + ...statusLog, + updatedFields: statusLog.updatedFields ?? Prisma.JsonNull + } + }) + }, + BsddRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddRevisionRequest.findMany({ where: { [key]: value } }), + setter: async (revisionRequest?: BsddRevisionRequest) => + revisionRequest && + prismaLocal.bsddRevisionRequest.create({ + data: { + ...revisionRequest, wasteDetailsPackagingInfos: - bsd.wasteDetailsPackagingInfos ?? Prisma.JsonNull, - wasteDetailsParcelNumbers: - bsd.wasteDetailsParcelNumbers ?? Prisma.JsonNull + revisionRequest.wasteDetailsPackagingInfos ?? Prisma.JsonNull + } + }) + }, + IntermediaryFormAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.intermediaryFormAssociation.findMany({ + where: { [key]: value } + }), + setter: async (intermediaryFormAssociation?: IntermediaryFormAssociation) => + intermediaryFormAssociation && + prismaLocal.intermediaryFormAssociation.create({ + data: intermediaryFormAssociation + }) + }, + BsddFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddFinalOperation.findMany({ where: { [key]: value } }), + setter: async (bsddFinalOperation?: BsddFinalOperation) => + bsddFinalOperation && + prismaLocal.bsddFinalOperation.create({ data: bsddFinalOperation }) + }, + TraderReceipt: { + getter: async (key: string, value?: string) => + value && prismaRemote.traderReceipt.findMany({ where: { [key]: value } }), + setter: async (traderReceipt?: TraderReceipt) => + traderReceipt && prismaLocal.traderReceipt.create({ data: traderReceipt }) + }, + BrokerReceipt: { + getter: async (key: string, value?: string) => + value && prismaRemote.brokerReceipt.findMany({ where: { [key]: value } }), + setter: async (brokerReceipt?: BrokerReceipt) => + brokerReceipt && prismaLocal.brokerReceipt.create({ data: brokerReceipt }) + }, + TransporterReceipt: { + getter: async (key: string, value?: string) => + value && + prismaRemote.transporterReceipt.findMany({ where: { [key]: value } }), + setter: async (transporterReceipt?: TransporterReceipt) => + transporterReceipt && + prismaLocal.transporterReceipt.create({ data: transporterReceipt }) + }, + VhuAgrement: { + getter: async (key: string, value?: string) => + value && prismaRemote.vhuAgrement.findMany({ where: { [key]: value } }), + setter: async (vhuAgrement?: VhuAgrement) => + vhuAgrement && prismaLocal.vhuAgrement.create({ data: vhuAgrement }) + }, + WorkerCertification: { + getter: async (key: string, value?: string) => + value && + prismaRemote.workerCertification.findMany({ where: { [key]: value } }), + setter: async (workerCertification?: WorkerCertification) => + workerCertification && + prismaLocal.workerCertification.create({ data: workerCertification }) + }, + CompanyAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.companyAssociation.findMany({ where: { [key]: value } }), + setter: async (companyAssociation?: CompanyAssociation) => + companyAssociation && + prismaLocal.companyAssociation.create({ data: companyAssociation }) + }, + MembershipRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.membershipRequest.findMany({ where: { [key]: value } }), + setter: async (membershipRequest?: MembershipRequest) => + membershipRequest && + prismaLocal.membershipRequest.create({ data: membershipRequest }) + }, + SignatureAutomation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.signatureAutomation.findMany({ where: { [key]: value } }), + setter: async (signatureAutomation?: SignatureAutomation) => + signatureAutomation && + prismaLocal.signatureAutomation.create({ data: signatureAutomation }) + }, + GovernmentAccount: { + getter: async (key: string, value?: string) => + value && + prismaRemote.governmentAccount.findMany({ where: { [key]: value } }), + setter: async (governmentAccount?: GovernmentAccount) => + governmentAccount && + prismaLocal.governmentAccount.create({ data: governmentAccount }) + }, + AccessToken: { + getter: async (key: string, value?: string) => + value && prismaRemote.accessToken.findMany({ where: { [key]: value } }), + setter: async (accessToken?: AccessToken) => + accessToken && prismaLocal.accessToken.create({ data: accessToken }) + }, + Application: { + getter: async (key: string, value?: string) => + value && prismaRemote.application.findMany({ where: { [key]: value } }), + setter: async (application?: Application) => + application && prismaLocal.application.create({ data: application }) + }, + FeatureFlag: { + getter: async (key: string, value?: string) => + value && prismaRemote.featureFlag.findMany({ where: { [key]: value } }), + setter: async (featureFlag?: FeatureFlag) => + featureFlag && prismaLocal.featureFlag.create({ data: featureFlag }) + }, + Grant: { + getter: async (key: string, value?: string) => + value && prismaRemote.grant.findMany({ where: { [key]: value } }), + setter: async (grant?: Grant) => + grant && prismaLocal.grant.create({ data: grant }) + }, + UserResetPasswordHash: { + getter: async (key: string, value?: string) => + value && + prismaRemote.userResetPasswordHash.findMany({ where: { [key]: value } }), + setter: async (userResetPasswordHash?: UserResetPasswordHash) => + userResetPasswordHash && + prismaLocal.userResetPasswordHash.create({ data: userResetPasswordHash }) + }, + UserActivationHash: { + getter: async (key: string, value?: string) => + value && + prismaRemote.userActivationHash.findMany({ where: { [key]: value } }), + setter: async (userActivationHash?: UserActivationHash) => + userActivationHash && + prismaLocal.userActivationHash.create({ data: userActivationHash }) + }, + BsddRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async (bsddRevisionRequestApproval?: BsddRevisionRequestApproval) => + bsddRevisionRequestApproval && + prismaLocal.bsddRevisionRequestApproval.create({ + data: bsddRevisionRequestApproval + }) + }, + BsdasriRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriRevisionRequest.findMany({ + where: { [key]: value } + }), + setter: async (bsdasriRevisionRequest?: BsdasriRevisionRequest) => + bsdasriRevisionRequest && + prismaLocal.bsdasriRevisionRequest.create({ + data: { + ...bsdasriRevisionRequest, + destinationWastePackagings: + bsdasriRevisionRequest.destinationWastePackagings ?? Prisma.JsonNull } }) + }, + BsdasriFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsdasriFinalOperation?: BsdasriFinalOperation) => + bsdasriFinalOperation && + prismaLocal.bsdasriFinalOperation.create({ + data: bsdasriFinalOperation + }) + }, + BsdasriRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async ( + bsdasriRevisionRequestApproval?: BsdasriRevisionRequestApproval + ) => + bsdasriRevisionRequestApproval && + prismaLocal.bsdasriRevisionRequestApproval.create({ + data: bsdasriRevisionRequestApproval + }) + }, + BsdaTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bsdaTransporter?: BsdaTransporter) => + bsdaTransporter && + prismaLocal.bsdaTransporter.create({ + data: bsdaTransporter + }) + }, + BsdaRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaRevisionRequest.findMany({ + where: { [key]: value } + }), + setter: async (bsdaRevisionRequest?: BsdaRevisionRequest) => + bsdaRevisionRequest && + prismaLocal.bsdaRevisionRequest.create({ + data: { + ...bsdaRevisionRequest, + packagings: bsdaRevisionRequest.packagings ?? Prisma.JsonNull + } + }) + }, + IntermediaryBsdaAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.intermediaryBsdaAssociation.findMany({ + where: { [key]: value } + }), + setter: async (intermediaryBsdaAssociation?: IntermediaryBsdaAssociation) => + intermediaryBsdaAssociation && + prismaLocal.intermediaryBsdaAssociation.create({ + data: intermediaryBsdaAssociation + }) + }, + BsdaFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsdaFinalOperation?: BsdaFinalOperation) => + bsdaFinalOperation && + prismaLocal.bsdaFinalOperation.create({ + data: bsdaFinalOperation + }) + }, + BsdaRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async (bsdaRevisionRequestApproval?: BsdaRevisionRequestApproval) => + bsdaRevisionRequestApproval && + prismaLocal.bsdaRevisionRequestApproval.create({ + data: bsdaRevisionRequestApproval + }) + }, + BsffFicheIntervention: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsff + .findUnique({ + where: { id: value } + }) + .ficheInterventions(), + setter: async (bsffFicheIntervention?: BsffFicheIntervention) => + bsffFicheIntervention && + prismaLocal.bsffFicheIntervention.create({ + data: bsffFicheIntervention + }) + }, + BsffPackaging: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffPackaging.findMany({ + where: { [key]: value } + }), + setter: async (bsffPackaging?: BsffPackaging) => + bsffPackaging && + prismaLocal.bsffPackaging.create({ + data: bsffPackaging + }) + }, + BsffTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bsffTransporter?: BsffTransporter) => + bsffTransporter && + prismaLocal.bsffTransporter.create({ + data: bsffTransporter + }) + }, + BsffPackagingFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffPackagingFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsffPackagingFinalOperation?: BsffPackagingFinalOperation) => + bsffPackagingFinalOperation && + prismaLocal.bsffPackagingFinalOperation.create({ + data: bsffPackagingFinalOperation + }) + }, + BspaohTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bspaohTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bspaohTransporter?: BspaohTransporter) => + bspaohTransporter && + prismaLocal.bspaohTransporter.create({ + data: bspaohTransporter + }) + }, + WebhookSetting: { + getter: async (key: string, value?: string) => + value && + prismaRemote.webhookSetting.findMany({ + where: { [key]: value } + }), + setter: async (webhookSetting?: WebhookSetting) => + webhookSetting && + prismaLocal.webhookSetting.create({ + data: webhookSetting + }) } }; @@ -98,35 +545,851 @@ const traversals = { Form: [ { type: "Company", - localKey: "emitterCompanySiret" + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "recipientCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "nextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "recipientCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "nextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsddTransporter", + localKey: "id", + foreignKey: "formId" + }, + { + type: "FormGroupement", + localKey: "id", + foreignKey: "nextFormId" + }, + { + type: "FormGroupement", + localKey: "id", + foreignKey: "initialFormId" + }, + { + type: "User", + localKey: "ownerId", + foreignKey: "id" + }, + { + type: "StatusLog", + localKey: "id", + foreignKey: "formId" + }, + { + type: "BsddRevisionRequest", + localKey: "id", + foreignKey: "bsddId" + }, + { + type: "IntermediaryFormAssociation", + localKey: "id", + foreignKey: "formId" + }, + { + type: "Form", + localKey: "forwardedInId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "id", + foreignKey: "forwardedInId" + }, + { + type: "BsddFinalOperation", + localKey: "id", + foreignKey: "finalFormId" + }, + { + type: "BsddFinalOperation", + localKey: "id", + foreignKey: "initialFormId" + } + ], + Bsdasri: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "User", + localKey: "emissionSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "transportSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "receptionSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "operationSignatoryId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "groupedInId", + foreignKey: "id" }, + { + type: "Bsdasri", + localKey: "id", + foreignKey: "groupedInId" + }, + { + type: "Bsdasri", + localKey: "synthesizedInId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "id", + foreignKey: "synthesizedInId" + }, + { + type: "BsdasriRevisionRequest", + localKey: "id", + foreignKey: "bsdasriId" + }, + { + type: "BsdasriFinalOperation", + localKey: "id", + foreignKey: "finalBsdasriId" + }, + { + type: "BsdasriFinalOperation", + localKey: "id", + foreignKey: "initialBsdasriId" + } + ], + Bsda: [ { type: "Company", - localKey: "recipientCompanySiret" + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" }, { type: "Company", - localKey: "traderCompanySiret" + localKey: "destinationCompanySiret", + foreignKey: "orgId" }, { type: "Company", - localKey: "ecoOrganismeSiret" + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsdaTransporter", + localKey: "id", + foreignKey: "bsdaId" }, { type: "Company", - localKey: "brokerCompanySiret" + localKey: "workerCompanySiret", + foreignKey: "orgId" }, { type: "Company", - localKey: "nextDestinationCompanySiret" + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "workerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "Bsda", + localKey: "forwardingId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "id", + foreignKey: "forwardingId" + }, + { + type: "Bsda", + localKey: "groupedInId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "id", + foreignKey: "groupedInId" + }, + { + type: "BsdaRevisionRequest", + localKey: "id", + foreignKey: "bsdaId" + }, + { + type: "IntermediaryBsdaAssociation", + localKey: "id", + foreignKey: "bsdaId" + }, + { + type: "BsdaFinalOperation", + localKey: "id", + foreignKey: "finalBsdaId" + }, + { + type: "BsdaFinalOperation", + localKey: "id", + foreignKey: "initialBsdaId" + } + ], + Bsff: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsffFicheIntervention", + localKey: "id", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "id", + foreignKey: "bsffId" + }, + { + type: "BsffTransporter", + localKey: "id", + foreignKey: "bsffId" + } + ], + Bspaoh: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BspaohTransporter", + localKey: "id", + foreignKey: "bspaohId" + } + ], + Bsvhu: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + Company: [ + { + type: "TraderReceipt", + localKey: "traderReceiptId", + foreignKey: "id" + }, + { + type: "BrokerReceipt", + localKey: "brokerReceiptId", + foreignKey: "id" + }, + { + type: "TransporterReceipt", + localKey: "transporterReceiptId", + foreignKey: "id" + }, + { + type: "VhuAgrement", + localKey: "vhuAgrementDemolisseurId", + foreignKey: "id" + }, + { + type: "VhuAgrement", + localKey: "vhuAgrementBroyeurId", + foreignKey: "id" + }, + { + type: "WorkerCertification", + localKey: "workerCertificationId", + foreignKey: "id" + }, + { + type: "CompanyAssociation", + localKey: "id", + foreignKey: "companyId" + }, + { + type: "MembershipRequest", + localKey: "id", + foreignKey: "companyId" + }, + { + type: "SignatureAutomation", + localKey: "id", + foreignKey: "fromId" + }, + { + type: "SignatureAutomation", + localKey: "id", + foreignKey: "toId" + }, + { + type: "WebhookSetting", + localKey: "orgId", + foreignKey: "orgId" + } + ], + BsddTransporter: [], + FormGroupement: [ + { + type: "Form", + localKey: "nextFormId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "initialFormId", + foreignKey: "id" + } + ], + User: [ + { + type: "GovernmentAccount", + localKey: "governmentAccountId", + foreignKey: "id" + }, + { + type: "AccessToken", + localKey: "id", + foreignKey: "userId" + }, + { + type: "Application", + localKey: "id", + foreignKey: "adminId" + }, + { + type: "CompanyAssociation", + localKey: "id", + foreignKey: "userId" + }, + { + type: "FeatureFlag", + localKey: "id", + foreignKey: "userId" + }, + { + type: "Grant", + localKey: "id", + foreignKey: "userId" + }, + { + type: "MembershipRequest", + localKey: "id", + foreignKey: "userId" + }, + // { + // type: "StatusLog", + // localKey: "id", + // foreignKey: "userId" + // }, + { + type: "UserResetPasswordHash", + localKey: "id", + foreignKey: "userId" + }, + { + type: "UserActivationHash", + localKey: "id", + foreignKey: "userId" + } + ], + StatusLog: [], + BsddRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsddRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + }, + { + type: "Company", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + } + ], + IntermediaryFormAssociation: [ + { + type: "Company", + localKey: "siret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "siret", + foreignKey: "orgId" + } + ], + BsddFinalOperation: [ + { + type: "Form", + localKey: "finalFormId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "initialFormId", + foreignKey: "id" + } + ], + TraderReceipt: [], + BrokerReceipt: [], + TransporterReceipt: [], + VhuAgrement: [], + WorkerCertification: [], + CompanyAssociation: [ + { + type: "Company", + localKey: "companyId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + MembershipRequest: [ + { + type: "Company", + localKey: "companyId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + SignatureAutomation: [ + { + type: "Company", + localKey: "fromId", + foreignKey: "id" + }, + { + type: "Company", + localKey: "toId", + foreignKey: "id" + } + ], + GovernmentAccount: [], + AccessToken: [ + { + type: "Application", + localKey: "applicationId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + Application: [ + { + type: "User", + localKey: "adminId", + foreignKey: "id" + }, + { + type: "AccessToken", + localKey: "id", + foreignKey: "applicationId" + }, + { + type: "Grant", + localKey: "id", + foreignKey: "applicationId" + } + ], + FeatureFlag: [], + Grant: [ + { + type: "Application", + localKey: "applicationId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + UserResetPasswordHash: [], + UserActivationHash: [], + BsddRevisionRequestApproval: [], + BsdasriRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsdasriRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + } + ], + BsdasriFinalOperation: [ + { + type: "Bsdasri", + localKey: "finalBsdasriId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "initialBsdasriId", + foreignKey: "id" + } + ], + BsdasriRevisionRequestApproval: [], + BsdaTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + BsdaRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsdaRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + } + ], + IntermediaryBsdaAssociation: [], + BsdaFinalOperation: [ + { + type: "Bsda", + localKey: "finalBsdaId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "initialBsdaId", + foreignKey: "id" + } + ], + BsdaRevisionRequestApproval: [], + BsffFicheIntervention: [ + { + type: "Company", + localKey: "detenteurCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "operateurCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "detenteurCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "operateurCompanySiret", + foreignKey: "orgId" + } + ], + BsffPackaging: [ + { + type: "Company", + localKey: "operationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "operationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsffPackaging", + localKey: "nextPackagingId", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "id", + foreignKey: "nextPackagingId" + }, + { + type: "BsffPackagingFinalOperation", + localKey: "id", + foreignKey: "finalBsffPackagingId" + }, + { + type: "BsffPackagingFinalOperation", + localKey: "id", + foreignKey: "initialBsffPackagingId" + } + ], + BsffTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + BsffPackagingFinalOperation: [ + { + type: "BsffPackaging", + localKey: "finalBsffPackagingId", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "initialBsffPackagingId", + foreignKey: "id" + } + ], + BspaohTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" } ] }; const run = async () => { - let objectType: BsdType; + let originType: "Form" | "Bsdasri" | "Bsda" | "Bsff" | "Bspaoh" | "Bsvhu"; const objType = DUMP_OBJ.split("-")?.[0]; + const alreadyFetched: { [key: string]: any } = {}; if (!objType) { throw new Error("DUMP_OBJ is not a valid BSD id"); @@ -134,33 +1397,136 @@ const run = async () => { switch (objType) { case "BSD": - objectType = BsdType.Bsdd; + originType = "Form"; break; case "DASRI": - objectType = BsdType.Bsdasri; + originType = "Bsdasri"; break; case "BSDA": - objectType = BsdType.Bsda; + originType = "Bsda"; break; case "FF": - objectType = BsdType.Bsff; + originType = "Bsff"; break; case "PAOH": - objectType = BsdType.Bspaoh; + originType = "Bspaoh"; break; case "VHU": - objectType = BsdType.Bsvhu; + originType = "Bsvhu"; break; default: throw new Error("DUMP_OBJ is not a valid BSD id"); } - const bsd = await pipelines[origins[objectType].type].getter(DUMP_OBJ); - await pipelines[origins[objectType].type].setter(bsd); - for (const item of traversals[origins[objectType].type]) { - const getter = pipelines[item.type]?.getter; - const setter = pipelines[item.type]?.setter; - const obj = await getter?.(bsd[item.localKey]); - await setter?.(obj); + type structItem = { + type: string; + obj: any; + path: string; + depth: number; + children: structItem[]; + }; + + const struct: structItem[] = []; + const bsds = await pipelines[originType].getter("readableId", DUMP_OBJ); + const bsd = bsds?.[0]; + if (!bsd) { + throw new Error("root BSD not found"); + } + const bsdRoot: structItem = { + type: originType, + obj: bsd, + path: `${originType}(${bsd.id})`, + depth: 0, + children: [] + }; + struct.push(bsdRoot); + alreadyFetched[bsd.id] = { + type: originType, + obj: bsd + }; + console.log("got BSD Root"); + console.log(bsdRoot.obj.id); + let maxDepth = 0; + + const recursiveExtract = async (root: structItem) => { + console.log(`TRAVERSING ${root.path}`); + if (root.depth > maxDepth) { + maxDepth = root.depth; + } + if (!traversals[root.type]) { + console.log(`TRAVERSAL NOT AVAILABLE FOR ${root.type}`); + return; + } + for (const item of traversals[root.type]) { + console.log( + `fetching ${item.type}, fKey: ${item.foreignKey}, lKey: ${ + root.obj[item.localKey] + }` + ); + const getter = pipelines[item.type]?.getter; + if (!getter) { + console.log(`MISSING GETTER ${item.type}`); + } + // const setter = pipelines[item.type]?.setter; + const objects = await getter?.(item.foreignKey, root.obj[item.localKey]); + const filteredObjects = objects?.filter(obj => { + if (alreadyFetched[obj.id]) { + return false; + } + alreadyFetched[obj.id] = { + type: item.type, + obj + }; + // alreadyFetched[obj.id] = true; + return true; + }); + if (filteredObjects?.length) { + const subRoots: structItem[] = filteredObjects.map(obj => ({ + type: item.type, + obj, + path: `${root.path}>${item.type}(${obj.id})`, + depth: root.depth + 1, + children: [] + })); + root.children = [...root.children, ...subRoots]; + } + } + for (const subRoot of root.children) { + await recursiveExtract(subRoot); + } + }; + await recursiveExtract(bsdRoot); + console.log(Object.keys(alreadyFetched)); + console.log("max depth: ", maxDepth); + const statsByType = {}; + for (const id of Object.keys(alreadyFetched)) { + if (!statsByType[alreadyFetched[id].type]) { + statsByType[alreadyFetched[id].type] = 1; + } else { + statsByType[alreadyFetched[id].type] += 1; + } + } + console.log(statsByType); + const alreadySaved: { [key: string]: boolean } = {}; + const allSaved = () => { + return !Object.keys(alreadyFetched).some(id => !alreadySaved[id]); + }; + while (!allSaved()) { + for (const id of Object.keys(alreadyFetched)) { + if (alreadySaved[id]) { + continue; + } + const setter = pipelines[alreadyFetched[id].type]?.setter; + if (!setter) { + throw new Error(`no setter for type ${alreadyFetched[id].type}`); + } + try { + await setter(alreadyFetched[id].obj); + alreadySaved[id] = true; + console.log(`saved ${alreadyFetched[id].type} ${id}`); + } catch (error) { + console.log(`could not save ${alreadyFetched[id].type} ${id}`); + } + } } }; From 96aa10dc4704bf04585b4344274bf6c2fc27406f Mon Sep 17 00:00:00 2001 From: silto Date: Mon, 3 Jun 2024 00:29:57 +0200 Subject: [PATCH 3/8] feat(scripts): finish partial dump script --- libs/back/partial-backup/project.json | 3 +- libs/back/partial-backup/src/main.ts | 1538 ++------------------ libs/back/partial-backup/src/pipelines.ts | 513 +++++++ libs/back/partial-backup/src/traversals.ts | 854 +++++++++++ 4 files changed, 1527 insertions(+), 1381 deletions(-) create mode 100644 libs/back/partial-backup/src/pipelines.ts create mode 100644 libs/back/partial-backup/src/traversals.ts diff --git a/libs/back/partial-backup/project.json b/libs/back/partial-backup/project.json index 4ae10abfdb..acbcb03ad3 100644 --- a/libs/back/partial-backup/project.json +++ b/libs/back/partial-backup/project.json @@ -40,7 +40,8 @@ "executor": "@nx/js:node", "defaultConfiguration": "development", "options": { - "buildTarget": "partial-backup:build" + "buildTarget": "partial-backup:build", + "watch": false }, "configurations": { "development": { diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts index dc751c9224..9dd807acad 100644 --- a/libs/back/partial-backup/src/main.ts +++ b/libs/back/partial-backup/src/main.ts @@ -1,55 +1,41 @@ import { unescape } from "node:querystring"; -import { - AccessToken, - AnonymousCompany, - Application, - BrokerReceipt, - Bsda, - BsdaFinalOperation, - BsdaRevisionRequest, - BsdaRevisionRequestApproval, - BsdaTransporter, - Bsdasri, - BsdasriFinalOperation, - BsdasriRevisionRequest, - BsdasriRevisionRequestApproval, - BsddFinalOperation, - BsddRevisionRequest, - BsddRevisionRequestApproval, - BsddTransporter, - Bsff, - BsffFicheIntervention, - BsffPackaging, - BsffPackagingFinalOperation, - BsffTransporter, - Bspaoh, - BspaohTransporter, - Company, - CompanyAssociation, - EcoOrganisme, - FeatureFlag, - Form, - FormGroupement, - GovernmentAccount, - Grant, - IntermediaryBsdaAssociation, - IntermediaryFormAssociation, - MembershipRequest, - Prisma, - PrismaClient, - SignatureAutomation, - StatusLog, - TraderReceipt, - TransporterReceipt, - User, - UserActivationHash, - UserResetPasswordHash, - VhuAgrement, - WebhookSetting, - WorkerCertification -} from "@prisma/client"; +import readLine from "node:readline"; +import getPipelines from "./pipelines"; +import traversals from "./traversals"; +import { PrismaClient } from "@prisma/client"; -const { DATABASE_URL, TUNNELED_DB, DUMP_OBJ } = process.env; +const { DATABASE_URL, TUNNELED_DB, ROOT_OBJ } = process.env; + +/* + Console utils +*/ + +const rl = readLine.createInterface({ + input: process.stdin, + output: process.stdout +}); + +const questionPromise = (question: string): Promise => { + return new Promise(resolve => { + rl.question(question, answer => { + resolve(answer); + }); + }); +}; + +const blank = "\n".repeat(process.stdout.rows); +console.log(blank); +readLine.cursorTo(process.stdout, 0, 0); +readLine.clearScreenDown(process.stdout); +const print = (info: string) => { + readLine.cursorTo(process.stdout, 0, 0); + readLine.clearScreenDown(process.stdout); + process.stdout.write(info); +}; + +/* + Database clients init +*/ if (!DATABASE_URL) { throw new Error("DATABASE_URL is not defined"); @@ -59,12 +45,6 @@ if (!TUNNELED_DB) { throw new Error("TUNNELED_DB is not defined"); } -if (!DUMP_OBJ) { - throw new Error( - "DUMP_OBJ is not defined, please specify an object t act as the dump starting point" - ); -} - function getDbUrlWithSchema(rawDatabaseUrl: string) { try { const dbUrl = new URL(rawDatabaseUrl); @@ -80,1324 +60,52 @@ const prismaLocal = new PrismaClient({ datasources: { db: { url: getDbUrlWithSchema(DATABASE_URL) } }, - log: ["info", "warn", "error"] + log: [] }); const prismaRemote = new PrismaClient({ datasources: { db: { url: getDbUrlWithSchema(TUNNELED_DB) } }, - log: ["info", "warn", "error"] + log: [] }); -const pipelines = { - Form: { - getter: async (key: string, value?: string) => - value && prismaRemote.form.findMany({ where: { [key]: value } }), - setter: async (bsd?: Form) => - bsd && - prismaLocal.form.create({ - data: { - ...bsd, - wasteDetailsPackagingInfos: - bsd.wasteDetailsPackagingInfos ?? Prisma.JsonNull, - wasteDetailsParcelNumbers: - bsd.wasteDetailsParcelNumbers ?? Prisma.JsonNull - } - }) - }, - Bsdasri: { - getter: async (key: string, value?: string) => - value && prismaRemote.bsdasri.findMany({ where: { [key]: value } }), - setter: async (bsdasri?: Bsdasri) => - bsdasri && - prismaLocal.bsdasri.create({ - data: { - ...bsdasri, - emitterWastePackagings: - bsdasri.emitterWastePackagings ?? Prisma.JsonNull, - transporterWastePackagings: - bsdasri.transporterWastePackagings ?? Prisma.JsonNull, - destinationWastePackagings: - bsdasri.destinationWastePackagings ?? Prisma.JsonNull - } - }) - }, - Bsda: { - getter: async (key: string, value?: string) => - value && prismaRemote.bsda.findMany({ where: { [key]: value } }), - setter: async (bsda?: Bsda) => - bsda && - prismaLocal.bsda.create({ - data: { - ...bsda, - packagings: bsda.packagings ?? Prisma.JsonNull - } - }) - }, - Bsff: { - getter: async (key: string, value?: string) => - value && prismaRemote.bsff.findMany({ where: { [key]: value } }), - setter: async (bsff?: Bsff) => - bsff && prismaLocal.bsff.create({ data: bsff }) - }, - Bspaoh: { - getter: async (key: string, value?: string) => - value && prismaRemote.bspaoh.findMany({ where: { [key]: value } }), - setter: async (bspaoh?: Bspaoh) => - bspaoh && - prismaLocal.bspaoh.create({ - data: { - ...bspaoh, - wastePackagings: bspaoh.wastePackagings ?? Prisma.JsonNull, - destinationReceptionWastePackagingsAcceptation: - bspaoh.destinationReceptionWastePackagingsAcceptation ?? - Prisma.JsonNull - } - }) - }, - Company: { - getter: async (key: string, value?: string) => - value && prismaRemote.company.findMany({ where: { [key]: value } }), - setter: async (company?: Company) => - company && prismaLocal.company.create({ data: company }) - }, - AnonymousCompany: { - getter: async (key: string, value?: string) => - value && - prismaRemote.anonymousCompany.findMany({ where: { [key]: value } }), - setter: async (anonymousCompany?: AnonymousCompany) => - anonymousCompany && - prismaLocal.anonymousCompany.create({ data: anonymousCompany }) - }, - EcoOrganisme: { - getter: async (key: string, value?: string) => - value && prismaRemote.ecoOrganisme.findMany({ where: { [key]: value } }), - setter: async (ecoOrganisme?: EcoOrganisme) => - ecoOrganisme && prismaLocal.ecoOrganisme.create({ data: ecoOrganisme }) - }, - BsddTransporter: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsddTransporter.findMany({ where: { [key]: value } }), - setter: async (transporter?: BsddTransporter) => - transporter && prismaLocal.bsddTransporter.create({ data: transporter }) - }, - FormGroupement: { - getter: async (key: string, value?: string) => - value && - prismaRemote.formGroupement.findMany({ where: { [key]: value } }), - setter: async (groupement?: FormGroupement) => - groupement && prismaLocal.formGroupement.create({ data: groupement }) - }, - User: { - getter: async (key: string, value?: string) => - value && prismaRemote.user.findMany({ where: { [key]: value } }), - setter: async (user?: User) => - user && prismaLocal.user.create({ data: user }) - }, - StatusLog: { - getter: async (key: string, value?: string) => - value && prismaRemote.statusLog.findMany({ where: { [key]: value } }), - setter: async (statusLog?: StatusLog) => - statusLog && - prismaLocal.statusLog.create({ - data: { - ...statusLog, - updatedFields: statusLog.updatedFields ?? Prisma.JsonNull - } - }) - }, - BsddRevisionRequest: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsddRevisionRequest.findMany({ where: { [key]: value } }), - setter: async (revisionRequest?: BsddRevisionRequest) => - revisionRequest && - prismaLocal.bsddRevisionRequest.create({ - data: { - ...revisionRequest, - wasteDetailsPackagingInfos: - revisionRequest.wasteDetailsPackagingInfos ?? Prisma.JsonNull - } - }) - }, - IntermediaryFormAssociation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.intermediaryFormAssociation.findMany({ - where: { [key]: value } - }), - setter: async (intermediaryFormAssociation?: IntermediaryFormAssociation) => - intermediaryFormAssociation && - prismaLocal.intermediaryFormAssociation.create({ - data: intermediaryFormAssociation - }) - }, - BsddFinalOperation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsddFinalOperation.findMany({ where: { [key]: value } }), - setter: async (bsddFinalOperation?: BsddFinalOperation) => - bsddFinalOperation && - prismaLocal.bsddFinalOperation.create({ data: bsddFinalOperation }) - }, - TraderReceipt: { - getter: async (key: string, value?: string) => - value && prismaRemote.traderReceipt.findMany({ where: { [key]: value } }), - setter: async (traderReceipt?: TraderReceipt) => - traderReceipt && prismaLocal.traderReceipt.create({ data: traderReceipt }) - }, - BrokerReceipt: { - getter: async (key: string, value?: string) => - value && prismaRemote.brokerReceipt.findMany({ where: { [key]: value } }), - setter: async (brokerReceipt?: BrokerReceipt) => - brokerReceipt && prismaLocal.brokerReceipt.create({ data: brokerReceipt }) - }, - TransporterReceipt: { - getter: async (key: string, value?: string) => - value && - prismaRemote.transporterReceipt.findMany({ where: { [key]: value } }), - setter: async (transporterReceipt?: TransporterReceipt) => - transporterReceipt && - prismaLocal.transporterReceipt.create({ data: transporterReceipt }) - }, - VhuAgrement: { - getter: async (key: string, value?: string) => - value && prismaRemote.vhuAgrement.findMany({ where: { [key]: value } }), - setter: async (vhuAgrement?: VhuAgrement) => - vhuAgrement && prismaLocal.vhuAgrement.create({ data: vhuAgrement }) - }, - WorkerCertification: { - getter: async (key: string, value?: string) => - value && - prismaRemote.workerCertification.findMany({ where: { [key]: value } }), - setter: async (workerCertification?: WorkerCertification) => - workerCertification && - prismaLocal.workerCertification.create({ data: workerCertification }) - }, - CompanyAssociation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.companyAssociation.findMany({ where: { [key]: value } }), - setter: async (companyAssociation?: CompanyAssociation) => - companyAssociation && - prismaLocal.companyAssociation.create({ data: companyAssociation }) - }, - MembershipRequest: { - getter: async (key: string, value?: string) => - value && - prismaRemote.membershipRequest.findMany({ where: { [key]: value } }), - setter: async (membershipRequest?: MembershipRequest) => - membershipRequest && - prismaLocal.membershipRequest.create({ data: membershipRequest }) - }, - SignatureAutomation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.signatureAutomation.findMany({ where: { [key]: value } }), - setter: async (signatureAutomation?: SignatureAutomation) => - signatureAutomation && - prismaLocal.signatureAutomation.create({ data: signatureAutomation }) - }, - GovernmentAccount: { - getter: async (key: string, value?: string) => - value && - prismaRemote.governmentAccount.findMany({ where: { [key]: value } }), - setter: async (governmentAccount?: GovernmentAccount) => - governmentAccount && - prismaLocal.governmentAccount.create({ data: governmentAccount }) - }, - AccessToken: { - getter: async (key: string, value?: string) => - value && prismaRemote.accessToken.findMany({ where: { [key]: value } }), - setter: async (accessToken?: AccessToken) => - accessToken && prismaLocal.accessToken.create({ data: accessToken }) - }, - Application: { - getter: async (key: string, value?: string) => - value && prismaRemote.application.findMany({ where: { [key]: value } }), - setter: async (application?: Application) => - application && prismaLocal.application.create({ data: application }) - }, - FeatureFlag: { - getter: async (key: string, value?: string) => - value && prismaRemote.featureFlag.findMany({ where: { [key]: value } }), - setter: async (featureFlag?: FeatureFlag) => - featureFlag && prismaLocal.featureFlag.create({ data: featureFlag }) - }, - Grant: { - getter: async (key: string, value?: string) => - value && prismaRemote.grant.findMany({ where: { [key]: value } }), - setter: async (grant?: Grant) => - grant && prismaLocal.grant.create({ data: grant }) - }, - UserResetPasswordHash: { - getter: async (key: string, value?: string) => - value && - prismaRemote.userResetPasswordHash.findMany({ where: { [key]: value } }), - setter: async (userResetPasswordHash?: UserResetPasswordHash) => - userResetPasswordHash && - prismaLocal.userResetPasswordHash.create({ data: userResetPasswordHash }) - }, - UserActivationHash: { - getter: async (key: string, value?: string) => - value && - prismaRemote.userActivationHash.findMany({ where: { [key]: value } }), - setter: async (userActivationHash?: UserActivationHash) => - userActivationHash && - prismaLocal.userActivationHash.create({ data: userActivationHash }) - }, - BsddRevisionRequestApproval: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsddRevisionRequestApproval.findMany({ - where: { [key]: value } - }), - setter: async (bsddRevisionRequestApproval?: BsddRevisionRequestApproval) => - bsddRevisionRequestApproval && - prismaLocal.bsddRevisionRequestApproval.create({ - data: bsddRevisionRequestApproval - }) - }, - BsdasriRevisionRequest: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdasriRevisionRequest.findMany({ - where: { [key]: value } - }), - setter: async (bsdasriRevisionRequest?: BsdasriRevisionRequest) => - bsdasriRevisionRequest && - prismaLocal.bsdasriRevisionRequest.create({ - data: { - ...bsdasriRevisionRequest, - destinationWastePackagings: - bsdasriRevisionRequest.destinationWastePackagings ?? Prisma.JsonNull - } - }) - }, - BsdasriFinalOperation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdasriFinalOperation.findMany({ - where: { [key]: value } - }), - setter: async (bsdasriFinalOperation?: BsdasriFinalOperation) => - bsdasriFinalOperation && - prismaLocal.bsdasriFinalOperation.create({ - data: bsdasriFinalOperation - }) - }, - BsdasriRevisionRequestApproval: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdasriRevisionRequestApproval.findMany({ - where: { [key]: value } - }), - setter: async ( - bsdasriRevisionRequestApproval?: BsdasriRevisionRequestApproval - ) => - bsdasriRevisionRequestApproval && - prismaLocal.bsdasriRevisionRequestApproval.create({ - data: bsdasriRevisionRequestApproval - }) - }, - BsdaTransporter: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdaTransporter.findMany({ - where: { [key]: value } - }), - setter: async (bsdaTransporter?: BsdaTransporter) => - bsdaTransporter && - prismaLocal.bsdaTransporter.create({ - data: bsdaTransporter - }) - }, - BsdaRevisionRequest: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdaRevisionRequest.findMany({ - where: { [key]: value } - }), - setter: async (bsdaRevisionRequest?: BsdaRevisionRequest) => - bsdaRevisionRequest && - prismaLocal.bsdaRevisionRequest.create({ - data: { - ...bsdaRevisionRequest, - packagings: bsdaRevisionRequest.packagings ?? Prisma.JsonNull - } - }) - }, - IntermediaryBsdaAssociation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.intermediaryBsdaAssociation.findMany({ - where: { [key]: value } - }), - setter: async (intermediaryBsdaAssociation?: IntermediaryBsdaAssociation) => - intermediaryBsdaAssociation && - prismaLocal.intermediaryBsdaAssociation.create({ - data: intermediaryBsdaAssociation - }) - }, - BsdaFinalOperation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdaFinalOperation.findMany({ - where: { [key]: value } - }), - setter: async (bsdaFinalOperation?: BsdaFinalOperation) => - bsdaFinalOperation && - prismaLocal.bsdaFinalOperation.create({ - data: bsdaFinalOperation - }) - }, - BsdaRevisionRequestApproval: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsdaRevisionRequestApproval.findMany({ - where: { [key]: value } - }), - setter: async (bsdaRevisionRequestApproval?: BsdaRevisionRequestApproval) => - bsdaRevisionRequestApproval && - prismaLocal.bsdaRevisionRequestApproval.create({ - data: bsdaRevisionRequestApproval - }) - }, - BsffFicheIntervention: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsff - .findUnique({ - where: { id: value } - }) - .ficheInterventions(), - setter: async (bsffFicheIntervention?: BsffFicheIntervention) => - bsffFicheIntervention && - prismaLocal.bsffFicheIntervention.create({ - data: bsffFicheIntervention - }) - }, - BsffPackaging: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsffPackaging.findMany({ - where: { [key]: value } - }), - setter: async (bsffPackaging?: BsffPackaging) => - bsffPackaging && - prismaLocal.bsffPackaging.create({ - data: bsffPackaging - }) - }, - BsffTransporter: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsffTransporter.findMany({ - where: { [key]: value } - }), - setter: async (bsffTransporter?: BsffTransporter) => - bsffTransporter && - prismaLocal.bsffTransporter.create({ - data: bsffTransporter - }) - }, - BsffPackagingFinalOperation: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bsffPackagingFinalOperation.findMany({ - where: { [key]: value } - }), - setter: async (bsffPackagingFinalOperation?: BsffPackagingFinalOperation) => - bsffPackagingFinalOperation && - prismaLocal.bsffPackagingFinalOperation.create({ - data: bsffPackagingFinalOperation - }) - }, - BspaohTransporter: { - getter: async (key: string, value?: string) => - value && - prismaRemote.bspaohTransporter.findMany({ - where: { [key]: value } - }), - setter: async (bspaohTransporter?: BspaohTransporter) => - bspaohTransporter && - prismaLocal.bspaohTransporter.create({ - data: bspaohTransporter - }) - }, - WebhookSetting: { - getter: async (key: string, value?: string) => - value && - prismaRemote.webhookSetting.findMany({ - where: { [key]: value } - }), - setter: async (webhookSetting?: WebhookSetting) => - webhookSetting && - prismaLocal.webhookSetting.create({ - data: webhookSetting - }) - } -}; - -const traversals = { - Form: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "recipientCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "traderCompanySiret", - foreignKey: "orgId" - }, - { - type: "EcoOrganisme", - localKey: "ecoOrganismeSiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "nextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "recipientCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "traderCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "nextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "BsddTransporter", - localKey: "id", - foreignKey: "formId" - }, - { - type: "FormGroupement", - localKey: "id", - foreignKey: "nextFormId" - }, - { - type: "FormGroupement", - localKey: "id", - foreignKey: "initialFormId" - }, - { - type: "User", - localKey: "ownerId", - foreignKey: "id" - }, - { - type: "StatusLog", - localKey: "id", - foreignKey: "formId" - }, - { - type: "BsddRevisionRequest", - localKey: "id", - foreignKey: "bsddId" - }, - { - type: "IntermediaryFormAssociation", - localKey: "id", - foreignKey: "formId" - }, - { - type: "Form", - localKey: "forwardedInId", - foreignKey: "id" - }, - { - type: "Form", - localKey: "id", - foreignKey: "forwardedInId" - }, - { - type: "BsddFinalOperation", - localKey: "id", - foreignKey: "finalFormId" - }, - { - type: "BsddFinalOperation", - localKey: "id", - foreignKey: "initialFormId" - } - ], - Bsdasri: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "EcoOrganisme", - localKey: "ecoOrganismeSiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "User", - localKey: "emissionSignatoryId", - foreignKey: "id" - }, - { - type: "User", - localKey: "transportSignatoryId", - foreignKey: "id" - }, - { - type: "User", - localKey: "receptionSignatoryId", - foreignKey: "id" - }, - { - type: "User", - localKey: "operationSignatoryId", - foreignKey: "id" - }, - { - type: "Bsdasri", - localKey: "groupedInId", - foreignKey: "id" - }, - { - type: "Bsdasri", - localKey: "id", - foreignKey: "groupedInId" - }, - { - type: "Bsdasri", - localKey: "synthesizedInId", - foreignKey: "id" - }, - { - type: "Bsdasri", - localKey: "id", - foreignKey: "synthesizedInId" - }, - { - type: "BsdasriRevisionRequest", - localKey: "id", - foreignKey: "bsdasriId" - }, - { - type: "BsdasriFinalOperation", - localKey: "id", - foreignKey: "finalBsdasriId" - }, - { - type: "BsdasriFinalOperation", - localKey: "id", - foreignKey: "initialBsdasriId" - } - ], - Bsda: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "EcoOrganisme", - localKey: "ecoOrganismeSiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationOperationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationOperationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "BsdaTransporter", - localKey: "id", - foreignKey: "bsdaId" - }, - { - type: "Company", - localKey: "workerCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "workerCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "Bsda", - localKey: "forwardingId", - foreignKey: "id" - }, - { - type: "Bsda", - localKey: "id", - foreignKey: "forwardingId" - }, - { - type: "Bsda", - localKey: "groupedInId", - foreignKey: "id" - }, - { - type: "Bsda", - localKey: "id", - foreignKey: "groupedInId" - }, - { - type: "BsdaRevisionRequest", - localKey: "id", - foreignKey: "bsdaId" - }, - { - type: "IntermediaryBsdaAssociation", - localKey: "id", - foreignKey: "bsdaId" - }, - { - type: "BsdaFinalOperation", - localKey: "id", - foreignKey: "finalBsdaId" - }, - { - type: "BsdaFinalOperation", - localKey: "id", - foreignKey: "initialBsdaId" - } - ], - Bsff: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "BsffFicheIntervention", - localKey: "id", - foreignKey: "id" - }, - { - type: "BsffPackaging", - localKey: "id", - foreignKey: "bsffId" - }, - { - type: "BsffTransporter", - localKey: "id", - foreignKey: "bsffId" - } - ], - Bspaoh: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "BspaohTransporter", - localKey: "id", - foreignKey: "bspaohId" - } - ], - Bsvhu: [ - { - type: "Company", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "destinationOperationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "emitterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "destinationOperationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - } - ], - Company: [ - { - type: "TraderReceipt", - localKey: "traderReceiptId", - foreignKey: "id" - }, - { - type: "BrokerReceipt", - localKey: "brokerReceiptId", - foreignKey: "id" - }, - { - type: "TransporterReceipt", - localKey: "transporterReceiptId", - foreignKey: "id" - }, - { - type: "VhuAgrement", - localKey: "vhuAgrementDemolisseurId", - foreignKey: "id" - }, - { - type: "VhuAgrement", - localKey: "vhuAgrementBroyeurId", - foreignKey: "id" - }, - { - type: "WorkerCertification", - localKey: "workerCertificationId", - foreignKey: "id" - }, - { - type: "CompanyAssociation", - localKey: "id", - foreignKey: "companyId" - }, - { - type: "MembershipRequest", - localKey: "id", - foreignKey: "companyId" - }, - { - type: "SignatureAutomation", - localKey: "id", - foreignKey: "fromId" - }, - { - type: "SignatureAutomation", - localKey: "id", - foreignKey: "toId" - }, - { - type: "WebhookSetting", - localKey: "orgId", - foreignKey: "orgId" - } - ], - BsddTransporter: [], - FormGroupement: [ - { - type: "Form", - localKey: "nextFormId", - foreignKey: "id" - }, - { - type: "Form", - localKey: "initialFormId", - foreignKey: "id" - } - ], - User: [ - { - type: "GovernmentAccount", - localKey: "governmentAccountId", - foreignKey: "id" - }, - { - type: "AccessToken", - localKey: "id", - foreignKey: "userId" - }, - { - type: "Application", - localKey: "id", - foreignKey: "adminId" - }, - { - type: "CompanyAssociation", - localKey: "id", - foreignKey: "userId" - }, - { - type: "FeatureFlag", - localKey: "id", - foreignKey: "userId" - }, - { - type: "Grant", - localKey: "id", - foreignKey: "userId" - }, - { - type: "MembershipRequest", - localKey: "id", - foreignKey: "userId" - }, - // { - // type: "StatusLog", - // localKey: "id", - // foreignKey: "userId" - // }, - { - type: "UserResetPasswordHash", - localKey: "id", - foreignKey: "userId" - }, - { - type: "UserActivationHash", - localKey: "id", - foreignKey: "userId" - } - ], - StatusLog: [], - BsddRevisionRequest: [ - { - type: "Company", - localKey: "authoringCompanyId", - foreignKey: "id" - }, - { - type: "BsddRevisionRequestApproval", - localKey: "id", - foreignKey: "revisionRequestId" - }, - { - type: "Company", - localKey: "traderCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "traderCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - } - ], - IntermediaryFormAssociation: [ - { - type: "Company", - localKey: "siret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "siret", - foreignKey: "orgId" - } - ], - BsddFinalOperation: [ - { - type: "Form", - localKey: "finalFormId", - foreignKey: "id" - }, - { - type: "Form", - localKey: "initialFormId", - foreignKey: "id" - } - ], - TraderReceipt: [], - BrokerReceipt: [], - TransporterReceipt: [], - VhuAgrement: [], - WorkerCertification: [], - CompanyAssociation: [ - { - type: "Company", - localKey: "companyId", - foreignKey: "id" - }, - { - type: "User", - localKey: "userId", - foreignKey: "id" - } - ], - MembershipRequest: [ - { - type: "Company", - localKey: "companyId", - foreignKey: "id" - }, - { - type: "User", - localKey: "userId", - foreignKey: "id" - } - ], - SignatureAutomation: [ - { - type: "Company", - localKey: "fromId", - foreignKey: "id" - }, - { - type: "Company", - localKey: "toId", - foreignKey: "id" - } - ], - GovernmentAccount: [], - AccessToken: [ - { - type: "Application", - localKey: "applicationId", - foreignKey: "id" - }, - { - type: "User", - localKey: "userId", - foreignKey: "id" - } - ], - Application: [ - { - type: "User", - localKey: "adminId", - foreignKey: "id" - }, - { - type: "AccessToken", - localKey: "id", - foreignKey: "applicationId" - }, - { - type: "Grant", - localKey: "id", - foreignKey: "applicationId" - } - ], - FeatureFlag: [], - Grant: [ - { - type: "Application", - localKey: "applicationId", - foreignKey: "id" - }, - { - type: "User", - localKey: "userId", - foreignKey: "id" - } - ], - UserResetPasswordHash: [], - UserActivationHash: [], - BsddRevisionRequestApproval: [], - BsdasriRevisionRequest: [ - { - type: "Company", - localKey: "authoringCompanyId", - foreignKey: "id" - }, - { - type: "BsdasriRevisionRequestApproval", - localKey: "id", - foreignKey: "revisionRequestId" - } - ], - BsdasriFinalOperation: [ - { - type: "Bsdasri", - localKey: "finalBsdasriId", - foreignKey: "id" - }, - { - type: "Bsdasri", - localKey: "initialBsdasriId", - foreignKey: "id" - } - ], - BsdasriRevisionRequestApproval: [], - BsdaTransporter: [ - { - type: "Company", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - } - ], - BsdaRevisionRequest: [ - { - type: "Company", - localKey: "authoringCompanyId", - foreignKey: "id" - }, - { - type: "BsdaRevisionRequestApproval", - localKey: "id", - foreignKey: "revisionRequestId" - }, - { - type: "Company", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "brokerCompanySiret", - foreignKey: "orgId" - } - ], - IntermediaryBsdaAssociation: [], - BsdaFinalOperation: [ - { - type: "Bsda", - localKey: "finalBsdaId", - foreignKey: "id" - }, - { - type: "Bsda", - localKey: "initialBsdaId", - foreignKey: "id" - } - ], - BsdaRevisionRequestApproval: [], - BsffFicheIntervention: [ - { - type: "Company", - localKey: "detenteurCompanySiret", - foreignKey: "orgId" - }, - { - type: "Company", - localKey: "operateurCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "detenteurCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "operateurCompanySiret", - foreignKey: "orgId" - } - ], - BsffPackaging: [ - { - type: "Company", - localKey: "operationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "operationNextDestinationCompanySiret", - foreignKey: "orgId" - }, - { - type: "BsffPackaging", - localKey: "nextPackagingId", - foreignKey: "id" - }, - { - type: "BsffPackaging", - localKey: "id", - foreignKey: "nextPackagingId" - }, - { - type: "BsffPackagingFinalOperation", - localKey: "id", - foreignKey: "finalBsffPackagingId" - }, - { - type: "BsffPackagingFinalOperation", - localKey: "id", - foreignKey: "initialBsffPackagingId" - } - ], - BsffTransporter: [ - { - type: "Company", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - } - ], - BsffPackagingFinalOperation: [ - { - type: "BsffPackaging", - localKey: "finalBsffPackagingId", - foreignKey: "id" - }, - { - type: "BsffPackaging", - localKey: "initialBsffPackagingId", - foreignKey: "id" - } - ], - BspaohTransporter: [ - { - type: "Company", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - }, - { - type: "AnonymousCompany", - localKey: "transporterCompanySiret", - foreignKey: "orgId" - } - ] -}; +const pipelines = getPipelines(prismaLocal, prismaRemote); +/* + The main Run method +*/ const run = async () => { + /* + get the origin BSD id/readableId either from ROOT_OBJ en var or user input + */ + let rootObjId = ROOT_OBJ; + if (!rootObjId) { + rootObjId = await questionPromise( + "Enter the id of the BSD (or readable ID for BSDD) you want to use as root for this partial dump : " + ); + if (!rootObjId) { + console.error( + "The root BSD is not defined, please specify an object to act as the dump starting point, either by passing a ROOT_OBJ environment variable or through this prompt." + ); + return; + } + } let originType: "Form" | "Bsdasri" | "Bsda" | "Bsff" | "Bspaoh" | "Bsvhu"; - - const objType = DUMP_OBJ.split("-")?.[0]; - const alreadyFetched: { [key: string]: any } = {}; + let originId: "id" | "readableId" = "id"; + const objType = rootObjId.split("-")?.[0]; if (!objType) { - throw new Error("DUMP_OBJ is not a valid BSD id"); + console.error("The root object id entered is not a valid BSD id"); + return; } - + /* + Deduce the type of BSD we're starting from + */ switch (objType) { case "BSD": originType = "Form"; + originId = "readableId"; break; case "DASRI": originType = "Bsdasri"; @@ -1415,8 +123,14 @@ const run = async () => { originType = "Bsvhu"; break; default: - throw new Error("DUMP_OBJ is not a valid BSD id"); + console.error("The root object id entered is not a valid BSD id"); + return; } + + /* + Each object that gets loaded is put into one of those structItem. + During the loading process, it becomes a deeply nested object where everything is saved. + */ type structItem = { type: string; obj: any; @@ -1426,10 +140,20 @@ const run = async () => { }; const struct: structItem[] = []; - const bsds = await pipelines[originType].getter("readableId", DUMP_OBJ); + + /* + Each object loaded also goes into this flat object, indexed by its id. + This is the object we are getting the data when writing to the destination database + */ + const alreadyFetched: { [key: string]: { type: string; obj: any } } = {}; + /* + Load the root BSD + */ + const bsds = await pipelines[originType].getter(originId, rootObjId); const bsd = bsds?.[0]; if (!bsd) { - throw new Error("root BSD not found"); + console.error("Root BSD not found"); + return; } const bsdRoot: structItem = { type: originType, @@ -1443,28 +167,30 @@ const run = async () => { type: originType, obj: bsd }; - console.log("got BSD Root"); - console.log(bsdRoot.obj.id); - let maxDepth = 0; + /* + This method recursively loads the objects related to the root BSD. + It uses the traversal object to know what to fetch and how, + then save it to structItems and the alreadyFetched object. + if an object is already in the "alreadyFetched" object, it doesn't get fetched again. + Normally, at some point, we reach the end of each recursive branch + because all related objects are already fetched or there is no related objects to fetch. + */ const recursiveExtract = async (root: structItem) => { - console.log(`TRAVERSING ${root.path}`); - if (root.depth > maxDepth) { - maxDepth = root.depth; - } + print(`TRAVERSING ${root.path}`); if (!traversals[root.type]) { - console.log(`TRAVERSAL NOT AVAILABLE FOR ${root.type}`); + // console.log(`TRAVERSAL NOT AVAILABLE FOR ${root.type}`); return; } for (const item of traversals[root.type]) { - console.log( - `fetching ${item.type}, fKey: ${item.foreignKey}, lKey: ${ - root.obj[item.localKey] - }` - ); + // console.log( + // `fetching ${item.type}, fKey: ${item.foreignKey}, lKey: ${ + // root.obj[item.localKey] + // }` + // ); const getter = pipelines[item.type]?.getter; if (!getter) { - console.log(`MISSING GETTER ${item.type}`); + // console.log(`MISSING GETTER FOR ${item.type}`); } // const setter = pipelines[item.type]?.setter; const objects = await getter?.(item.foreignKey, root.obj[item.localKey]); @@ -1483,7 +209,9 @@ const run = async () => { const subRoots: structItem[] = filteredObjects.map(obj => ({ type: item.type, obj, - path: `${root.path}>${item.type}(${obj.id})`, + path: `${root.path}\n${">".repeat(root.depth + 1)}${item.type}(${ + obj.id + })`, depth: root.depth + 1, children: [] })); @@ -1495,8 +223,10 @@ const run = async () => { } }; await recursiveExtract(bsdRoot); - console.log(Object.keys(alreadyFetched)); - console.log("max depth: ", maxDepth); + + /* + build a little recap object to know how many objects of which type have been loaded + */ const statsByType = {}; for (const id of Object.keys(alreadyFetched)) { if (!statsByType[alreadyFetched[id].type]) { @@ -1505,12 +235,47 @@ const run = async () => { statsByType[alreadyFetched[id].type] += 1; } } + + print(`DUMP COMPLETE!`); + console.log("What will be copied :"); console.log(statsByType); + + // console.log(statsByType); + const continueRes = await questionPromise( + "Do you want to write this to the destination database? (make sure it is empty and the schema is built) Y/N : " + ); + if ( + continueRes !== "y" && + continueRes !== "Y" && + continueRes.toLowerCase() !== "yes" + ) { + console.log("ABORTING"); + return; + } + + /* + Now we save the loaded objects to the destination DB. + Since there are some foreign key constraints, some objects have to be written before others + or the writing fails. + Since it's complicated to know the right order, I chose a more bruteforce approach: + - Try to write all the objects + - Remember which ones got saved + - Do it again with the ones that didn't get saved + - Stop when everything is saved + + This has a risk of never ending, for example if there is a write error that is not + caused by a foreign key constraint (conflicting id of the db was not empty, + wrong format if the schema is different between source and destination, ...). + To avoid an infinite loop, I had a check that at least one object is saved on each iteration. + This way if we're stuck and nothing gets saved, we abort. + */ const alreadySaved: { [key: string]: boolean } = {}; const allSaved = () => { return !Object.keys(alreadyFetched).some(id => !alreadySaved[id]); }; + let successfulSaves; while (!allSaved()) { + successfulSaves = 0; for (const id of Object.keys(alreadyFetched)) { if (alreadySaved[id]) { continue; @@ -1522,12 +287,25 @@ const run = async () => { try { await setter(alreadyFetched[id].obj); alreadySaved[id] = true; - console.log(`saved ${alreadyFetched[id].type} ${id}`); + successfulSaves += 1; + print(`saved ${alreadyFetched[id].type} ${id}`); } catch (error) { - console.log(`could not save ${alreadyFetched[id].type} ${id}`); + // console.log(`could not save ${alreadyFetched[id].type} ${id}`); } } + if (successfulSaves === 0) { + console.error( + "There seems to be a problem writing to the database. Is it empty? Is the schema the same as the one you're trying to copy from?" + ); + return; + } } + print( + "ALL DONE ! remember to reindex to elastic ( > npx nx run back:reindex-all-bsds-bulk -- -f )" + ); }; -run(); +run().then( + () => rl.close(), + () => rl.close() +); diff --git a/libs/back/partial-backup/src/pipelines.ts b/libs/back/partial-backup/src/pipelines.ts new file mode 100644 index 0000000000..7f26165593 --- /dev/null +++ b/libs/back/partial-backup/src/pipelines.ts @@ -0,0 +1,513 @@ +import { + AccessToken, + AnonymousCompany, + Application, + BrokerReceipt, + Bsda, + BsdaFinalOperation, + BsdaRevisionRequest, + BsdaRevisionRequestApproval, + BsdaTransporter, + Bsdasri, + BsdasriFinalOperation, + BsdasriRevisionRequest, + BsdasriRevisionRequestApproval, + BsddFinalOperation, + BsddRevisionRequest, + BsddRevisionRequestApproval, + BsddTransporter, + Bsff, + BsffFicheIntervention, + BsffPackaging, + BsffPackagingFinalOperation, + BsffTransporter, + Bspaoh, + BspaohTransporter, + Company, + CompanyAssociation, + EcoOrganisme, + FeatureFlag, + Form, + FormGroupement, + GovernmentAccount, + Grant, + IntermediaryBsdaAssociation, + IntermediaryFormAssociation, + MembershipRequest, + Prisma, + PrismaClient, + SignatureAutomation, + StatusLog, + TraderReceipt, + TransporterReceipt, + User, + UserActivationHash, + UserResetPasswordHash, + VhuAgrement, + WebhookSetting, + WorkerCertification +} from "@prisma/client"; + +/* + This is an object that contains getters and setters to fetch various object types from the source DB + and write them to the destionation DB. + + Most of them look the same, but some setters are a bit different to handle Json values, and some getters + are different to handle things like implicit many-to-many relations. +*/ + +const getPipelines = ( + prismaLocal: PrismaClient, + prismaRemote: PrismaClient +) => ({ + Form: { + getter: async (key: string, value?: string) => + value && prismaRemote.form.findMany({ where: { [key]: value } }), + setter: async (bsd?: Form) => + bsd && + prismaLocal.form.create({ + data: { + ...bsd, + wasteDetailsPackagingInfos: + bsd.wasteDetailsPackagingInfos ?? Prisma.JsonNull, + wasteDetailsParcelNumbers: + bsd.wasteDetailsParcelNumbers ?? Prisma.JsonNull + } + }) + }, + Bsdasri: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsdasri.findMany({ where: { [key]: value } }), + setter: async (bsdasri?: Bsdasri) => + bsdasri && + prismaLocal.bsdasri.create({ + data: { + ...bsdasri, + emitterWastePackagings: + bsdasri.emitterWastePackagings ?? Prisma.JsonNull, + transporterWastePackagings: + bsdasri.transporterWastePackagings ?? Prisma.JsonNull, + destinationWastePackagings: + bsdasri.destinationWastePackagings ?? Prisma.JsonNull + } + }) + }, + Bsda: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsda.findMany({ where: { [key]: value } }), + setter: async (bsda?: Bsda) => + bsda && + prismaLocal.bsda.create({ + data: { + ...bsda, + packagings: bsda.packagings ?? Prisma.JsonNull + } + }) + }, + Bsff: { + getter: async (key: string, value?: string) => + value && prismaRemote.bsff.findMany({ where: { [key]: value } }), + setter: async (bsff?: Bsff) => + bsff && prismaLocal.bsff.create({ data: bsff }) + }, + Bspaoh: { + getter: async (key: string, value?: string) => + value && prismaRemote.bspaoh.findMany({ where: { [key]: value } }), + setter: async (bspaoh?: Bspaoh) => + bspaoh && + prismaLocal.bspaoh.create({ + data: { + ...bspaoh, + wastePackagings: bspaoh.wastePackagings ?? Prisma.JsonNull, + destinationReceptionWastePackagingsAcceptation: + bspaoh.destinationReceptionWastePackagingsAcceptation ?? + Prisma.JsonNull + } + }) + }, + Company: { + getter: async (key: string, value?: string) => + value && prismaRemote.company.findMany({ where: { [key]: value } }), + setter: async (company?: Company) => + company && prismaLocal.company.create({ data: company }) + }, + AnonymousCompany: { + getter: async (key: string, value?: string) => + value && + prismaRemote.anonymousCompany.findMany({ where: { [key]: value } }), + setter: async (anonymousCompany?: AnonymousCompany) => + anonymousCompany && + prismaLocal.anonymousCompany.create({ data: anonymousCompany }) + }, + EcoOrganisme: { + getter: async (key: string, value?: string) => + value && prismaRemote.ecoOrganisme.findMany({ where: { [key]: value } }), + setter: async (ecoOrganisme?: EcoOrganisme) => + ecoOrganisme && prismaLocal.ecoOrganisme.create({ data: ecoOrganisme }) + }, + BsddTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddTransporter.findMany({ where: { [key]: value } }), + setter: async (transporter?: BsddTransporter) => + transporter && prismaLocal.bsddTransporter.create({ data: transporter }) + }, + FormGroupement: { + getter: async (key: string, value?: string) => + value && + prismaRemote.formGroupement.findMany({ where: { [key]: value } }), + setter: async (groupement?: FormGroupement) => + groupement && prismaLocal.formGroupement.create({ data: groupement }) + }, + User: { + getter: async (key: string, value?: string) => + value && prismaRemote.user.findMany({ where: { [key]: value } }), + setter: async (user?: User) => + user && prismaLocal.user.create({ data: user }) + }, + StatusLog: { + getter: async (key: string, value?: string) => + value && prismaRemote.statusLog.findMany({ where: { [key]: value } }), + setter: async (statusLog?: StatusLog) => + statusLog && + prismaLocal.statusLog.create({ + data: { + ...statusLog, + updatedFields: statusLog.updatedFields ?? Prisma.JsonNull + } + }) + }, + BsddRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddRevisionRequest.findMany({ where: { [key]: value } }), + setter: async (revisionRequest?: BsddRevisionRequest) => + revisionRequest && + prismaLocal.bsddRevisionRequest.create({ + data: { + ...revisionRequest, + wasteDetailsPackagingInfos: + revisionRequest.wasteDetailsPackagingInfos ?? Prisma.JsonNull + } + }) + }, + IntermediaryFormAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.intermediaryFormAssociation.findMany({ + where: { [key]: value } + }), + setter: async (intermediaryFormAssociation?: IntermediaryFormAssociation) => + intermediaryFormAssociation && + prismaLocal.intermediaryFormAssociation.create({ + data: intermediaryFormAssociation + }) + }, + BsddFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddFinalOperation.findMany({ where: { [key]: value } }), + setter: async (bsddFinalOperation?: BsddFinalOperation) => + bsddFinalOperation && + prismaLocal.bsddFinalOperation.create({ data: bsddFinalOperation }) + }, + TraderReceipt: { + getter: async (key: string, value?: string) => + value && prismaRemote.traderReceipt.findMany({ where: { [key]: value } }), + setter: async (traderReceipt?: TraderReceipt) => + traderReceipt && prismaLocal.traderReceipt.create({ data: traderReceipt }) + }, + BrokerReceipt: { + getter: async (key: string, value?: string) => + value && prismaRemote.brokerReceipt.findMany({ where: { [key]: value } }), + setter: async (brokerReceipt?: BrokerReceipt) => + brokerReceipt && prismaLocal.brokerReceipt.create({ data: brokerReceipt }) + }, + TransporterReceipt: { + getter: async (key: string, value?: string) => + value && + prismaRemote.transporterReceipt.findMany({ where: { [key]: value } }), + setter: async (transporterReceipt?: TransporterReceipt) => + transporterReceipt && + prismaLocal.transporterReceipt.create({ data: transporterReceipt }) + }, + VhuAgrement: { + getter: async (key: string, value?: string) => + value && prismaRemote.vhuAgrement.findMany({ where: { [key]: value } }), + setter: async (vhuAgrement?: VhuAgrement) => + vhuAgrement && prismaLocal.vhuAgrement.create({ data: vhuAgrement }) + }, + WorkerCertification: { + getter: async (key: string, value?: string) => + value && + prismaRemote.workerCertification.findMany({ where: { [key]: value } }), + setter: async (workerCertification?: WorkerCertification) => + workerCertification && + prismaLocal.workerCertification.create({ data: workerCertification }) + }, + CompanyAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.companyAssociation.findMany({ where: { [key]: value } }), + setter: async (companyAssociation?: CompanyAssociation) => + companyAssociation && + prismaLocal.companyAssociation.create({ data: companyAssociation }) + }, + MembershipRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.membershipRequest.findMany({ where: { [key]: value } }), + setter: async (membershipRequest?: MembershipRequest) => + membershipRequest && + prismaLocal.membershipRequest.create({ data: membershipRequest }) + }, + SignatureAutomation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.signatureAutomation.findMany({ where: { [key]: value } }), + setter: async (signatureAutomation?: SignatureAutomation) => + signatureAutomation && + prismaLocal.signatureAutomation.create({ data: signatureAutomation }) + }, + GovernmentAccount: { + getter: async (key: string, value?: string) => + value && + prismaRemote.governmentAccount.findMany({ where: { [key]: value } }), + setter: async (governmentAccount?: GovernmentAccount) => + governmentAccount && + prismaLocal.governmentAccount.create({ data: governmentAccount }) + }, + AccessToken: { + getter: async (key: string, value?: string) => + value && prismaRemote.accessToken.findMany({ where: { [key]: value } }), + setter: async (accessToken?: AccessToken) => + accessToken && prismaLocal.accessToken.create({ data: accessToken }) + }, + Application: { + getter: async (key: string, value?: string) => + value && prismaRemote.application.findMany({ where: { [key]: value } }), + setter: async (application?: Application) => + application && prismaLocal.application.create({ data: application }) + }, + FeatureFlag: { + getter: async (key: string, value?: string) => + value && prismaRemote.featureFlag.findMany({ where: { [key]: value } }), + setter: async (featureFlag?: FeatureFlag) => + featureFlag && prismaLocal.featureFlag.create({ data: featureFlag }) + }, + Grant: { + getter: async (key: string, value?: string) => + value && prismaRemote.grant.findMany({ where: { [key]: value } }), + setter: async (grant?: Grant) => + grant && prismaLocal.grant.create({ data: grant }) + }, + UserResetPasswordHash: { + getter: async (key: string, value?: string) => + value && + prismaRemote.userResetPasswordHash.findMany({ where: { [key]: value } }), + setter: async (userResetPasswordHash?: UserResetPasswordHash) => + userResetPasswordHash && + prismaLocal.userResetPasswordHash.create({ data: userResetPasswordHash }) + }, + UserActivationHash: { + getter: async (key: string, value?: string) => + value && + prismaRemote.userActivationHash.findMany({ where: { [key]: value } }), + setter: async (userActivationHash?: UserActivationHash) => + userActivationHash && + prismaLocal.userActivationHash.create({ data: userActivationHash }) + }, + BsddRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsddRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async (bsddRevisionRequestApproval?: BsddRevisionRequestApproval) => + bsddRevisionRequestApproval && + prismaLocal.bsddRevisionRequestApproval.create({ + data: bsddRevisionRequestApproval + }) + }, + BsdasriRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriRevisionRequest.findMany({ + where: { [key]: value } + }), + setter: async (bsdasriRevisionRequest?: BsdasriRevisionRequest) => + bsdasriRevisionRequest && + prismaLocal.bsdasriRevisionRequest.create({ + data: { + ...bsdasriRevisionRequest, + destinationWastePackagings: + bsdasriRevisionRequest.destinationWastePackagings ?? Prisma.JsonNull + } + }) + }, + BsdasriFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsdasriFinalOperation?: BsdasriFinalOperation) => + bsdasriFinalOperation && + prismaLocal.bsdasriFinalOperation.create({ + data: bsdasriFinalOperation + }) + }, + BsdasriRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdasriRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async ( + bsdasriRevisionRequestApproval?: BsdasriRevisionRequestApproval + ) => + bsdasriRevisionRequestApproval && + prismaLocal.bsdasriRevisionRequestApproval.create({ + data: bsdasriRevisionRequestApproval + }) + }, + BsdaTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bsdaTransporter?: BsdaTransporter) => + bsdaTransporter && + prismaLocal.bsdaTransporter.create({ + data: bsdaTransporter + }) + }, + BsdaRevisionRequest: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaRevisionRequest.findMany({ + where: { [key]: value } + }), + setter: async (bsdaRevisionRequest?: BsdaRevisionRequest) => + bsdaRevisionRequest && + prismaLocal.bsdaRevisionRequest.create({ + data: { + ...bsdaRevisionRequest, + packagings: bsdaRevisionRequest.packagings ?? Prisma.JsonNull + } + }) + }, + IntermediaryBsdaAssociation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.intermediaryBsdaAssociation.findMany({ + where: { [key]: value } + }), + setter: async (intermediaryBsdaAssociation?: IntermediaryBsdaAssociation) => + intermediaryBsdaAssociation && + prismaLocal.intermediaryBsdaAssociation.create({ + data: intermediaryBsdaAssociation + }) + }, + BsdaFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsdaFinalOperation?: BsdaFinalOperation) => + bsdaFinalOperation && + prismaLocal.bsdaFinalOperation.create({ + data: bsdaFinalOperation + }) + }, + BsdaRevisionRequestApproval: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsdaRevisionRequestApproval.findMany({ + where: { [key]: value } + }), + setter: async (bsdaRevisionRequestApproval?: BsdaRevisionRequestApproval) => + bsdaRevisionRequestApproval && + prismaLocal.bsdaRevisionRequestApproval.create({ + data: bsdaRevisionRequestApproval + }) + }, + BsffFicheIntervention: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsff + .findUnique({ + where: { id: value } + }) + .ficheInterventions(), + setter: async (bsffFicheIntervention?: BsffFicheIntervention) => + bsffFicheIntervention && + prismaLocal.bsffFicheIntervention.create({ + data: bsffFicheIntervention + }) + }, + BsffPackaging: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffPackaging.findMany({ + where: { [key]: value } + }), + setter: async (bsffPackaging?: BsffPackaging) => + bsffPackaging && + prismaLocal.bsffPackaging.create({ + data: bsffPackaging + }) + }, + BsffTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bsffTransporter?: BsffTransporter) => + bsffTransporter && + prismaLocal.bsffTransporter.create({ + data: bsffTransporter + }) + }, + BsffPackagingFinalOperation: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bsffPackagingFinalOperation.findMany({ + where: { [key]: value } + }), + setter: async (bsffPackagingFinalOperation?: BsffPackagingFinalOperation) => + bsffPackagingFinalOperation && + prismaLocal.bsffPackagingFinalOperation.create({ + data: bsffPackagingFinalOperation + }) + }, + BspaohTransporter: { + getter: async (key: string, value?: string) => + value && + prismaRemote.bspaohTransporter.findMany({ + where: { [key]: value } + }), + setter: async (bspaohTransporter?: BspaohTransporter) => + bspaohTransporter && + prismaLocal.bspaohTransporter.create({ + data: bspaohTransporter + }) + }, + WebhookSetting: { + getter: async (key: string, value?: string) => + value && + prismaRemote.webhookSetting.findMany({ + where: { [key]: value } + }), + setter: async (webhookSetting?: WebhookSetting) => + webhookSetting && + prismaLocal.webhookSetting.create({ + data: webhookSetting + }) + } +}); + +export default getPipelines; diff --git a/libs/back/partial-backup/src/traversals.ts b/libs/back/partial-backup/src/traversals.ts new file mode 100644 index 0000000000..438bec9b73 --- /dev/null +++ b/libs/back/partial-backup/src/traversals.ts @@ -0,0 +1,854 @@ +/* +This is an object that contains relations to load for each type of object. +This is used to "traverse" the database from the origin BSD, and get all related objects. + +This is not a complete representation of the DB's relations, as some could lead to loading useless BSDs and going too deep. +Also some objects that are not very useful for reproducing issues and take a LOT of queries to fetch (StatusLog, Events) are sometimes omitted. +*/ + +const traversals = { + Form: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "recipientCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "nextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "recipientCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "nextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsddTransporter", + localKey: "id", + foreignKey: "formId" + }, + { + type: "FormGroupement", + localKey: "id", + foreignKey: "nextFormId" + }, + { + type: "FormGroupement", + localKey: "id", + foreignKey: "initialFormId" + }, + { + type: "User", + localKey: "ownerId", + foreignKey: "id" + }, + { + type: "StatusLog", + localKey: "id", + foreignKey: "formId" + }, + { + type: "BsddRevisionRequest", + localKey: "id", + foreignKey: "bsddId" + }, + { + type: "IntermediaryFormAssociation", + localKey: "id", + foreignKey: "formId" + }, + { + type: "Form", + localKey: "forwardedInId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "id", + foreignKey: "forwardedInId" + }, + { + type: "BsddFinalOperation", + localKey: "id", + foreignKey: "finalFormId" + }, + { + type: "BsddFinalOperation", + localKey: "id", + foreignKey: "initialFormId" + } + ], + Bsdasri: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "User", + localKey: "emissionSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "transportSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "receptionSignatoryId", + foreignKey: "id" + }, + { + type: "User", + localKey: "operationSignatoryId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "groupedInId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "id", + foreignKey: "groupedInId" + }, + { + type: "Bsdasri", + localKey: "synthesizedInId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "id", + foreignKey: "synthesizedInId" + }, + { + type: "BsdasriRevisionRequest", + localKey: "id", + foreignKey: "bsdasriId" + }, + { + type: "BsdasriFinalOperation", + localKey: "id", + foreignKey: "finalBsdasriId" + }, + { + type: "BsdasriFinalOperation", + localKey: "id", + foreignKey: "initialBsdasriId" + } + ], + Bsda: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "EcoOrganisme", + localKey: "ecoOrganismeSiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsdaTransporter", + localKey: "id", + foreignKey: "bsdaId" + }, + { + type: "Company", + localKey: "workerCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "workerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "Bsda", + localKey: "forwardingId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "id", + foreignKey: "forwardingId" + }, + { + type: "Bsda", + localKey: "groupedInId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "id", + foreignKey: "groupedInId" + }, + { + type: "BsdaRevisionRequest", + localKey: "id", + foreignKey: "bsdaId" + }, + { + type: "IntermediaryBsdaAssociation", + localKey: "id", + foreignKey: "bsdaId" + }, + { + type: "BsdaFinalOperation", + localKey: "id", + foreignKey: "finalBsdaId" + }, + { + type: "BsdaFinalOperation", + localKey: "id", + foreignKey: "initialBsdaId" + } + ], + Bsff: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsffFicheIntervention", + localKey: "id", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "id", + foreignKey: "bsffId" + }, + { + type: "BsffTransporter", + localKey: "id", + foreignKey: "bsffId" + } + ], + Bspaoh: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BspaohTransporter", + localKey: "id", + foreignKey: "bspaohId" + } + ], + Bsvhu: [ + { + type: "Company", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "emitterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "destinationOperationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + Company: [ + { + type: "TraderReceipt", + localKey: "traderReceiptId", + foreignKey: "id" + }, + { + type: "BrokerReceipt", + localKey: "brokerReceiptId", + foreignKey: "id" + }, + { + type: "TransporterReceipt", + localKey: "transporterReceiptId", + foreignKey: "id" + }, + { + type: "VhuAgrement", + localKey: "vhuAgrementDemolisseurId", + foreignKey: "id" + }, + { + type: "VhuAgrement", + localKey: "vhuAgrementBroyeurId", + foreignKey: "id" + }, + { + type: "WorkerCertification", + localKey: "workerCertificationId", + foreignKey: "id" + }, + { + type: "CompanyAssociation", + localKey: "id", + foreignKey: "companyId" + }, + { + type: "MembershipRequest", + localKey: "id", + foreignKey: "companyId" + }, + { + type: "SignatureAutomation", + localKey: "id", + foreignKey: "fromId" + }, + { + type: "SignatureAutomation", + localKey: "id", + foreignKey: "toId" + }, + { + type: "WebhookSetting", + localKey: "orgId", + foreignKey: "orgId" + } + ], + BsddTransporter: [], + FormGroupement: [ + { + type: "Form", + localKey: "nextFormId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "initialFormId", + foreignKey: "id" + } + ], + User: [ + { + type: "GovernmentAccount", + localKey: "governmentAccountId", + foreignKey: "id" + }, + { + type: "AccessToken", + localKey: "id", + foreignKey: "userId" + }, + { + type: "Application", + localKey: "id", + foreignKey: "adminId" + }, + { + type: "CompanyAssociation", + localKey: "id", + foreignKey: "userId" + }, + { + type: "FeatureFlag", + localKey: "id", + foreignKey: "userId" + }, + { + type: "Grant", + localKey: "id", + foreignKey: "userId" + }, + { + type: "MembershipRequest", + localKey: "id", + foreignKey: "userId" + }, + // { + // type: "StatusLog", + // localKey: "id", + // foreignKey: "userId" + // }, + { + type: "UserResetPasswordHash", + localKey: "id", + foreignKey: "userId" + }, + { + type: "UserActivationHash", + localKey: "id", + foreignKey: "userId" + } + ], + StatusLog: [], + BsddRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsddRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + }, + { + type: "Company", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "traderCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + } + ], + IntermediaryFormAssociation: [ + { + type: "Company", + localKey: "siret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "siret", + foreignKey: "orgId" + } + ], + BsddFinalOperation: [ + { + type: "Form", + localKey: "finalFormId", + foreignKey: "id" + }, + { + type: "Form", + localKey: "initialFormId", + foreignKey: "id" + } + ], + TraderReceipt: [], + BrokerReceipt: [], + TransporterReceipt: [], + VhuAgrement: [], + WorkerCertification: [], + CompanyAssociation: [ + { + type: "Company", + localKey: "companyId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + MembershipRequest: [ + { + type: "Company", + localKey: "companyId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + SignatureAutomation: [ + { + type: "Company", + localKey: "fromId", + foreignKey: "id" + }, + { + type: "Company", + localKey: "toId", + foreignKey: "id" + } + ], + GovernmentAccount: [], + AccessToken: [ + { + type: "Application", + localKey: "applicationId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + Application: [ + { + type: "User", + localKey: "adminId", + foreignKey: "id" + }, + { + type: "AccessToken", + localKey: "id", + foreignKey: "applicationId" + }, + { + type: "Grant", + localKey: "id", + foreignKey: "applicationId" + } + ], + FeatureFlag: [], + Grant: [ + { + type: "Application", + localKey: "applicationId", + foreignKey: "id" + }, + { + type: "User", + localKey: "userId", + foreignKey: "id" + } + ], + UserResetPasswordHash: [], + UserActivationHash: [], + BsddRevisionRequestApproval: [], + BsdasriRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsdasriRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + } + ], + BsdasriFinalOperation: [ + { + type: "Bsdasri", + localKey: "finalBsdasriId", + foreignKey: "id" + }, + { + type: "Bsdasri", + localKey: "initialBsdasriId", + foreignKey: "id" + } + ], + BsdasriRevisionRequestApproval: [], + BsdaTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + BsdaRevisionRequest: [ + { + type: "Company", + localKey: "authoringCompanyId", + foreignKey: "id" + }, + { + type: "BsdaRevisionRequestApproval", + localKey: "id", + foreignKey: "revisionRequestId" + }, + { + type: "Company", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "brokerCompanySiret", + foreignKey: "orgId" + } + ], + IntermediaryBsdaAssociation: [], + BsdaFinalOperation: [ + { + type: "Bsda", + localKey: "finalBsdaId", + foreignKey: "id" + }, + { + type: "Bsda", + localKey: "initialBsdaId", + foreignKey: "id" + } + ], + BsdaRevisionRequestApproval: [], + BsffFicheIntervention: [ + { + type: "Company", + localKey: "detenteurCompanySiret", + foreignKey: "orgId" + }, + { + type: "Company", + localKey: "operateurCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "detenteurCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "operateurCompanySiret", + foreignKey: "orgId" + } + ], + BsffPackaging: [ + { + type: "Company", + localKey: "operationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "operationNextDestinationCompanySiret", + foreignKey: "orgId" + }, + { + type: "BsffPackaging", + localKey: "nextPackagingId", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "id", + foreignKey: "nextPackagingId" + }, + { + type: "BsffPackagingFinalOperation", + localKey: "id", + foreignKey: "finalBsffPackagingId" + }, + { + type: "BsffPackagingFinalOperation", + localKey: "id", + foreignKey: "initialBsffPackagingId" + } + ], + BsffTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + BsffPackagingFinalOperation: [ + { + type: "BsffPackaging", + localKey: "finalBsffPackagingId", + foreignKey: "id" + }, + { + type: "BsffPackaging", + localKey: "initialBsffPackagingId", + foreignKey: "id" + } + ], + BspaohTransporter: [ + { + type: "Company", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + }, + { + type: "AnonymousCompany", + localKey: "transporterCompanySiret", + foreignKey: "orgId" + } + ], + AnonymousCompany: [] +}; + +export default traversals; From 2b10da4f95927ed5e2c3b63483b0eb048f480a65 Mon Sep 17 00:00:00 2001 From: silto Date: Mon, 3 Jun 2024 00:52:43 +0200 Subject: [PATCH 4/8] docs(CONTRIBUTING): add documentation to contributing.md --- CONTRIBUTING.md | 56 ++++++++++++++++++++++++++++ libs/back/partial-backup/src/main.ts | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ec1aa69a0..56ca4b3bc1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -626,6 +626,62 @@ $ ./restore-db.sh # La première question détermine si vous souhaitez utiliser un backup distant ou local ``` +#### Procédure automatique de restauration partielle d'une base de donnée de production + +Un script permettant de faire un dump partiel d'une DB a été créé. Il part d'un BSD spécifique qui doit être testé, et traverse récursivement la db pour trouver tous les objets qui y sont reliés, de façon à avoir un environnement de test complet pour reproduire un problème. + +Etapes préliminaires: + +- créer une nouvelle DB vide et la mettre dans la variable _DATABASE_URL_ du fichier .env +- appliquer `npx prisma migrate dev` +- ouvrir un tunnel SSH vers la db à dumper en utilisant le client scalingo + - `scalingo login` (nécéssite d'avoir une clé SSH renseignée dans Scalingo) + - `scalingo -a db-tunnel SCALINGO_POSTGRESQL_URL` +- ajouter l'url de la DB tunnelée dans _TUNNELED_DB_ dans le fichier .env. Utiliser un utilisateur read-only pour l'accès, voir avec l'équipe pour en créer un ou obtenir ses credentials. + +Utilisation du script: + +```bash +$ npx nx run partial-backup:run +``` + +Le script vous demandera l'id du BSD de départ (utiliser le readableId "BSD-..." pour les BSDD/Form), puis se chargera de charger tous les objets en relation. Une fois le chargement fait, vous aurez un aperçu des données sous cette forme : + +``` +What will be copied : +{ + Bsdasri: 3, + Company: 297, + AnonymousCompany: 3, + User: 78, + TransporterReceipt: 65, + CompanyAssociation: 408, + MembershipRequest: 65, + VhuAgrement: 54, + BrokerReceipt: 12, + AccessToken: 77, + Grant: 12, + Application: 3, + WorkerCertification: 35, + SignatureAutomation: 40, + TraderReceipt: 11, + UserActivationHash: 3, + UserResetPasswordHash: 11, + FeatureFlag: 1 +} +``` + +Si les informations semblent raisonnables, vous pouvez accepter d'écrire dans votre DB de destination en tapant "Y". + +Si une erreur survient lors du processus d'écriture, il est possible que ce soit dû à: + +- le schema utilisé en local ne correspond pas à celui de la db source +- le schema Prisma ne correspond pas au schema de la db source +- la DB de destination n'est pas vide +- le schema de la DB de destination n'a pas été créé (`npx prisma migrate dev`) + +Si tout se passe correctement, il ne vous reste plus qu'à reconstruire l'index elastic avec les données chargées en appliquant `npx nx run back:reindex-all-bsds-bulk -- -f`. + #### Procédure manuelle 1. Télécharger un backup de la base de donnée nommée `prisma` que vous souhaitez restaurer diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts index 9dd807acad..8f3d24668a 100644 --- a/libs/back/partial-backup/src/main.ts +++ b/libs/back/partial-backup/src/main.ts @@ -237,7 +237,7 @@ const run = async () => { } print(`DUMP COMPLETE!`); - console.log("What will be copied :"); + console.log("\n\nWhat will be copied :"); console.log(statsByType); // console.log(statsByType); From b08a4ae2f761dfc8a2785aeb865059f02dfd06ee Mon Sep 17 00:00:00 2001 From: silto Date: Tue, 4 Jun 2024 21:30:00 +0200 Subject: [PATCH 5/8] fix(scripts): limit traversal through company to users --- libs/back/partial-backup/src/traversals.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/back/partial-backup/src/traversals.ts b/libs/back/partial-backup/src/traversals.ts index 438bec9b73..f6359bc53b 100644 --- a/libs/back/partial-backup/src/traversals.ts +++ b/libs/back/partial-backup/src/traversals.ts @@ -505,11 +505,11 @@ const traversals = { localKey: "id", foreignKey: "adminId" }, - { - type: "CompanyAssociation", - localKey: "id", - foreignKey: "userId" - }, + // { + // type: "CompanyAssociation", + // localKey: "id", + // foreignKey: "userId" + // }, { type: "FeatureFlag", localKey: "id", From 7e1dbc054e5d055b9a2b3424af97ea76c2b90f38 Mon Sep 17 00:00:00 2001 From: silto Date: Wed, 26 Jun 2024 13:55:13 +0200 Subject: [PATCH 6/8] fix(partial-backup): catch db errors and cleanup comments --- libs/back/partial-backup/src/main.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts index 8f3d24668a..13e7421331 100644 --- a/libs/back/partial-backup/src/main.ts +++ b/libs/back/partial-backup/src/main.ts @@ -149,7 +149,12 @@ const run = async () => { /* Load the root BSD */ - const bsds = await pipelines[originType].getter(originId, rootObjId); + let bsds; + try { + bsds = await pipelines[originType].getter(originId, rootObjId); + } catch (error) { + console.log(error); + } const bsd = bsds?.[0]; if (!bsd) { console.error("Root BSD not found"); @@ -183,16 +188,11 @@ const run = async () => { return; } for (const item of traversals[root.type]) { - // console.log( - // `fetching ${item.type}, fKey: ${item.foreignKey}, lKey: ${ - // root.obj[item.localKey] - // }` - // ); const getter = pipelines[item.type]?.getter; if (!getter) { // console.log(`MISSING GETTER FOR ${item.type}`); + continue; } - // const setter = pipelines[item.type]?.setter; const objects = await getter?.(item.foreignKey, root.obj[item.localKey]); const filteredObjects = objects?.filter(obj => { if (alreadyFetched[obj.id]) { @@ -202,7 +202,6 @@ const run = async () => { type: item.type, obj }; - // alreadyFetched[obj.id] = true; return true; }); if (filteredObjects?.length) { From 6155d39f8110c8f7055bc092a2c65e3cd4355ac5 Mon Sep 17 00:00:00 2001 From: silto Date: Wed, 26 Jun 2024 14:58:08 +0200 Subject: [PATCH 7/8] fix(scripts): limit traversal through MembershipRequest --- libs/back/partial-backup/src/traversals.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/back/partial-backup/src/traversals.ts b/libs/back/partial-backup/src/traversals.ts index f6359bc53b..83ba0ba36b 100644 --- a/libs/back/partial-backup/src/traversals.ts +++ b/libs/back/partial-backup/src/traversals.ts @@ -520,11 +520,11 @@ const traversals = { localKey: "id", foreignKey: "userId" }, - { - type: "MembershipRequest", - localKey: "id", - foreignKey: "userId" - }, + // { + // type: "MembershipRequest", + // localKey: "id", + // foreignKey: "userId" + // }, // { // type: "StatusLog", // localKey: "id", From b2e5a37872fe018bef8959faaa994d47773833c2 Mon Sep 17 00:00:00 2001 From: silto Date: Tue, 9 Jul 2024 17:43:55 +0200 Subject: [PATCH 8/8] fix(Partial Dump): small fixes after review --- .../dist/libs/back/partial-backup/package.json | 6 ------ libs/back/partial-backup/src/main.ts | 5 +---- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 libs/back/partial-backup/dist/libs/back/partial-backup/package.json diff --git a/libs/back/partial-backup/dist/libs/back/partial-backup/package.json b/libs/back/partial-backup/dist/libs/back/partial-backup/package.json deleted file mode 100644 index 856e84d80f..0000000000 --- a/libs/back/partial-backup/dist/libs/back/partial-backup/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "partial-backup", - "version": "0.0.1", - "main": "./main.js", - "type": "commonjs" -} diff --git a/libs/back/partial-backup/src/main.ts b/libs/back/partial-backup/src/main.ts index 13e7421331..21149341d1 100644 --- a/libs/back/partial-backup/src/main.ts +++ b/libs/back/partial-backup/src/main.ts @@ -304,7 +304,4 @@ const run = async () => { ); }; -run().then( - () => rl.close(), - () => rl.close() -); +run().finally(() => rl.close());