From 81264fe5792cfcef6e66c63feeb3751d76a8870f Mon Sep 17 00:00:00 2001 From: Pl217 Date: Thu, 8 Feb 2024 17:30:32 +0100 Subject: [PATCH 1/4] Consider version in `getAllGlobalClustersForPlan` --- src/lib/data/globalClusters.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/data/globalClusters.ts b/src/lib/data/globalClusters.ts index 0dd2e7b0..8b997233 100644 --- a/src/lib/data/globalClusters.ts +++ b/src/lib/data/globalClusters.ts @@ -8,12 +8,19 @@ import { Op } from '../../db/util/conditions'; export const getAllGlobalClustersForPlan = async ({ database, planId, + version, }: { database: Database; planId: PlanId; + version: 'current' | 'latest'; }) => { const governingEntities = await database.governingEntity.find({ - where: { planId }, + where: { + planId, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), + }, }); const globalClusterAssociations = From ea90d9434b9c4ce38b239d93af5e153b780122b4 Mon Sep 17 00:00:00 2001 From: Pl217 Date: Thu, 8 Feb 2024 17:31:40 +0100 Subject: [PATCH 2/4] Don't default to latest version Many fetching utility methods accepted `version` parameter, but only allowed `'latest'` to be used. Now, allow for `'current'` to be used as well. To ensure versions are fetched correctly, base entities (attachment, planEntity and governingEntity) need to respect the version too --- src/lib/data/attachments.ts | 10 ++++++++-- src/lib/data/governingEntities.ts | 10 ++++++++-- src/lib/data/planEntities.ts | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/lib/data/attachments.ts b/src/lib/data/attachments.ts index c1dea86d..ef8785f5 100644 --- a/src/lib/data/attachments.ts +++ b/src/lib/data/attachments.ts @@ -103,6 +103,7 @@ export const getAllAttachments = async ({ database, planId, types, + version, planEntities, governingEntities, log, @@ -110,7 +111,7 @@ export const getAllAttachments = async ({ database: Database; planId: PlanId; types: AttachmentType[]; - version: 'latest'; + version: 'latest' | 'current'; /** * A map of plan entities, which **must** include any entity that is the * parent of the given attachment. @@ -146,6 +147,9 @@ export const getAllAttachments = async ({ type: { [Op.IN]: types, }, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }); const attachmentVersionsByAttachmentId = @@ -154,10 +158,12 @@ export const getAllAttachments = async ({ (t) => t.find({ where: { - latestVersion: true, attachmentId: { [Op.IN]: attachments.map((pa) => pa.id), }, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }), 'attachmentId' diff --git a/src/lib/data/governingEntities.ts b/src/lib/data/governingEntities.ts index ee272df9..bb3657fc 100644 --- a/src/lib/data/governingEntities.ts +++ b/src/lib/data/governingEntities.ts @@ -25,11 +25,12 @@ export type MapOfGoverningEntities = AnnotatedMap< export const getAllGoverningEntitiesForPlan = async ({ database, planId, + version, prototypes, }: { database: Database; planId: PlanId; - version: 'latest'; + version: 'latest' | 'current'; /** * A map of prototypes that **must** include the prototype for governing * entities of this plan. @@ -44,6 +45,9 @@ export const getAllGoverningEntitiesForPlan = async ({ const ges = await database.governingEntity.find({ where: { planId, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }); @@ -52,10 +56,12 @@ export const getAllGoverningEntitiesForPlan = async ({ (t) => t.find({ where: { - latestVersion: true, governingEntityId: { [Op.IN]: ges.map((ge) => ge.id), }, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }), 'governingEntityId' diff --git a/src/lib/data/planEntities.ts b/src/lib/data/planEntities.ts index e47f1cb7..ecf122e5 100644 --- a/src/lib/data/planEntities.ts +++ b/src/lib/data/planEntities.ts @@ -41,6 +41,7 @@ export const getAndValidateAllPlanEntities = async ({ database, planId, governingEntities, + version, prototypes, allowMissingPlanEntities, }: { @@ -51,7 +52,7 @@ export const getAndValidateAllPlanEntities = async ({ * generating references */ governingEntities: MapOfGoverningEntities; - version: 'latest'; + version: 'latest' | 'current'; /** * A map of prototypes that **must** include the prototype for governing * entities of this plan. @@ -74,6 +75,9 @@ export const getAndValidateAllPlanEntities = async ({ const planEntities = await database.planEntity.find({ where: { planId, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }); const planEntityIDs = new Set(planEntities.map((pe) => pe.id)); @@ -83,10 +87,12 @@ export const getAndValidateAllPlanEntities = async ({ (t) => t.find({ where: { - latestVersion: true, planEntityId: { [Op.IN]: planEntityIDs, }, + ...(version === 'latest' + ? { latestVersion: true } + : { currentVersion: true }), }, }), 'planEntityId' From 2304dacc6170ae3d303086b1adfbea21001efaa6 Mon Sep 17 00:00:00 2001 From: Pl217 Date: Thu, 8 Feb 2024 18:11:28 +0100 Subject: [PATCH 3/4] Allow to skip validation when querying tables with JSON columns This will allow to significantly speed up requests which fetch tons of data --- src/lib/data/attachments.ts | 31 ++++++++++++++++++++++++++----- src/lib/data/governingEntities.ts | 7 +++++++ src/lib/data/planEntities.ts | 7 +++++++ src/lib/data/projects.ts | 7 +++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/lib/data/attachments.ts b/src/lib/data/attachments.ts index ef8785f5..0524de9d 100644 --- a/src/lib/data/attachments.ts +++ b/src/lib/data/attachments.ts @@ -107,6 +107,7 @@ export const getAllAttachments = async ({ planEntities, governingEntities, log, + skipValidation = false, }: { database: Database; planId: PlanId; @@ -127,6 +128,13 @@ export const getAllAttachments = async ({ */ governingEntities: MapOfGoverningEntities; log: SharedLogContext; + /** + * Skip validation when fetching data from tables which have + * JSON columns, as those are expensive to verify. + * + * Also, skip type-checking attachment values against their types. + */ + skipValidation?: boolean; }): Promise => { const attachmentPrototypesById = await findAndOrganizeObjectsByUniqueProperty( database.attachmentPrototype, @@ -138,6 +146,7 @@ export const getAllAttachments = async ({ [Op.IN]: types, }, }, + skipValidation, }), 'id' ); @@ -165,6 +174,7 @@ export const getAllAttachments = async ({ ? { latestVersion: true } : { currentVersion: true }), }, + skipValidation, }), 'attachmentId' ); @@ -206,11 +216,22 @@ export const getAllAttachments = async ({ }) ); } - const data = typeCheckAttachmentData({ - attachment, - attachmentVersion, - log, - }); + + let data: AttachmentData; + if (!skipValidation) { + data = typeCheckAttachmentData({ + attachment, + attachmentVersion, + log, + }); + } else { + // Here, we sacrifice type-safety for speed, if `skipValidation` is enabled + data = { + type: attachment.type, + value: attachmentVersion.value, + } as unknown as AttachmentData; + } + result.set(attachment.id, { id: attachment.id, customRef, diff --git a/src/lib/data/governingEntities.ts b/src/lib/data/governingEntities.ts index bb3657fc..b5bd073c 100644 --- a/src/lib/data/governingEntities.ts +++ b/src/lib/data/governingEntities.ts @@ -27,6 +27,7 @@ export const getAllGoverningEntitiesForPlan = async ({ planId, version, prototypes, + skipValidation = false, }: { database: Database; planId: PlanId; @@ -41,6 +42,11 @@ export const getAllGoverningEntitiesForPlan = async ({ EntityPrototypeId, InstanceDataOfModel >; + /** + * Skip validation when fetching data from tables which have + * JSON columns, as those are expensive to verify + */ + skipValidation?: boolean; }): Promise => { const ges = await database.governingEntity.find({ where: { @@ -63,6 +69,7 @@ export const getAllGoverningEntitiesForPlan = async ({ ? { latestVersion: true } : { currentVersion: true }), }, + skipValidation, }), 'governingEntityId' ); diff --git a/src/lib/data/planEntities.ts b/src/lib/data/planEntities.ts index ecf122e5..4c92e794 100644 --- a/src/lib/data/planEntities.ts +++ b/src/lib/data/planEntities.ts @@ -44,6 +44,7 @@ export const getAndValidateAllPlanEntities = async ({ version, prototypes, allowMissingPlanEntities, + skipValidation = false, }: { database: Database; planId: PlanId; @@ -71,6 +72,11 @@ export const getAndValidateAllPlanEntities = async ({ * This can happen for example when another entity is soft-deleted. */ allowMissingPlanEntities?: boolean; + /** + * Skip validation when fetching data from tables which have + * JSON columns, as those are expensive to verify + */ + skipValidation?: boolean; }): Promise => { const planEntities = await database.planEntity.find({ where: { @@ -94,6 +100,7 @@ export const getAndValidateAllPlanEntities = async ({ ? { latestVersion: true } : { currentVersion: true }), }, + skipValidation, }), 'planEntityId' ); diff --git a/src/lib/data/projects.ts b/src/lib/data/projects.ts index 348306e6..ee957bed 100644 --- a/src/lib/data/projects.ts +++ b/src/lib/data/projects.ts @@ -258,6 +258,7 @@ export const getProjectBudgetsByOrgAndCluster = async < projects, log, ignoreInconsistentBudgets, + skipValidation = false, }: { database: Database; projects: Map; @@ -273,6 +274,11 @@ export const getProjectBudgetsByOrgAndCluster = async < * to for example calculate a plan's overall requirements. */ ignoreInconsistentBudgets?: true; + /** + * Skip validation when fetching data from tables which have + * JSON columns, as those are expensive to verify + */ + skipValidation?: boolean; }): Promise> => { const projectVersionIds = [...projects.values()].map( (p) => p.projectVersion.id @@ -298,6 +304,7 @@ export const getProjectBudgetsByOrgAndCluster = async < [Op.IN]: segments.map((s) => s.id), }, }, + skipValidation, }); const breakdownsBySegment = groupObjectsByProperty( From 8c8fa01099de6496c0f6c8be07908da115ba53d2 Mon Sep 17 00:00:00 2001 From: Pl217 Date: Mon, 11 Mar 2024 16:43:53 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=96=20Bump=20version=20to=20`v7.5.?= =?UTF-8?q?0`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b8a9da31..167530fe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@unocha/hpc-api-core", - "version": "7.4.0", + "version": "7.5.0", "description": "Core libraries supporting HPC.Tools API Backend", "license": "Apache-2.0", "private": false,