From bed399feb6ba093ef73d553bbf24081931e520a2 Mon Sep 17 00:00:00 2001 From: bduran Date: Tue, 10 Sep 2024 17:09:04 -0700 Subject: [PATCH 1/3] utilize bulk query transactions --- src/packages/plan/gql.ts | 23 +++--- src/packages/plan/plan.ts | 148 ++++++++++++++++++++------------------ 2 files changed, 89 insertions(+), 82 deletions(-) diff --git a/src/packages/plan/gql.ts b/src/packages/plan/gql.ts index 89ceb3e..10c1fd0 100644 --- a/src/packages/plan/gql.ts +++ b/src/packages/plan/gql.ts @@ -1,9 +1,11 @@ export default { - CREATE_ACTIVITY_DIRECTIVE: `#graphql - mutation CreateActivityDirective($activityDirectiveInsertInput: activity_directive_insert_input!) { - insert_activity_directive_one(object: $activityDirectiveInsertInput) { - id - type + CREATE_ACTIVITY_DIRECTIVES: `#graphql + mutation CreateActivityDirectives($activityDirectivesInsertInput: [activity_directive_insert_input!]!) { + insert_activity_directive(objects: $activityDirectivesInsertInput) { + returning { + id + type + } } } `, @@ -70,13 +72,12 @@ export default { } } `, - UPDATE_ACTIVITY_DIRECTIVE: `#graphql - mutation UpdateActivityDirective($id: Int!, $plan_id: Int!, $activityDirectiveSetInput: activity_directive_set_input!) { - update_activity_directive_by_pk( - pk_columns: { id: $id, plan_id: $plan_id }, _set: $activityDirectiveSetInput + UPDATE_ACTIVITY_DIRECTIVES: `#graphql + mutation UpdateActivityDirective($updates: [activity_directive_updates!]!) { + update_activity_directive_many( + updates: $updates ) { - anchor_id - id + affected_rows } } `, diff --git a/src/packages/plan/plan.ts b/src/packages/plan/plan.ts index 9381efd..89df699 100644 --- a/src/packages/plan/plan.ts +++ b/src/packages/plan/plan.ts @@ -8,7 +8,6 @@ import { auth } from '../auth/middleware.js'; import type { ActivityDirective, ActivityDirectiveInsertInput, - ActivityDirectiveSetInput, ImportPlanPayload, PlanInsertInput, PlanSchema, @@ -181,87 +180,94 @@ export async function importPlan(req: Request, res: Response) { } const activityRemap: Record = {}; - await Promise.all( - activities.map( - async ({ + const activityDirectivesInsertInput = activities.map( + ({ + anchored_to_start: anchoredToStart, + arguments: activityArguments, + metadata, + name: activityName, + start_offset: startOffset, + tags, + type, + }) => { + const activityDirectiveInsertInput: ActivityDirectiveInsertInput = { + anchor_id: null, anchored_to_start: anchoredToStart, arguments: activityArguments, - id, metadata, name: activityName, + plan_id: (createdPlan as PlanSchema).id, start_offset: startOffset, - tags, + tags: { + data: + tags?.map(({ tag: { name } }) => ({ + tag_id: tagsMap[name].id, + })) ?? [], + }, type, - }) => { - const activityDirectiveInsertInput: ActivityDirectiveInsertInput = { - anchor_id: null, - anchored_to_start: anchoredToStart, - arguments: activityArguments, - metadata, - name: activityName, - plan_id: (createdPlan as PlanSchema).id, - start_offset: startOffset, - tags: { - data: - tags?.map(({ tag: { name } }) => ({ - tag_id: tagsMap[name].id, - })) ?? [], - }, - type, - }; - - const createdActivityDirectiveResponse = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.CREATE_ACTIVITY_DIRECTIVE, - variables: { activityDirectiveInsertInput }, - }), - headers, - method: 'POST', - }); - - const createdActivityDirectiveData = (await createdActivityDirectiveResponse.json()) as { - data: { - insert_activity_directive_one: ActivityDirective; - }; - } | null; - - if (createdActivityDirectiveData) { - const { - data: { insert_activity_directive_one: createdActivityDirective }, - } = createdActivityDirectiveData; - activityRemap[id] = createdActivityDirective.id; - } - }, - ), + }; + + return activityDirectiveInsertInput; + }, ); + const createdActivitiesResponse = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.CREATE_ACTIVITY_DIRECTIVES, + variables: { + activityDirectivesInsertInput, + }, + }), + headers, + method: 'POST', + }); + + const createdActivityDirectivesData = (await createdActivitiesResponse.json()) as { + data: { + insert_activity_directive: { + returning: ActivityDirective[]; + }; + }; + } | null; + + if (createdActivityDirectivesData) { + const { + data: { + insert_activity_directive: { returning: createdActivityDirectives }, + }, + } = createdActivityDirectivesData; + + if (createdActivityDirectives.length === activities.length) { + createdActivityDirectives.forEach((createdActivityDirective, index) => { + const { id } = activities[index]; + + activityRemap[id] = createdActivityDirective.id; + }); + } else { + throw new Error('Activity insertion failed.'); + } + } + // remap all the anchor ids to the newly created activity directives logger.info(`POST /importPlan: Re-assigning anchors: ${name}`); - await Promise.all( - activities.map(async ({ anchor_id: anchorId, id }) => { - if (anchorId !== null && activityRemap[anchorId] != null && activityRemap[id] != null) { - logger.info( - `POST /importPlan: Re-assigning anchor ${anchorId} to ${activityRemap[anchorId]} for activity ${activityRemap[id]}: ${name}`, - ); - const activityDirectiveSetInput: ActivityDirectiveSetInput = { - anchor_id: activityRemap[anchorId], - }; - - return fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.UPDATE_ACTIVITY_DIRECTIVE, - variables: { - activityDirectiveSetInput, - id: activityRemap[id], - plan_id: (createdPlan as PlanSchema).id, - }, - }), - headers, - method: 'POST', - }); - } + + const activityDirectivesSetInput = activities + .filter(({ anchor_id: anchorId }) => anchorId !== null) + .map(({ anchor_id: anchorId, id }) => ({ + _set: { anchor_id: activityRemap[anchorId as number] }, + where: { id: { _eq: activityRemap[id] }, plan_id: { _eq: (createdPlan as PlanSchema).id } }, + })); + + await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.UPDATE_ACTIVITY_DIRECTIVES, + variables: { + updates: activityDirectivesSetInput, + }, }), - ); + headers, + method: 'POST', + }); // associate the tags with the newly created plan logger.info(`POST /importPlan: Importing plan tags: ${name}`); From c517dd576b40dc5aad0018f0d1a0b32ce1decf1d Mon Sep 17 00:00:00 2001 From: bduran Date: Wed, 11 Sep 2024 14:00:17 -0700 Subject: [PATCH 2/3] cleanup tags on failure --- src/packages/plan/gql.ts | 11 +++++++++++ src/packages/plan/plan.ts | 24 ++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/packages/plan/gql.ts b/src/packages/plan/gql.ts index 10c1fd0..16c79f5 100644 --- a/src/packages/plan/gql.ts +++ b/src/packages/plan/gql.ts @@ -61,6 +61,17 @@ export default { } } `, + DELETE_TAGS: `#graphql + mutation DeleteTags($tagIds: [Int!]! = []) { + delete_tags( + where: { + id: { _in: $tagIds } + } + ) { + affected_rows + } + } + `, GET_TAGS: `#graphql query GetTags { tags(order_by: { name: desc }) { diff --git a/src/packages/plan/plan.ts b/src/packages/plan/plan.ts index 89df699..d61f71c 100644 --- a/src/packages/plan/plan.ts +++ b/src/packages/plan/plan.ts @@ -52,6 +52,7 @@ export async function importPlan(req: Request, res: Response) { }; let createdPlan: PlanSchema | null = null; + let createdTags: Tag[] = []; try { const { activities, simulation_arguments }: PlanTransfer = await new Promise(resolve => { @@ -143,7 +144,7 @@ export async function importPlan(req: Request, res: Response) { {}, ); - await fetch(GQL_API_URL, { + const createdTagsResponse = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.CREATE_TAGS, variables: { tags: Object.values(activityTags) }, @@ -152,6 +153,16 @@ export async function importPlan(req: Request, res: Response) { method: 'POST', }); + const { data } = (await createdTagsResponse.json()) as { + data: { + insert_tags: { returning: Tag[] }; + }; + }; + + if (data && data.insert_tags && data.insert_tags.returning.length) { + createdTags = data.insert_tags.returning; + } + const tagsResponse = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.GET_TAGS, @@ -294,14 +305,23 @@ export async function importPlan(req: Request, res: Response) { logger.error(`POST /importPlan: Error occurred during plan ${name} import`); logger.error(error); + // cleanup the imported plan if it failed along the way if (createdPlan) { + // delete the plan - activities associated to the plan will be automatically cleaned up await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.DELETE_PLAN, variables: { id: createdPlan.id } }), headers, method: 'POST', }); + + // if any activity tags were created as a result of this import, remove them + await fetch(GQL_API_URL, { + body: JSON.stringify({ query: gql.DELETE_TAGS, variables: { tagIds: createdTags.map(({ id }) => id) } }), + headers, + method: 'POST', + }); } - res.send(500); + res.sendStatus(500); } } From 264afda5ed21d64db968546bef13ce6c1108a21c Mon Sep 17 00:00:00 2001 From: bduran Date: Tue, 1 Oct 2024 18:01:40 -0700 Subject: [PATCH 3/3] be more explicit as to what new tags are created --- src/packages/plan/gql.ts | 6 +-- src/packages/plan/plan.ts | 90 +++++++++++++++++++++++---------------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/packages/plan/gql.ts b/src/packages/plan/gql.ts index 16c79f5..1f49ac0 100644 --- a/src/packages/plan/gql.ts +++ b/src/packages/plan/gql.ts @@ -39,11 +39,7 @@ export default { `, CREATE_TAGS: `#graphql mutation CreateTags($tags: [tags_insert_input!]!) { - insert_tags(objects: $tags, on_conflict: { - constraint: tags_name_key, - update_columns: [] - }) { - affected_rows + insert_tags(objects: $tags) { returning { color created_at diff --git a/src/packages/plan/plan.ts b/src/packages/plan/plan.ts index d61f71c..5e31dd2 100644 --- a/src/packages/plan/plan.ts +++ b/src/packages/plan/plan.ts @@ -123,22 +123,57 @@ export async function importPlan(req: Request, res: Response) { // insert all the imported activities into the plan logger.info(`POST /importPlan: Importing activities: ${name}`); + const tagsResponse = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.GET_TAGS, + }), + headers, + method: 'POST', + }); + + const tagsResponseJSON = (await tagsResponse.json()) as { + data: { + tags: Tag[]; + }; + }; + + let tagsMap: Record = {}; + if (tagsResponseJSON != null && tagsResponseJSON.data != null) { + const { + data: { tags }, + } = tagsResponseJSON; + tagsMap = tags.reduce((prevTagsMap: Record, tag) => { + return { + ...prevTagsMap, + [tag.name]: tag, + }; + }, {}); + } + + // derive a map of uniquely named tags from the list of activities that doesn't already exist in the database const activityTags = activities.reduce( (prevActivitiesTagsMap: Record>, { tags }) => { - const tagsMap = - tags?.reduce((prevTagsMap: Record>, { tag }) => { - return { - ...prevTagsMap, - [tag.name]: { - color: tag.color, - name: tag.name, - }, - }; - }, {}) ?? {}; + const currentTagsMap = + tags?.reduce( + (prevTagsMap: Record>, { tag: { name: tagName, color } }) => { + // If the tag doesn't exist already, add it + if (tagsMap[tagName] === undefined) { + return { + ...prevTagsMap, + [tagName]: { + color, + name: tagName, + }, + }; + } + return prevTagsMap; + }, + {}, + ) ?? {}; return { ...prevActivitiesTagsMap, - ...tagsMap, + ...currentTagsMap, }; }, {}, @@ -160,35 +195,18 @@ export async function importPlan(req: Request, res: Response) { }; if (data && data.insert_tags && data.insert_tags.returning.length) { + // track the newly created tags for cleanup if an error occurs during plan import createdTags = data.insert_tags.returning; } - const tagsResponse = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.GET_TAGS, + // add the newly created tags to the `tagsMap` + tagsMap = createdTags.reduce( + (prevTagsMap: Record, tag) => ({ + ...prevTagsMap, + [tag.name]: tag, }), - headers, - method: 'POST', - }); - - const tagsResponseJSON = (await tagsResponse.json()) as { - data: { - tags: Tag[]; - }; - }; - - let tagsMap: Record = {}; - if (tagsResponseJSON != null && tagsResponseJSON.data != null) { - const { - data: { tags }, - } = tagsResponseJSON; - tagsMap = tags.reduce((prevTagsMap: Record, tag) => { - return { - ...prevTagsMap, - [tag.name]: tag, - }; - }, {}); - } + tagsMap, + ); const activityRemap: Record = {}; const activityDirectivesInsertInput = activities.map(