From d82af2bbe338dd24ad11228cf0b6f344095944e5 Mon Sep 17 00:00:00 2001 From: Azlam <43767972+azlam-abdulsalam@users.noreply.github.com> Date: Sat, 11 Feb 2023 08:57:20 +1100 Subject: [PATCH] fix(pool): fix error handling (#1220) * fix(pool): fix error handling fix incorrect handling of errors during timeout etc make it more robust [ci skip] --- .../src/scratchorg/pool/PoolCreateImpl.ts | 200 +++++++++++------- .../src/impl/prepare/PrepareOrgJob.ts | 4 +- 2 files changed, 122 insertions(+), 82 deletions(-) diff --git a/packages/core/src/scratchorg/pool/PoolCreateImpl.ts b/packages/core/src/scratchorg/pool/PoolCreateImpl.ts index 488df2b9f..efff5eded 100644 --- a/packages/core/src/scratchorg/pool/PoolCreateImpl.ts +++ b/packages/core/src/scratchorg/pool/PoolCreateImpl.ts @@ -17,7 +17,6 @@ import { EOL } from 'os'; import OrgDetailsFetcher from '../../org/OrgDetailsFetcher'; import ScratchOrgOperator from '../ScratchOrgOperator'; import PoolFetchImpl from './PoolFetchImpl'; -import { values } from 'lodash'; import { COLOR_SUCCESS } from '@dxatscale/sfp-logger'; import { COLOR_ERROR } from '@dxatscale/sfp-logger'; import getFormattedTime from '../../utils/GetFormattedTime'; @@ -66,42 +65,60 @@ export default class PoolCreateImpl extends PoolBaseImpl { fs.mkdirpSync('script_exec_outputs'); //Compute allocation - if (!this.pool.snapshotPool) { - SFPLogger.log(COLOR_KEY_MESSAGE('Computing Allocation..'), LoggerLevel.INFO); - try { - this.totalToBeAllocated = await this.computeAllocation(); - } catch (error) { - return err({ - success: 0, - failed: 0, - message: `Unable to access fields on ScratchOrgInfo, Please check the profile being used`, - errorCode: PoolErrorCodes.PrerequisiteMissing, - }); - } - - if (this.totalToBeAllocated === 0) { - if (this.limits.ActiveScratchOrgs.Remaining > 0) { - return err({ - success: 0, - failed: 0, - message: `The tag provided ${this.pool.tag} is currently at the maximum capacity , No scratch orgs will be allocated`, - errorCode: PoolErrorCodes.Max_Capacity, - }); - } else { + try { + if (!this.pool.snapshotPool) { + SFPLogger.log(COLOR_KEY_MESSAGE('Computing Allocation..'), LoggerLevel.INFO); + try { + this.totalToBeAllocated = await this.computeAllocation(); + } catch (error) { return err({ success: 0, failed: 0, - message: `There is no capacity to create a pool at this time, Please try again later`, - errorCode: PoolErrorCodes.No_Capacity, + message: `Unable to access fields on ScratchOrgInfo, Please check the profile being used`, + errorCode: PoolErrorCodes.PrerequisiteMissing, }); } - } - //Generate Scratch Orgs - await this.generateScratchOrgs(); - } else { - await this.fetchScratchOrgsFromSnapshotPool(); + if (this.totalToBeAllocated === 0) { + if (this.limits.ActiveScratchOrgs.Remaining > 0) { + return err({ + success: 0, + failed: 0, + message: `The tag provided ${this.pool.tag} is currently at the maximum capacity , No scratch orgs will be allocated`, + errorCode: PoolErrorCodes.Max_Capacity, + }); + } else { + return err({ + success: 0, + failed: 0, + message: `There is no capacity to create a pool at this time, Please try again later`, + errorCode: PoolErrorCodes.No_Capacity, + }); + } + } + + //Generate Scratch Orgs + this.pool.scratchOrgs = await this.generateScratchOrgs( + this.pool, + this.scratchOrgOperator, + this.scratchOrgInfoAssigner + ); + } else { + this.pool.scratchOrgs = await this.fetchScratchOrgsFromSnapshotPool( + this.pool, + this.scratchOrgInfoFetcher, + this.scratchOrgInfoAssigner + ); + } + } catch (error) { + return err({ + success: 0, + failed: this.pool.failedToCreate, + message: `All requested scratch orgs failed to provision, Please check your code or config \n Failed with ${error.message}`, + errorCode: PoolErrorCodes.UnableToProvisionAny, + }); } + // Assign workers to executed scripts for (let scratchOrg of this.pool.scratchOrgs) { let result = this.scriptExecutorWrappedForBottleneck(scratchOrg, this.hubOrg.getUsername()); @@ -110,9 +127,13 @@ export default class PoolCreateImpl extends PoolBaseImpl { await Promise.all(scriptExecPromises); - await this.finalizeGeneratedScratchOrgs(); + this.pool = await this.finalizeGeneratedScratchOrgs( + this.pool, + this.scratchOrgOperator, + this.scratchOrgInfoFetcher + ); - if (this.pool.scratchOrgs.length == 0) { + if (!this.pool.scratchOrgs || this.pool.scratchOrgs.length == 0) { return err({ success: 0, failed: this.pool.failedToCreate, @@ -154,20 +175,24 @@ export default class PoolCreateImpl extends PoolBaseImpl { return pool.to_allocate; } - private async generateScratchOrgs() { + private async generateScratchOrgs( + pool: PoolConfig, + scratchOrgOperator: ScratchOrgOperator, + scratchOrgInfoAssigner: ScratchOrgInfoAssigner + ) { //Generate Scratch Orgs SFPLogger.log(COLOR_KEY_MESSAGE('Generate Scratch Orgs..'), LoggerLevel.INFO); let scratchOrgPromises = new Array>(); const scratchOrgCreationLimiter = new Bottleneck({ - maxConcurrent: this.pool.batchSize, + maxConcurrent: pool.batchSize, }); let startTime = Date.now(); - for (let i = 1; i <= this.pool.to_allocate; i++) { + for (let i = 1; i <= pool.to_allocate; i++) { let scratchOrgPromise: Promise = scratchOrgCreationLimiter.schedule(() => - this.scratchOrgOperator.create(`SO` + i, this.pool.configFilePath, this.pool.expiry, this.pool.waitTime) + scratchOrgOperator.create(`SO` + i, this.pool.configFilePath, this.pool.expiry, this.pool.waitTime) ); scratchOrgPromises.push(scratchOrgPromise); } @@ -179,51 +204,51 @@ export default class PoolCreateImpl extends PoolBaseImpl { const isFulfilled = (p: PromiseSettledResult): p is PromiseFulfilledResult => p.status === 'fulfilled'; const isRejected = (p: PromiseSettledResult): p is PromiseRejectedResult => p.status === 'rejected'; - this.pool.scratchOrgs = scratchOrgCreationResults.filter(isFulfilled).map((p) => p.value); + let scratchOrgs = scratchOrgCreationResults.filter(isFulfilled).map((p) => p.value); const rejectedScratchOrgs = scratchOrgCreationResults.filter(isRejected).map((p) => p.reason); for (const reason of rejectedScratchOrgs) { if (reason.message.includes(`The client has timed out`)) { - //Display how many we were able to create - let elapsedTime = Date.now() - startTime; + //Log how many we were able to create + let elapsedTime = Date.now() - startTime; SFPLogger.log( - `A scratch org creation was rejected due to saleforce not responding within the set wait time of ${this.pool.waitTime} mins \n` + - `Time elasped so far ${COLOR_KEY_MESSAGE(getFormattedTime(elapsedTime))},You might need to inrease the wait time further and rety ` + `A scratch org creation was rejected due to saleforce not responding within the set wait time of ${pool.waitTime} mins \n` + + `Time elasped so far ${COLOR_KEY_MESSAGE( + getFormattedTime(elapsedTime) + )},You might need to inrease the wait time further and rety ` ); } else SFPLogger.log(`A scratch org creation was rejected due to ${reason.message}`); } - //Display how many we were able to create + //Log how many we were able to create let elapsedTime = Date.now() - startTime; SFPLogger.log( - `Created ${COLOR_SUCCESS(this.pool.scratchOrgs.length)} of ${ - this.totalToBeAllocated - } successfully with ${COLOR_ERROR(rejectedScratchOrgs.length)} failures in ${COLOR_KEY_MESSAGE( - getFormattedTime(elapsedTime) - )}` + `Created ${COLOR_SUCCESS(scratchOrgs.length)} of ${pool.to_allocate} successfully with ${COLOR_ERROR( + rejectedScratchOrgs.length + )} failures in ${COLOR_KEY_MESSAGE(getFormattedTime(elapsedTime))}` ); - SFPStatsSender.logElapsedTime(`pool.scratchorg.creation.time`,elapsedTime,{pool:this.pool.tag}) - //Splice scratchorgs that are having incorrect status of deleted , Why salesforce why?? - let index = this.pool.scratchOrgs.length; - while (index--) { - try { - let orgDetails = await new OrgDetailsFetcher(this.pool.scratchOrgs[index].username).getOrgDetails(); - if (orgDetails.status === 'Deleted') { - throw new Error( - `Throwing away scratch org ${this.pool.scratchOrgs[index].alias} as it has a status of deleted` - ); + SFPStatsSender.logElapsedTime(`pool.scratchorg.creation.time`, elapsedTime, { pool: pool.tag }); + if (scratchOrgs && scratchOrgs.length > 0) { + //Splice scratchorgs that are having incorrect status of deleted , Why salesforce why?? + let index = scratchOrgs.length; + while (index--) { + try { + let orgDetails = await new OrgDetailsFetcher(scratchOrgs[index].username).getOrgDetails(); + if (orgDetails.status === 'Deleted') { + throw new Error( + `Throwing away scratch org ${this.pool.scratchOrgs[index].alias} as it has a status of deleted` + ); + } + } catch (error) { + scratchOrgs.splice(index, 1); } - } catch (error) { - this.pool.scratchOrgs.splice(index, 1); } - } - this.pool.scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(this.pool.scratchOrgs); + scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(scratchOrgs); - let scratchOrgInprogress = []; + let scratchOrgInprogress = []; - if (this.pool.scratchOrgs) { - this.pool.scratchOrgs.forEach((scratchOrg) => { + scratchOrgs.forEach((scratchOrg) => { scratchOrgInprogress.push({ Id: scratchOrg.recordId, Pooltag__c: this.pool.tag, @@ -235,19 +260,24 @@ export default class PoolCreateImpl extends PoolBaseImpl { if (scratchOrgInprogress.length > 0) { //set pool tag - await this.scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress); + await scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress); } - } + return scratchOrgs; + } else throw new Error(`No scratch orgs were sucesfully generated`); } - private async fetchScratchOrgsFromSnapshotPool() { + private async fetchScratchOrgsFromSnapshotPool( + pool: PoolConfig, + scratchOrgInfoFetcher: ScratchOrgInfoFetcher, + scratchOrgInfoAssigner: ScratchOrgInfoAssigner + ) { //Generate Scratch Orgs SFPLogger.log( COLOR_KEY_MESSAGE(`Fetching Scratch Orgs from snapshot pool ${this.pool.snapshotPool}`), LoggerLevel.INFO ); - this.pool.scratchOrgs = (await new PoolFetchImpl( + let scratchOrgs = (await new PoolFetchImpl( this.hubOrg, this.pool.snapshotPool, false, @@ -258,12 +288,12 @@ export default class PoolCreateImpl extends PoolBaseImpl { true, this.pool.maxAllocation ).execute()) as ScratchOrg[]; - this.pool.scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(this.pool.scratchOrgs); + scratchOrgs = await scratchOrgInfoFetcher.getScratchOrgRecordId(scratchOrgs); let scratchOrgInprogress = []; - if (this.pool.scratchOrgs) { - this.pool.scratchOrgs.forEach((scratchOrg) => { + if (scratchOrgs && scratchOrgs.length>0) { + scratchOrgs.forEach((scratchOrg) => { scratchOrgInprogress.push({ Id: scratchOrg.recordId, Pooltag__c: this.pool.tag, @@ -275,15 +305,24 @@ export default class PoolCreateImpl extends PoolBaseImpl { if (scratchOrgInprogress.length > 0) { //set pool tag - await this.scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress); + await scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress); } + return scratchOrgs; + } + else + { + throw new Error('No scratch orgs were found to be fetched'); } } - private async finalizeGeneratedScratchOrgs() { - this.pool.failedToCreate = 0; - for (let i = this.pool.scratchOrgs.length - 1; i >= 0; i--) { - let scratchOrg = this.pool.scratchOrgs[i]; + private async finalizeGeneratedScratchOrgs( + pool: PoolConfig, + scratchOrgOperator: ScratchOrgOperator, + scratchOrgInfoFetcher: ScratchOrgInfoFetcher + ) { + pool.failedToCreate = 0; + for (let i = pool.scratchOrgs.length - 1; i >= 0; i--) { + let scratchOrg = pool.scratchOrgs[i]; if (scratchOrg.isScriptExecuted) { continue; } @@ -296,11 +335,11 @@ export default class PoolCreateImpl extends PoolBaseImpl { try { //Delete scratchorgs that failed to execute script - let activeScratchOrgRecordId = await this.scratchOrgInfoFetcher.getActiveScratchOrgRecordIdGivenScratchOrg( + let activeScratchOrgRecordId = await scratchOrgInfoFetcher.getActiveScratchOrgRecordIdGivenScratchOrg( scratchOrg.orgId ); - await this.scratchOrgOperator.delete([activeScratchOrgRecordId]); + await scratchOrgOperator.delete([activeScratchOrgRecordId]); console.log(`Succesfully deleted scratchorg ${scratchOrg.username}`); } catch (error) { SFPLogger.log( @@ -309,9 +348,10 @@ export default class PoolCreateImpl extends PoolBaseImpl { ); } - this.pool.failedToCreate += 1; - this.pool.scratchOrgs.splice(i, 1); + pool.failedToCreate += 1; + pool.scratchOrgs.splice(i, 1); } + return pool; } private async scriptExecutor(scratchOrg: ScratchOrg): Promise { diff --git a/packages/sfpowerscripts-cli/src/impl/prepare/PrepareOrgJob.ts b/packages/sfpowerscripts-cli/src/impl/prepare/PrepareOrgJob.ts index 00570b020..03eed3e82 100644 --- a/packages/sfpowerscripts-cli/src/impl/prepare/PrepareOrgJob.ts +++ b/packages/sfpowerscripts-cli/src/impl/prepare/PrepareOrgJob.ts @@ -163,8 +163,8 @@ export default class PrepareOrgJob extends PoolJobExecutor implements PreDeployH logger: FileLogger, deploymentMode: DeploymentMode ) { - SFPLogger.log(`Deploying packages in the repo to ${scratchOrg.alias}`); - SFPLogger.log(`Deploying packages in the repo to ${scratchOrg.alias}`, LoggerLevel.INFO, logger); + SFPLogger.log(`Deploying packages to ${scratchOrg.alias}`); + SFPLogger.log(`Deploying packages to ${scratchOrg.alias}`, LoggerLevel.INFO, logger); let deployProps: DeployProps = { targetUsername: scratchOrg.username,