Skip to content
This repository has been archived by the owner on Mar 18, 2024. It is now read-only.

Commit

Permalink
fix(pool): fix error handling (#1220)
Browse files Browse the repository at this point in the history 
* fix(pool): fix error handling

fix incorrect handling of errors during timeout etc
make it more robust


[ci skip]
  • Loading branch information
@azlam-abdulsalam
azlam-abdulsalam authored Feb 10, 2023
1 parent 8f7be32 commit d82af2b
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 82 deletions.
200 changes: 120 additions & 80 deletions packages/core/src/scratchorg/pool/PoolCreateImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import { EOL } from 'os';
import OrgDetailsFetcher from '../../org/OrgDetailsFetcher';
import ScratchOrgOperator from '../ScratchOrgOperator';
import PoolFetchImpl from './PoolFetchImpl';
import { values } from 'lodash';
import { COLOR_SUCCESS } from '@dxatscale/sfp-logger';
import { COLOR_ERROR } from '@dxatscale/sfp-logger';
import getFormattedTime from '../../utils/GetFormattedTime';
Expand Down Expand Up @@ -66,42 +65,60 @@ export default class PoolCreateImpl extends PoolBaseImpl {
fs.mkdirpSync('script_exec_outputs');

//Compute allocation
if (!this.pool.snapshotPool) {
SFPLogger.log(COLOR_KEY_MESSAGE('Computing Allocation..'), LoggerLevel.INFO);
try {
this.totalToBeAllocated = await this.computeAllocation();
} catch (error) {
return err({
success: 0,
failed: 0,
message: `Unable to access fields on ScratchOrgInfo, Please check the profile being used`,
errorCode: PoolErrorCodes.PrerequisiteMissing,
});
}

if (this.totalToBeAllocated === 0) {
if (this.limits.ActiveScratchOrgs.Remaining > 0) {
return err({
success: 0,
failed: 0,
message: `The tag provided ${this.pool.tag} is currently at the maximum capacity , No scratch orgs will be allocated`,
errorCode: PoolErrorCodes.Max_Capacity,
});
} else {
try {
if (!this.pool.snapshotPool) {
SFPLogger.log(COLOR_KEY_MESSAGE('Computing Allocation..'), LoggerLevel.INFO);
try {
this.totalToBeAllocated = await this.computeAllocation();
} catch (error) {
return err({
success: 0,
failed: 0,
message: `There is no capacity to create a pool at this time, Please try again later`,
errorCode: PoolErrorCodes.No_Capacity,
message: `Unable to access fields on ScratchOrgInfo, Please check the profile being used`,
errorCode: PoolErrorCodes.PrerequisiteMissing,
});
}
}

//Generate Scratch Orgs
await this.generateScratchOrgs();
} else {
await this.fetchScratchOrgsFromSnapshotPool();
if (this.totalToBeAllocated === 0) {
if (this.limits.ActiveScratchOrgs.Remaining > 0) {
return err({
success: 0,
failed: 0,
message: `The tag provided ${this.pool.tag} is currently at the maximum capacity , No scratch orgs will be allocated`,
errorCode: PoolErrorCodes.Max_Capacity,
});
} else {
return err({
success: 0,
failed: 0,
message: `There is no capacity to create a pool at this time, Please try again later`,
errorCode: PoolErrorCodes.No_Capacity,
});
}
}

//Generate Scratch Orgs
this.pool.scratchOrgs = await this.generateScratchOrgs(
this.pool,
this.scratchOrgOperator,
this.scratchOrgInfoAssigner
);
} else {
this.pool.scratchOrgs = await this.fetchScratchOrgsFromSnapshotPool(
this.pool,
this.scratchOrgInfoFetcher,
this.scratchOrgInfoAssigner
);
}
} catch (error) {
return err({
success: 0,
failed: this.pool.failedToCreate,
message: `All requested scratch orgs failed to provision, Please check your code or config \n Failed with ${error.message}`,
errorCode: PoolErrorCodes.UnableToProvisionAny,
});
}

// Assign workers to executed scripts
for (let scratchOrg of this.pool.scratchOrgs) {
let result = this.scriptExecutorWrappedForBottleneck(scratchOrg, this.hubOrg.getUsername());
Expand All @@ -110,9 +127,13 @@ export default class PoolCreateImpl extends PoolBaseImpl {

await Promise.all(scriptExecPromises);

await this.finalizeGeneratedScratchOrgs();
this.pool = await this.finalizeGeneratedScratchOrgs(
this.pool,
this.scratchOrgOperator,
this.scratchOrgInfoFetcher
);

if (this.pool.scratchOrgs.length == 0) {
if (!this.pool.scratchOrgs || this.pool.scratchOrgs.length == 0) {
return err({
success: 0,
failed: this.pool.failedToCreate,
Expand Down Expand Up @@ -154,20 +175,24 @@ export default class PoolCreateImpl extends PoolBaseImpl {
return pool.to_allocate;
}

private async generateScratchOrgs() {
private async generateScratchOrgs(
pool: PoolConfig,
scratchOrgOperator: ScratchOrgOperator,
scratchOrgInfoAssigner: ScratchOrgInfoAssigner
) {
//Generate Scratch Orgs
SFPLogger.log(COLOR_KEY_MESSAGE('Generate Scratch Orgs..'), LoggerLevel.INFO);

let scratchOrgPromises = new Array<Promise<ScratchOrg>>();

const scratchOrgCreationLimiter = new Bottleneck({
maxConcurrent: this.pool.batchSize,
maxConcurrent: pool.batchSize,
});

let startTime = Date.now();
for (let i = 1; i <= this.pool.to_allocate; i++) {
for (let i = 1; i <= pool.to_allocate; i++) {
let scratchOrgPromise: Promise<ScratchOrg> = scratchOrgCreationLimiter.schedule(() =>
this.scratchOrgOperator.create(`SO` + i, this.pool.configFilePath, this.pool.expiry, this.pool.waitTime)
scratchOrgOperator.create(`SO` + i, this.pool.configFilePath, this.pool.expiry, this.pool.waitTime)
);
scratchOrgPromises.push(scratchOrgPromise);
}
Expand All @@ -179,51 +204,51 @@ export default class PoolCreateImpl extends PoolBaseImpl {
const isFulfilled = <T>(p: PromiseSettledResult<T>): p is PromiseFulfilledResult<T> => p.status === 'fulfilled';
const isRejected = <T>(p: PromiseSettledResult<T>): p is PromiseRejectedResult => p.status === 'rejected';

this.pool.scratchOrgs = scratchOrgCreationResults.filter(isFulfilled).map((p) => p.value);
let scratchOrgs = scratchOrgCreationResults.filter(isFulfilled).map((p) => p.value);
const rejectedScratchOrgs = scratchOrgCreationResults.filter(isRejected).map((p) => p.reason);
for (const reason of rejectedScratchOrgs) {
if (reason.message.includes(`The client has timed out`)) {
//Display how many we were able to create
let elapsedTime = Date.now() - startTime;
//Log how many we were able to create
let elapsedTime = Date.now() - startTime;
SFPLogger.log(
`A scratch org creation was rejected due to saleforce not responding within the set wait time of ${this.pool.waitTime} mins \n` +
`Time elasped so far ${COLOR_KEY_MESSAGE(getFormattedTime(elapsedTime))},You might need to inrease the wait time further and rety `
`A scratch org creation was rejected due to saleforce not responding within the set wait time of ${pool.waitTime} mins \n` +
`Time elasped so far ${COLOR_KEY_MESSAGE(
getFormattedTime(elapsedTime)
)},You might need to inrease the wait time further and rety `
);
} else SFPLogger.log(`A scratch org creation was rejected due to ${reason.message}`);
}

//Display how many we were able to create
//Log how many we were able to create
let elapsedTime = Date.now() - startTime;
SFPLogger.log(
`Created ${COLOR_SUCCESS(this.pool.scratchOrgs.length)} of ${
this.totalToBeAllocated
} successfully with ${COLOR_ERROR(rejectedScratchOrgs.length)} failures in ${COLOR_KEY_MESSAGE(
getFormattedTime(elapsedTime)
)}`
`Created ${COLOR_SUCCESS(scratchOrgs.length)} of ${pool.to_allocate} successfully with ${COLOR_ERROR(
rejectedScratchOrgs.length
)} failures in ${COLOR_KEY_MESSAGE(getFormattedTime(elapsedTime))}`
);

SFPStatsSender.logElapsedTime(`pool.scratchorg.creation.time`,elapsedTime,{pool:this.pool.tag})
//Splice scratchorgs that are having incorrect status of deleted , Why salesforce why??
let index = this.pool.scratchOrgs.length;
while (index--) {
try {
let orgDetails = await new OrgDetailsFetcher(this.pool.scratchOrgs[index].username).getOrgDetails();
if (orgDetails.status === 'Deleted') {
throw new Error(
`Throwing away scratch org ${this.pool.scratchOrgs[index].alias} as it has a status of deleted`
);
SFPStatsSender.logElapsedTime(`pool.scratchorg.creation.time`, elapsedTime, { pool: pool.tag });
if (scratchOrgs && scratchOrgs.length > 0) {
//Splice scratchorgs that are having incorrect status of deleted , Why salesforce why??
let index = scratchOrgs.length;
while (index--) {
try {
let orgDetails = await new OrgDetailsFetcher(scratchOrgs[index].username).getOrgDetails();
if (orgDetails.status === 'Deleted') {
throw new Error(
`Throwing away scratch org ${this.pool.scratchOrgs[index].alias} as it has a status of deleted`
);
}
} catch (error) {
scratchOrgs.splice(index, 1);
}
} catch (error) {
this.pool.scratchOrgs.splice(index, 1);
}
}

this.pool.scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(this.pool.scratchOrgs);
scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(scratchOrgs);

let scratchOrgInprogress = [];
let scratchOrgInprogress = [];

if (this.pool.scratchOrgs) {
this.pool.scratchOrgs.forEach((scratchOrg) => {
scratchOrgs.forEach((scratchOrg) => {
scratchOrgInprogress.push({
Id: scratchOrg.recordId,
Pooltag__c: this.pool.tag,
Expand All @@ -235,19 +260,24 @@ export default class PoolCreateImpl extends PoolBaseImpl {

if (scratchOrgInprogress.length > 0) {
//set pool tag
await this.scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress);
await scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress);
}
}
return scratchOrgs;
} else throw new Error(`No scratch orgs were sucesfully generated`);
}

private async fetchScratchOrgsFromSnapshotPool() {
private async fetchScratchOrgsFromSnapshotPool(
pool: PoolConfig,
scratchOrgInfoFetcher: ScratchOrgInfoFetcher,
scratchOrgInfoAssigner: ScratchOrgInfoAssigner
) {
//Generate Scratch Orgs
SFPLogger.log(
COLOR_KEY_MESSAGE(`Fetching Scratch Orgs from snapshot pool ${this.pool.snapshotPool}`),
LoggerLevel.INFO
);

this.pool.scratchOrgs = (await new PoolFetchImpl(
let scratchOrgs = (await new PoolFetchImpl(
this.hubOrg,
this.pool.snapshotPool,
false,
Expand All @@ -258,12 +288,12 @@ export default class PoolCreateImpl extends PoolBaseImpl {
true,
this.pool.maxAllocation
).execute()) as ScratchOrg[];
this.pool.scratchOrgs = await this.scratchOrgInfoFetcher.getScratchOrgRecordId(this.pool.scratchOrgs);
scratchOrgs = await scratchOrgInfoFetcher.getScratchOrgRecordId(scratchOrgs);

let scratchOrgInprogress = [];

if (this.pool.scratchOrgs) {
this.pool.scratchOrgs.forEach((scratchOrg) => {
if (scratchOrgs && scratchOrgs.length>0) {
scratchOrgs.forEach((scratchOrg) => {
scratchOrgInprogress.push({
Id: scratchOrg.recordId,
Pooltag__c: this.pool.tag,
Expand All @@ -275,15 +305,24 @@ export default class PoolCreateImpl extends PoolBaseImpl {

if (scratchOrgInprogress.length > 0) {
//set pool tag
await this.scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress);
await scratchOrgInfoAssigner.setScratchOrgInfo(scratchOrgInprogress);
}
return scratchOrgs;
}
else
{
throw new Error('No scratch orgs were found to be fetched');
}
}

private async finalizeGeneratedScratchOrgs() {
this.pool.failedToCreate = 0;
for (let i = this.pool.scratchOrgs.length - 1; i >= 0; i--) {
let scratchOrg = this.pool.scratchOrgs[i];
private async finalizeGeneratedScratchOrgs(
pool: PoolConfig,
scratchOrgOperator: ScratchOrgOperator,
scratchOrgInfoFetcher: ScratchOrgInfoFetcher
) {
pool.failedToCreate = 0;
for (let i = pool.scratchOrgs.length - 1; i >= 0; i--) {
let scratchOrg = pool.scratchOrgs[i];
if (scratchOrg.isScriptExecuted) {
continue;
}
Expand All @@ -296,11 +335,11 @@ export default class PoolCreateImpl extends PoolBaseImpl {
try {
//Delete scratchorgs that failed to execute script

let activeScratchOrgRecordId = await this.scratchOrgInfoFetcher.getActiveScratchOrgRecordIdGivenScratchOrg(
let activeScratchOrgRecordId = await scratchOrgInfoFetcher.getActiveScratchOrgRecordIdGivenScratchOrg(
scratchOrg.orgId
);

await this.scratchOrgOperator.delete([activeScratchOrgRecordId]);
await scratchOrgOperator.delete([activeScratchOrgRecordId]);
console.log(`Succesfully deleted scratchorg ${scratchOrg.username}`);
} catch (error) {
SFPLogger.log(
Expand All @@ -309,9 +348,10 @@ export default class PoolCreateImpl extends PoolBaseImpl {
);
}

this.pool.failedToCreate += 1;
this.pool.scratchOrgs.splice(i, 1);
pool.failedToCreate += 1;
pool.scratchOrgs.splice(i, 1);
}
return pool;
}

private async scriptExecutor(scratchOrg: ScratchOrg): Promise<ScratchOrg> {
Expand Down
4 changes: 2 additions & 2 deletions packages/sfpowerscripts-cli/src/impl/prepare/PrepareOrgJob.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ export default class PrepareOrgJob extends PoolJobExecutor implements PreDeployH
logger: FileLogger,
deploymentMode: DeploymentMode
) {
SFPLogger.log(`Deploying packages in the repo to ${scratchOrg.alias}`);
SFPLogger.log(`Deploying packages in the repo to ${scratchOrg.alias}`, LoggerLevel.INFO, logger);
SFPLogger.log(`Deploying packages to ${scratchOrg.alias}`);
SFPLogger.log(`Deploying packages to ${scratchOrg.alias}`, LoggerLevel.INFO, logger);

let deployProps: DeployProps = {
targetUsername: scratchOrg.username,
Expand Down

0 comments on commit d82af2b

Please sign in to comment.