MI-85: Multi package managers & improvements

.github/workflows/build.yml

@@ -9,4 +9,5 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.release.target_commitish }}
-      - run: docker build --build-arg NODE_TAG=20 .
+      - name: Build docker image
+        run: docker build --build-arg NODE_TAG=20 .

.gitignore

@@ -1 +1,3 @@
 node_modules
+dist
+.vscode/settings.json

Dockerfile

@@ -1,17 +1,28 @@
 ARG NODE_TAG
+FROM node:${NODE_TAG}-alpine AS builder
+
+# `WORKDIR` will create the folder if it doesn't exsist
+WORKDIR /build-stage
+COPY package*.json ./
+RUN npm ci
+COPY . ./
+RUN npm run build
+
+# This remove dev dependencies from `node_modules` folder
+RUN npm prune --production
+
 FROM node:${NODE_TAG}-alpine
 
-RUN mkdir /pipe
 WORKDIR /pipe
 
-RUN apk add wget 
-RUN wget -P / https://bitbucket.org/bitbucketpipelines/bitbucket-pipes-toolkit-bash/raw/0.4.0/common.sh
+# The `--force` flag force replace `yarn` if it exist in base image
+# This ensure we have the latest version of package managers
+RUN npm install -g --force npm pnpm yarn
 
-COPY tsconfig.json ./
-COPY pack*.json ./
-RUN npm ci
-COPY entrypoint.sh ./
-COPY pipe ./pipe
-RUN chmod a+x ./pipe/*.ts entrypoint.sh
+COPY --from=builder /build-stage/node_modules ./node_modules
+COPY --from=builder /build-stage/dist/ ./
+COPY --from=builder /build-stage/entrypoint.sh ./
+
+RUN chmod a+x entrypoint.sh
 
 ENTRYPOINT ["/pipe/entrypoint.sh"]

README.md

@@ -1,6 +1,9 @@
-# Aligent Nx Serverless Deploy Pipe
+# Aligent Serverless Deploy Pipe
 
-This pipe is used to deploy multiple Serverless Framework applications in an Nx monorepo.
+This pipe is used to deploy:
+
+- Multiple Serverless Framework applications in an Nx monorepo.
+- Single Serverless Framework application in a polyrepo.
 
 ## YAML Definition
 
@@ -30,28 +33,37 @@ Add the following your `bitbucket-pipelines.yml` file:
 | UPLOAD_BADGE          | Whether or not to upload a deployment badge to the repositories downloads section. (Accepted values: `true`/`false`)                           |
 | APP_USERNAME          | The user to upload the badge as. Required if UPLOAD_BADGE is set to `true`.                                                                    |
 | APP_PASSWORD          | The app password of the user uploading the badge. Required if UPLOAD_BADGE is set to `true`.                                                   |
-| TIMEZONE              | Which time zone the time in the badge should use (Default: 'Australia/Adelaide')                                                               |
+| TIMEZONE              | Which time zone the time in the badge should use (Default: `Australia/Adelaide`)                                                               |
+| PROFILE               | The profile name that is used for deployment (Default: `bitbucket-deployer`)                                                                   |
+| CMD                   | The command that this pipe will run (Eg: `deploy`, `remove`. Default: `deploy`)                                                                |
+| SERVICES_PATH         | The relative path from root folder to the folder where applications are defined (Default: `services`)                                          |
 
 - Default pipelines variables that are available for builds: https://support.atlassian.com/bitbucket-cloud/docs/variables-and-secrets/
 - Please check: https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/ for how to generate an app password.
 
 ## Monorepo structure
 
-The pipe expects each application to be under a root folder called `services`, and to have a `serverless.yml` file in its own root folder.
+The pipe expects:
+
+1. A single `nx.json` file at the root folder.
+2. Each application to be under a folder as defined via `SERVICES_PATH` variable above (default as `services`), and to have a `serverless.yml` and a `project.json` files in its own folder.
 
 ```
-services
-  - application-one
-    - serverless.yml
-    - project.json
-    ... other files
-  - application-two
-    - serverless.yml
-    - project.json
-    ... other files
+.
+├── nx.json
+├── services/
+│   ├── application-one/
+│   │   ├── serverless.yml
+│   │   ├── project.json
+│   │   └── ...other files
+│   └── application-two/
+│       ├── serverless.yml
+│       ├── project.json
+│       └── ...other files
+└── ...other files
 ```
 
-Each application should also have a `project.json` file defining an `nx` target called `deploy`, which implements serverless deploy.
+The `project.json` file defining an Nx target called `deploy`, which implements serverless deployment command:
 
 ```json
 {
@@ -70,12 +82,36 @@ Each application should also have a `project.json` file defining an `nx` target
 }
 ```
 
+## Polyrepo structure
+
+The pipe expects:
+
+1. No `nx.json` file at the root folder.
+2. A `serverless.yml` file at the root folder.
+
+```
+.
+├── serverless.yml
+├── src/
+│   └── ...other files
+└── ...other files
+```
+
 ## Development
 
-To build the image locally: \
-`docker build --build-arg="NODE_TAG=18-alpine" -t aligent/nx-pipe:18-alpine .`
+1. build the image locally:
+
+```bash
+# Transpile our source code to Javascript
+npm run build
+# [Optional] Run this if we want to remove devDependencies from node_modules before building docker image
+# If we run this, we will need to re-run `npm ci` later on if we fix bug & want to build another image.
+npm prune --production
+# Build docker image
+docker build --build-arg="NODE_TAG=20" -t aligent/nx-pipe:20-alpine .
+```
 
-To run the container locally and mount current local directory to the /app/work folder:
+2. Run the container locally and mount current local directory to the /app/work folder:
 
 ```bash
 docker run -it --memory=4g --memory-swap=4g --memory-swappiness=0 --cpus=4 --entrypoint=/bin/sh \
@@ -89,7 +125,8 @@ docker run -it --memory=4g --memory-swap=4g --memory-swappiness=0 --cpus=4 --ent
   -e UPLOAD_BADGE=false \
   -e APP_USERNAME=test-app-username \
   -e APP_PASSWORD=test-app-password \
-  aligent/nx-pipe:18-alpine
+  -e DEBUG=true \
+  aligent/nx-pipe:20-alpine
 ```
 
 ## See also

bin/index.ts

@@ -0,0 +1,92 @@
+import chalk from 'chalk';
+import logSymbols from 'log-symbols';
+import { runCLICommand } from '../lib/cmd';
+import { env } from '../lib/env';
+import { nodeModulesDirectoryExist } from '../lib/findNodeModules';
+import { findServerlessYaml } from '../lib/findServerlessYaml';
+import { injectCfnRole } from '../lib/injectCfnRole';
+import {
+    detectPackageManager,
+    getInstallCommand,
+} from '../lib/packageManagers';
+import { isNxServerlessMonorepo } from '../lib/serverlessProjectType';
+import { uploadDeploymentBadge } from '../lib/uploadDeploymentBadge';
+
+async function main() {
+    let deploymentStatus = false;
+
+    try {
+        const {
+            awsAccessKeyId,
+            awsSecretAccessKey,
+            bitbucketCloneDir,
+            cfnRole,
+            cmd,
+            debug,
+            profile,
+            servicesPath,
+            stage,
+        } = env;
+
+        if (!awsAccessKeyId || !awsSecretAccessKey) {
+            throw new Error(
+                'AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY not set'
+            );
+        }
+
+        const packageManager = detectPackageManager(bitbucketCloneDir);
+        const isMonorepo = await isNxServerlessMonorepo(bitbucketCloneDir);
+
+        const searchDirectory = isMonorepo
+            ? `${bitbucketCloneDir}/${servicesPath}`
+            : bitbucketCloneDir;
+
+        let serverlessFiles = await findServerlessYaml(searchDirectory);
+
+        await Promise.all(
+            serverlessFiles.map((file) => injectCfnRole(file, cfnRole))
+        );
+
+        const commands = [
+            `npx serverless config credentials --provider aws --profile ${profile} --key ${awsAccessKeyId} --secret ${awsSecretAccessKey}`,
+        ];
+
+        const isNodeModulesExists = await nodeModulesDirectoryExist(
+            bitbucketCloneDir
+        );
+        if (!isNodeModulesExists) {
+            const installCmd = getInstallCommand(packageManager);
+            commands.unshift(installCmd);
+        }
+
+        const baseCommand = isMonorepo
+            ? `npx nx run-many -t ${cmd} --`
+            : `npx serverless ${cmd}`;
+        const verbose = debug ? '--verbose' : '';
+
+        commands.push(
+            `${baseCommand} --stage ${stage} --aws-profile ${profile} ${verbose}`
+        );
+
+        await runCLICommand(commands, bitbucketCloneDir);
+
+        deploymentStatus = true;
+    } catch (error) {
+        if (error instanceof Error) {
+            console.error(logSymbols.error, chalk.redBright(error.message));
+        }
+        console.error(
+            logSymbols.error,
+            chalk.redBright(
+                'Deployment failed! Please check the logs for more details.'
+            )
+        );
+        deploymentStatus = false;
+    } finally {
+        const statusCode = await uploadDeploymentBadge(deploymentStatus);
+        process.exit(statusCode);
+    }
+}
+
+// Execute the main function
+main();

entrypoint.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
 #set -x
 
-cd /pipe
-npx ts-node pipe/entrypoint.ts
+node /pipe/bin/index.js

pipe/cmd.ts → lib/cmd.ts

@@ -1,28 +1,41 @@
-import * as cp from 'child_process';
+import chalk from 'chalk';
+import { SpawnOptions, spawn } from 'child_process';
+import logSymbols from 'log-symbols';
 import { env } from './env';
 
+interface Command {
+    command: string;
+    args: ReadonlyArray<string>;
+}
+
+function splitCommandAndArgs(command: string): Command {
+    // Split the command string at all white spaces excluding white spaces wrapped with single quotes
+    const cmd = command.split(/\s(?=(?:[^']*'[^']*')*[^']*$)/g);
+    return {
+        command: cmd.shift() as string,
+        args: cmd,
+    };
+}
+
 // Wrap spawn in a promise
 function asyncSpawn(
     command: string,
-    args?: ReadonlyArray<string>,
-    options?: cp.SpawnOptionsWithoutStdio
+    args: ReadonlyArray<string>,
+    options: SpawnOptions
 ): Promise<number | null> {
     return new Promise(function (resolve, reject) {
-        const process = cp.spawn(command, args, options);
-        if (env.debug)
+        if (env.debug) {
+            const commandWithArgs = `${command} ${args?.join(' ')}`;
+            const optionsStr = JSON.stringify(options);
             console.log(
-                `ℹ️ Executing command: ${command} ${args?.join(
-                    ' '
-                )} with options: ${JSON.stringify(options)}`
+                logSymbols.info,
+                chalk.whiteBright(
+                    `Executing command: ${commandWithArgs} with options: ${optionsStr}`
+                )
             );
+        }
 
-        process.stdout.on('data', (data) => {
-            console.log(data.toString());
-        });
-
-        process.stderr.on('data', (data) => {
-            console.log(`Error: ${data.toString()}`);
-        });
+        const process = spawn(command, args, options);
 
         process.on('exit', function (code) {
             if (code !== 0) reject(code);
@@ -34,32 +47,27 @@ function asyncSpawn(
         });
     });
 }
-interface Command {
-    command: string;
-    args: ReadonlyArray<string>;
-}
-
-function splitCommandAndArgs(command: string): Command {
-    // Split the command string at all white spaces excluding white spaces wrapped with single quotes
-    const cmd = command.split(/\s(?=(?:[^']*'[^']*')*[^']*$)/g);
-    return {
-        command: cmd.shift() as string,
-        args: cmd,
-    };
-}
 
 function runCommandString(
     command: string,
-    workDir?: string
+    workDir: string
 ): Promise<number | null> {
-    console.log(`Running command: ${command}`);
+    console.log(
+        logSymbols.info,
+        chalk.whiteBright(`Running command: ${command}`)
+    );
     const cmd = splitCommandAndArgs(command);
-    return asyncSpawn(cmd.command, cmd.args, { cwd: workDir });
+    return asyncSpawn(cmd.command, cmd.args, {
+        cwd: workDir,
+        stdio: ['pipe', 'inherit', 'inherit'],
+    });
 }
 
-export async function runCLICommand(commandStr: Array<string>) {
-    const workDir = process.env.BITBUCKET_CLONE_DIR;
-    console.log(`Running commands in ${workDir}`);
+export async function runCLICommand(
+    commandStr: Array<string>,
+    workDir: string
+) {
+    console.log(logSymbols.info, chalk.white(`Running commands in ${workDir}`));
 
     for (const cmd of commandStr) {
         await runCommandString(cmd, workDir);

pipe/env.ts → lib/env.ts

@@ -11,16 +11,17 @@ interface Env {
     appPassword?: string;
     timezone: string;
     bitbucketBranch?: string;
+    bitbucketCloneDir: string;
     bitbucketRepoSlug?: string;
     bitbucketWorkspace?: string;
-    servicesPath?: string;
+    servicesPath: string;
 }
 
 export const env: Env = {
     debug: process.env.DEBUG === 'true',
     stage: process.env.STAGE || 'stg',
     profile: process.env.PROFILE || 'bitbucket-deployer',
-    cmd: process.env.cmd || 'deploy',
+    cmd: process.env.CMD || 'deploy',
     awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
     awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
     cfnRole: process.env.CFN_ROLE,
@@ -29,7 +30,8 @@ export const env: Env = {
     appPassword: process.env.APP_PASSWORD,
     timezone: process.env.TIMEZONE || 'Australia/Adelaide',
     bitbucketBranch: process.env.BITBUCKET_BRANCH,
+    bitbucketCloneDir: process.env.BITBUCKET_CLONE_DIR || '',
     bitbucketRepoSlug: process.env.BITBUCKET_REPO_SLUG,
     bitbucketWorkspace: process.env.BITBUCKET_WORKSPACE,
-    servicesPath: process.env.servicesPath || '/services',
+    servicesPath: process.env.SERVICES_PATH || 'services',
 };