[TM-1519] POC Airtable Sync

.env.local.sample

@@ -1,11 +1,18 @@
 USER_SERVICE_PORT=4010
 JOB_SERVICE_PORT=4020
 RESEARCH_SERVICE_PORT=4030
+UNIFIED_DATABASE_SERVICE_PORT=4040
 
 DB_HOST=localhost
 DB_PORT=3360
 DB_DATABASE=wri_restoration_marketplace_api
 DB_USERNAME=wri
 DB_PASSWORD=wri
 
+REDIS_HOST=localhost
+REDIS_PORT=6379
+
 JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt
+
+# Only needed for the unified database service. Most developers should not have this defined.
+AIRTABLE_API_KEY

.github/workflows/deploy-api-gateway.yml

@@ -24,6 +24,7 @@ env:
   AWS_ROLE_TO_ASSUME: arn:aws:iam::603634817705:role/terramatch-microservices-github-actions
   AWS_ROLE_SESSION_NAME: terramatch-microservices-cicd-api-gateway
   PHP_PROXY_TARGET: ${{ vars.PHP_PROXY_TARGET }}
+  ENABLED_SERVICES: ${{ vars.ENABLED_SERVICES }}
 
 jobs:
   deploy-api-gateway:

.github/workflows/deploy-service.yml

@@ -20,6 +20,7 @@ on:
           - job-service
           - research-service
           - user-service
+          - unified-database-service
       env:
         description: 'Deployment target environment'
         type: choice

.github/workflows/deploy-services.yml

@@ -19,25 +19,55 @@ permissions:
   contents: read
 
 jobs:
+  check-services:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.env }}
+    outputs:
+      job-service-enabled: ${{ steps.check-services.outputs.job }}
+      user-service-enabled: ${{ steps.check-services.outputs.user }}
+      research-service-enabled: ${{ steps.check-services.outputs.research }}
+      unified-database-service-enabled: ${{ steps.check-services.outputs.unified-database }}
+    steps:
+      - id: check-services
+        run: |
+          echo "job=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'job-service') }}" >> "$GITHUB_OUTPUT"
+          echo "user=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'user-service') }}" >> "$GITHUB_OUTPUT"
+          echo "research=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'research-service') }}" >> "$GITHUB_OUTPUT"
+          echo "unified-database=${{ vars.ENABLED_SERVICES == '' || contains(vars.ENABLED_SERVICES, 'unified-database-service') }}" >> "$GITHUB_OUTPUT"
+
   job-service:
+    needs: check-services
+    if: needs.check-services.outputs.job-service-enabled == 'true'
     uses: ./.github/workflows/deploy-service.yml
     with:
       env: ${{ inputs.env }}
       service: job-service
     secrets: inherit
 
+
   user-service:
+    needs: check-services
+    if: needs.check-services.outputs.user-service-enabled == 'true'
     uses: ./.github/workflows/deploy-service.yml
     with:
       env: ${{ inputs.env }}
       service: user-service
     secrets: inherit
 
   research-service:
+    needs: check-services
+    if: needs.check-services.outputs.research-service-enabled == 'true'
     uses: ./.github/workflows/deploy-service.yml
     with:
       env: ${{ inputs.env }}
       service: research-service
     secrets: inherit
 
-
+  unified-database-service:
+    needs: check-services
+    if: needs.check-services.outputs.unified-database-service-enabled == 'true'
+    uses: ./.github/workflows/deploy-service.yml
+    with:
+      env: ${{ inputs.env }}
+      service: unified-database-service
+    secrets: inherit

README.md

@@ -41,10 +41,12 @@ and main branches.
  * In your `.env` and `.env.local.sample`, add `_PORT` for the new service
  * In `api-gateway-stack.ts`, add the new service and namespace to `V3_SERVICES`
  * In your local web repo, follow directions in `README.md` for setting up a new service.
+   * This step can be skipped for services that will not be used by the FE website.
  * For deployment to AWS:
    * Add a Dockerfile in the new app directory. A simple copy and modify from user-service is sufficient
    * Add the new service name to the "service" workflow input options in `deploy-service.yml`
    * Add a new job to `deploy-services.yml` to include the new services in the "all" service deployment workflow.
+     * Make sure to update the `check-services` step and follow the pattern for the `if` conditions on the individual service deploy jobs.
    * In AWS:
      * Add ECR repositories for each env (follow the naming scheme from user-service, e.g. `terramatch-microservices/foo-service-staging`, etc)
        * Set the repo to Immutable

apps/unified-database-service-e2e/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

apps/unified-database-service-e2e/jest.config.ts

@@ -0,0 +1,19 @@
+/* eslint-disable */
+export default {
+  displayName: "unified-database-service-e2e",
+  preset: "../../jest.preset.js",
+  globalSetup: "<rootDir>/src/support/global-setup.ts",
+  globalTeardown: "<rootDir>/src/support/global-teardown.ts",
+  setupFiles: ["<rootDir>/src/support/test-setup.ts"],
+  testEnvironment: "node",
+  transform: {
+    "^.+\\.[tj]s$": [
+      "ts-jest",
+      {
+        tsconfig: "<rootDir>/tsconfig.spec.json"
+      }
+    ]
+  },
+  moduleFileExtensions: ["ts", "js", "html"],
+  coverageDirectory: "../../coverage/unified-database-service-e2e"
+};

apps/unified-database-service-e2e/project.json

@@ -0,0 +1,17 @@
+{
+  "name": "unified-database-service-e2e",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "application",
+  "implicitDependencies": ["unified-database-service"],
+  "targets": {
+    "e2e": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{e2eProjectRoot}"],
+      "options": {
+        "jestConfig": "apps/unified-database-service-e2e/jest.config.ts",
+        "passWithNoTests": true
+      },
+      "dependsOn": ["unified-database-service:build"]
+    }
+  }
+}

apps/unified-database-service-e2e/src/support/global-setup.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+var __TEARDOWN_MESSAGE__: string;
+
+module.exports = async function () {
+  // Start services that that the app needs to run (e.g. database, docker-compose, etc.).
+  console.log("\nSetting up...\n");
+
+  // Hint: Use `globalThis` to pass variables to global teardown.
+  globalThis.__TEARDOWN_MESSAGE__ = "\nTearing down...\n";
+};

apps/unified-database-service-e2e/src/support/global-teardown.ts

@@ -0,0 +1,7 @@
+/* eslint-disable */
+
+module.exports = async function () {
+  // Put clean up logic here (e.g. stopping services, docker-compose, etc.).
+  // Hint: `globalThis` is shared between setup and teardown.
+  console.log(globalThis.__TEARDOWN_MESSAGE__);
+};

apps/unified-database-service-e2e/src/support/test-setup.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+
+import axios from "axios";
+
+module.exports = async function () {
+  // Configure axios for tests to use.
+  const host = process.env.HOST ?? "localhost";
+  const port = process.env.PORT ?? "3000";
+  axios.defaults.baseURL = `http://${host}:${port}`;
+};

apps/unified-database-service-e2e/src/unified-database-service/unified-database-service.spec.ts

@@ -0,0 +1,10 @@
+import axios from "axios";
+
+describe("GET /api", () => {
+  it("should return a message", async () => {
+    const res = await axios.get(`/api`);
+
+    expect(res.status).toBe(200);
+    expect(res.data).toEqual({ message: "Hello API" });
+  });
+});

apps/unified-database-service-e2e/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ],
+  "compilerOptions": {
+    "esModuleInterop": true
+  }
+}

apps/unified-database-service-e2e/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["jest.config.ts", "src/**/*.ts"]
+}

apps/unified-database-service/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

apps/unified-database-service/Dockerfile

@@ -0,0 +1,15 @@
+FROM terramatch-microservices-base:nx-base AS builder
+
+ARG BUILD_FLAG
+WORKDIR /app/builder
+COPY . .
+RUN npx nx build unified-database-service ${BUILD_FLAG}
+
+FROM terramatch-microservices-base:nx-base
+
+ARG NODE_ENV
+WORKDIR /app
+COPY --from=builder /app/builder ./
+ENV NODE_ENV=${NODE_ENV}
+
+CMD ["node", "./dist/apps/unified-database-service/main.js"]

apps/unified-database-service/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: "unified-database-service",
+  preset: "../../jest.preset.js",
+  testEnvironment: "node",
+  transform: {
+    "^.+\\.[tj]s$": ["ts-jest", { tsconfig: "<rootDir>/tsconfig.spec.json" }]
+  },
+  moduleFileExtensions: ["ts", "js", "html"],
+  coverageDirectory: "../../coverage/apps/unified-database-service"
+};

apps/unified-database-service/project.json

@@ -0,0 +1,26 @@
+{
+  "name": "unified-database-service",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/unified-database-service/src",
+  "projectType": "application",
+  "tags": [],
+  "targets": {
+    "serve": {
+      "executor": "@nx/js:node",
+      "defaultConfiguration": "development",
+      "dependsOn": ["build"],
+      "options": {
+        "buildTarget": "unified-database-service:build",
+        "runBuildTargetDependencies": false
+      },
+      "configurations": {
+        "development": {
+          "buildTarget": "unified-database-service:build:development"
+        },
+        "production": {
+          "buildTarget": "unified-database-service:build:production"
+        }
+      }
+    }
+  }
+}

apps/unified-database-service/src/airtable/airtable.module.ts

@@ -0,0 +1,30 @@
+import { DatabaseModule } from "@terramatch-microservices/database";
+import { CommonModule } from "@terramatch-microservices/common";
+import { ConfigModule, ConfigService } from "@nestjs/config";
+import { BullModule } from "@nestjs/bullmq";
+import { Module } from "@nestjs/common";
+import { AirtableService } from "./airtable.service";
+import { AirtableProcessor } from "./airtable.processor";
+
+@Module({
+  imports: [
+    DatabaseModule,
+    CommonModule,
+    ConfigModule.forRoot(),
+    BullModule.forRootAsync({
+      imports: [ConfigModule],
+      inject: [ConfigService],
+      useFactory: (configService: ConfigService) => ({
+        connection: {
+          host: configService.get("REDIS_HOST"),
+          port: configService.get("REDIS_PORT"),
+          prefix: "unified-database-service"
+        }
+      })
+    }),
+    BullModule.registerQueue({ name: "airtable" })
+  ],
+  providers: [AirtableService, AirtableProcessor],
+  exports: [AirtableService]
+})
+export class AirtableModule {}