fix(mw): added in memory caching and reuse of etl tokens

jobstash · Nov 5, 2024 · f5204e1 · f5204e1
1 parent ebdb0d0
commit f5204e1
Show file tree

Hide file tree

Showing 10 changed files with 117 additions and 74 deletions.
diff --git a/src/app.module.ts b/src/app.module.ts
@@ -28,6 +28,7 @@ import { GoogleBigQueryModule } from "./google-bigquery/google-bigquery.module";
 import { SearchModule } from "./search/search.module";
 import { PaymentsModule } from "./payments/payments.module";
 import { UserModule } from "./user/user.module";
+import { Auth0Module } from './auth0/auth0.module';
 
 @Module({
   imports: [
@@ -73,6 +74,7 @@ import { UserModule } from "./user/user.module";
     GoogleBigQueryModule,
     SearchModule,
     PaymentsModule,
+    Auth0Module,
   ],
   controllers: [AppController],
   providers: [AppService],

diff --git a/src/auth/profile/profile.module.ts b/src/auth/profile/profile.module.ts
@@ -19,9 +19,11 @@ import { PrivyModule } from "../privy/privy.module";
 import { GithubModule } from "../github/github.module";
 import { UserModule } from "src/user/user.module";
 import { PaymentsModule } from "src/payments/payments.module";
+import { Auth0Module } from "src/auth0/auth0.module";
 
 @Module({
   imports: [
+    Auth0Module,
     forwardRef(() => AuthModule),
     forwardRef(() => UserModule),
     forwardRef(() => PrivyModule),

diff --git a/src/auth0/auth0.module.ts b/src/auth0/auth0.module.ts
@@ -0,0 +1,10 @@
+import { Module } from "@nestjs/common";
+import { Auth0Service } from "./auth0.service";
+import { CacheModule } from "@nestjs/cache-manager";
+
+@Module({
+  imports: [CacheModule.register()],
+  providers: [Auth0Service],
+  exports: [Auth0Service],
+})
+export class Auth0Module {}
diff --git a/src/auth0/auth0.service.ts b/src/auth0/auth0.service.ts
@@ -0,0 +1,55 @@
+import { CACHE_MANAGER } from "@nestjs/cache-manager";
+import { Inject, Injectable } from "@nestjs/common";
+import { ConfigService } from "@nestjs/config";
+import axios from "axios";
+import { Cache } from "cache-manager";
+import * as Sentry from "@sentry/node";
+import { CustomLogger } from "src/shared/utils/custom-logger";
+
+@Injectable()
+export class Auth0Service {
+  private readonly logger = new CustomLogger(Auth0Service.name);
+  constructor(
+    private readonly configService: ConfigService,
+    @Inject(CACHE_MANAGER) private cacheManager: Cache,
+  ) {}
+
+  async getETLToken(): Promise<string | undefined> {
+    try {
+      const existingToken = await this.cacheManager.get<string>("etl-token");
+      if (existingToken) {
+        return existingToken;
+      } else {
+        const clientId = this.configService.get<string>("ETL_CLIENT_ID");
+        const clientSecret =
+          this.configService.get<string>("ETL_CLIENT_SECRET");
+
+        const auth0Domain = this.configService.get<string>("AUTH0_DOMAIN");
+        const audience = this.configService.get<string>("AUTH0_AUDIENCE");
+        const response = await axios.post(`${auth0Domain}/oauth/token`, {
+          client_id: clientId,
+          client_secret: clientSecret,
+          audience,
+          grant_type: "client_credentials",
+        });
+        if (response.data) {
+          const authToken = response.data.access_token;
+          await this.cacheManager.set("etl-token", authToken, 43_200_000);
+          return authToken;
+        } else {
+          throw new Error("Error fetching auth token");
+        }
+      }
+    } catch (err) {
+      Sentry.withScope(scope => {
+        scope.setTags({
+          action: "external-api-call",
+          source: "projects.service",
+        });
+        Sentry.captureException(err);
+      });
+      this.logger.error(`Auth0Service::getETLToken ${err.message}`);
+      return undefined;
+    }
+  }
+}
diff --git a/src/jobs/jobs.module.ts b/src/jobs/jobs.module.ts
@@ -15,9 +15,11 @@ import * as https from "https";
 import { ProfileModule } from "src/auth/profile/profile.module";
 import { PaymentsModule } from "src/payments/payments.module";
 import { PrivyModule } from "src/auth/privy/privy.module";
+import { Auth0Module } from "src/auth0/auth0.module";
 
 @Module({
   imports: [
+    Auth0Module,
     forwardRef(() => UserModule),
     forwardRef(() => ProfileModule),
     forwardRef(() => PrivyModule),

diff --git a/src/organizations/organizations.module.ts b/src/organizations/organizations.module.ts
@@ -3,9 +3,10 @@ import { OrganizationsService } from "./organizations.service";
 import { OrganizationsController } from "./organizations.controller";
 import { ModelService } from "src/model/model.service";
 import { AuthModule } from "src/auth/auth.module";
+import { Auth0Module } from "src/auth0/auth0.module";
 
 @Module({
-  imports: [AuthModule],
+  imports: [AuthModule, Auth0Module],
   controllers: [OrganizationsController],
   providers: [OrganizationsService, ModelService],
 })

diff --git a/src/organizations/organizations.service.ts b/src/organizations/organizations.service.ts
@@ -54,6 +54,7 @@ import { UpdateOrgProjectInput } from "./dto/update-organization-projects.input"
 import { AddOrganizationByUrlInput } from "./dto/add-organization-by-url.input";
 import { ConfigService } from "@nestjs/config";
 import axios from "axios";
+import { Auth0Service } from "src/auth0/auth0.service";
 
 @Injectable()
 export class OrganizationsService {
@@ -63,6 +64,7 @@ export class OrganizationsService {
     private neogma: Neogma,
     private models: ModelService,
     private configService: ConfigService,
+    private readonly auth0Service: Auth0Service,
   ) {}
 
   getOrgListResults = async (): Promise<OrgDetailsResult[]> => {
@@ -1104,46 +1106,28 @@ export class OrganizationsService {
     dto: AddOrganizationByUrlInput,
   ): Promise<ResponseWithNoData> {
     try {
-      const clientId = this.configService.get<string>("ETL_CLIENT_ID");
-      const clientSecret = this.configService.get<string>("ETL_CLIENT_SECRET");
       const url = this.configService.get<string>("ETL_DOMAIN");
-
-      const auth0Domain = this.configService.get<string>("AUTH0_DOMAIN");
-      const audience = this.configService.get<string>("AUTH0_AUDIENCE");
-      const response = await axios.post(`${auth0Domain}/oauth/token`, {
-        client_id: clientId,
-        client_secret: clientSecret,
-        audience,
-        grant_type: "client_credentials",
-      });
-      if (response.data) {
-        const authToken = response.data.access_token;
-        const response2 = await axios.get(
-          `${url}/organization-importer/import-organization-by-url?url=${dto.url}&name=${dto.name}`,
-          {
-            headers: {
-              Authorization: `Bearer ${authToken}`,
-            },
+      const authToken = await this.auth0Service.getETLToken();
+      const response2 = await axios.get(
+        `${url}/organization-importer/import-organization-by-url?url=${dto.url}&name=${dto.name}`,
+        {
+          headers: {
+            Authorization: authToken ? `Bearer ${authToken}` : undefined,
           },
-        );
-        if ([200, 201, 202].includes(response2.status)) {
-          return {
-            success: true,
-            message: "Organization queued for import successfully",
-          };
-        } else {
-          this.logger.warn(
-            `Error queueing organization ${dto} for import: ${response2.data}`,
-          );
-          return {
-            success: false,
-            message: "Error adding organization",
-          };
-        }
+        },
+      );
+      if ([200, 201, 202].includes(response2.status)) {
+        return {
+          success: true,
+          message: "Organization queued for import successfully",
+        };
       } else {
+        this.logger.warn(
+          `Error queueing organization ${dto} for import: ${response2.data}`,
+        );
         return {
           success: false,
-          message: "Error fetching auth token",
+          message: "Error adding organization",
         };
       }
     } catch (err) {

diff --git a/src/projects/projects.module.ts b/src/projects/projects.module.ts
@@ -5,9 +5,10 @@ import { ModelService } from "src/model/model.service";
 import { ProjectCategoryService } from "./project-category.service";
 import { OrganizationsService } from "src/organizations/organizations.service";
 import { AuthModule } from "src/auth/auth.module";
+import { Auth0Module } from "src/auth0/auth0.module";
 
 @Module({
-  imports: [AuthModule],
+  imports: [AuthModule, Auth0Module],
   controllers: [ProjectsController],
   providers: [
     ProjectsService,

diff --git a/src/projects/projects.service.ts b/src/projects/projects.service.ts
@@ -47,6 +47,7 @@ import { ConfigService } from "@nestjs/config";
 import { omit } from "lodash";
 import { ActivateProjectJobsiteInput } from "./dto/activate-project-jobsites.input";
 import { UpdateProjectJobsitesInput } from "./dto/update-project-jobsites.input";
+import { Auth0Service } from "src/auth0/auth0.service";
 
 @Injectable()
 export class ProjectsService {
@@ -56,6 +57,7 @@ export class ProjectsService {
     private neogma: Neogma,
     private models: ModelService,
     private readonly configService: ConfigService,
+    private readonly auth0Service: Auth0Service,
   ) {}
 
   async getProjectsListWithSearch(
@@ -771,48 +773,30 @@ export class ProjectsService {
     dto: AddProjectByUrlInput,
   ): Promise<ResponseWithNoData> {
     try {
-      const clientId = this.configService.get<string>("ETL_CLIENT_ID");
-      const clientSecret = this.configService.get<string>("ETL_CLIENT_SECRET");
       const url = this.configService.get<string>("ETL_DOMAIN");
-
-      const auth0Domain = this.configService.get<string>("AUTH0_DOMAIN");
-      const audience = this.configService.get<string>("AUTH0_AUDIENCE");
-      const response = await axios.post(`${auth0Domain}/oauth/token`, {
-        client_id: clientId,
-        client_secret: clientSecret,
-        audience,
-        grant_type: "client_credentials",
-      });
-      if (response.data) {
-        const authToken = response.data.access_token;
-        const response2 = await axios.get(
-          `${url}/project-importer/import-project-by-url?url=${dto.url}&name=${
-            dto.name
-          }&orgId=${dto.orgId ?? ""}&defiLlamaSlug=${dto.defiLlamaSlug ?? ""}`,
-          {
-            headers: {
-              Authorization: `Bearer ${authToken}`,
-            },
+      const authToken = await this.auth0Service.getETLToken();
+      const response2 = await axios.get(
+        `${url}/project-importer/import-project-by-url?url=${dto.url}&name=${
+          dto.name
+        }&orgId=${dto.orgId ?? ""}&defiLlamaSlug=${dto.defiLlamaSlug ?? ""}`,
+        {
+          headers: {
+            Authorization: authToken ? `Bearer ${authToken}` : undefined,
           },
-        );
-        if ([200, 201, 202].includes(response2.status)) {
-          return {
-            success: true,
-            message: "Project queued for import successfully",
-          };
-        } else {
-          this.logger.warn(
-            `Error queueing project ${dto} for import: ${response2.data}`,
-          );
-          return {
-            success: false,
-            message: "Error adding project",
-          };
-        }
+        },
+      );
+      if ([200, 201, 202].includes(response2.status)) {
+        return {
+          success: true,
+          message: "Project queued for import successfully",
+        };
       } else {
+        this.logger.warn(
+          `Error queueing project ${dto} for import: ${response2.data}`,
+        );
         return {
           success: false,
-          message: "Error fetching auth token",
+          message: "Error adding project",
         };
       }
     } catch (err) {

diff --git a/src/user/user.module.ts b/src/user/user.module.ts
@@ -21,9 +21,11 @@ import * as https from "https";
 import { PaymentsModule } from "src/payments/payments.module";
 import { PermissionService } from "./permission.service";
 import { UserController } from "./user.controller";
+import { Auth0Module } from "src/auth0/auth0.module";
 
 @Module({
   imports: [
+    Auth0Module,
     forwardRef(() => GithubModule),
     forwardRef(() => PrivyModule),
     forwardRef(() => ProfileModule),