diff --git a/package.json b/package.json index a960560..c3103aa 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "argon2": "^0.31.1", "axios": "^1.4.0", "cache-manager": "^4.1.0", + "cache-manager-redis-store": "^2.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "codecov": "^3.8.3", @@ -69,6 +70,7 @@ "posthog-node": "^2.6.0", "prom-client": "^14.2.0", "prompt-engine": "^0.0.31", + "redis": "^4.6.7", "reflect-metadata": "^0.1.13", "rxjs": "^7.2.0", "winston": "^3.8.2", diff --git a/prisma/migrations/20241129023712_removed_telemetry_logs/migration.sql b/prisma/migrations/20241129023712_removed_telemetry_logs/migration.sql new file mode 100644 index 0000000..9a1336f --- /dev/null +++ b/prisma/migrations/20241129023712_removed_telemetry_logs/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the `telemetry_logs` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropTable +DROP TABLE "telemetry_logs"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7835874..b8d3f01 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -52,40 +52,6 @@ model Metrics { value String } -model telemetry_logs { - eid String @db.Uuid @default(uuid()) - createdAt DateTime @default(now()) - eventType String - eventName String - producer Json - platform String - // producerChannel String - // producerDeviceID String? - // producerProducerID String @db.Uuid - // producerPlatform String - context Json - sessionId String @db.Uuid - // contextUserID String @db.Uuid - // contextConversationID String? @db.Uuid - // contextPageID String? - // contextRollup Json? - eventData Json - // eventDataTimestamp DateTime @default(now()) - // eventDataDuration String? - // eventDataAudioURL String? - // eventDataQuestionGenerated String? - // eventDataQuestionSubmitted String? - // eventDataComparisonScore Int? - // eventDataAnswer String? - // eventDataFeedbackScore Int @default(0) @db.SmallInt - // eventDataLogData Json? - // eventDataErrorData Json? - errorType String? - tags Json? - - @@unique([eid,createdAt]) -} - model feedback { id Int @id @default(autoincrement()) conversation conversation? @relation(fields: [conversationId], references: [id]) diff --git a/src/app.controller.ts b/src/app.controller.ts index 365ed0d..5fe01a6 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -21,11 +21,12 @@ import { } from "./common/utils"; import { ConversationService } from "./modules/conversation/conversation.service"; import { PrismaService } from "./global-services/prisma.service"; -import { CustomLogger } from "./common/logger"; import { MonitoringService } from "./modules/monitoring/monitoring.service"; import { PromptServices } from "./xstate/prompt/prompt.service"; -import { TelemetryService } from "./modules/telemetry/telemetry.service"; import { Cache } from "cache-manager"; +import { LokiLogger } from "./modules/loki-logger/loki-logger.service"; +import { HttpService } from '@nestjs/axios'; +import { ApiTags, ApiOperation, ApiResponse, ApiParam, ApiBody, ApiHeader } from '@nestjs/swagger'; const uuid = require("uuid"); const path = require("path"); const filePath = path.resolve(__dirname, "./common/en.json"); @@ -65,6 +66,7 @@ export class PromptDto { identifier?: string; } +@ApiTags('App') @Controller() export class AppController { private configService: ConfigService; @@ -72,12 +74,12 @@ export class AppController { private conversationService: ConversationService; private prismaService: PrismaService; private promptService: PromptServices; - private logger: CustomLogger; + private logger: LokiLogger constructor( private readonly appService: AppService, private readonly monitoringService: MonitoringService, - private readonly telemetryService: TelemetryService, + private readonly httpService: HttpService, @Inject(CACHE_MANAGER) private readonly cacheManager: Cache ) { this.prismaService = new PrismaService(); @@ -85,6 +87,7 @@ export class AppController { this.aiToolsService = new AiToolsService( this.configService, this.monitoringService, + this.httpService, this.cacheManager ); this.conversationService = new ConversationService( @@ -95,16 +98,30 @@ export class AppController { this.prismaService, this.configService, this.aiToolsService, - this.monitoringService + this.monitoringService, + this.httpService + ); + this.logger = new LokiLogger( + AppController.name, + httpService, + this.configService, ); - this.logger = new CustomLogger("AppController"); } + @ApiOperation({ summary: 'Get hello message' }) + @ApiResponse({ status: 200, description: 'Returns hello message' }) @Get("/") getHello(): string { return this.appService.getHello(); } + @ApiOperation({ summary: 'Process user prompt' }) + @ApiParam({ name: 'configid', description: 'Configuration ID' }) + @ApiBody({ type: PromptDto }) + @ApiHeader({ name: 'user-id', description: 'User ID' }) + @ApiHeader({ name: 'session-id', description: 'Session ID' }) + @ApiResponse({ status: 200, description: 'Returns processed prompt response' }) + @ApiResponse({ status: 400, description: 'Missing required headers' }) @Post("/prompt/:configid") async prompt( @Body() promptDto: any, @@ -115,8 +132,8 @@ export class AppController { //get userId from headers const userId = headers["user-id"]; const sessionId = headers["session-id"]; - console.log("userId =", userId); - console.log("sessionId =", sessionId); + this.logger.log("userId =", userId); + this.logger.log("sessionId =", sessionId); if (!userId) { return { text: "", @@ -156,37 +173,6 @@ export class AppController { }); } catch { this.monitoringService.incrementTotalSessionsCount(); - // verboseLogger("creating new user with id =",userId) - await this.telemetryService.capture({ - eventName: "Conversation start", - eventType: "START_CONVERSATION", - producer: { - channel: "Bot", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - startTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: promptDto.text, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "conversation_start"], - }); } if (!user) { user = await this.prismaService.user.create({ @@ -234,7 +220,7 @@ export class AppController { configid ) - // console.log("fetched conversation: ", conversation) + // this.logger.log("fetched conversation: ", conversation) //handle text and audio if (promptDto.text) { type = "Text"; @@ -242,83 +228,19 @@ export class AppController { if (/^\d+$/.test(userInput)) { prompt.inputLanguage = Language.en; } else { - // console.log("IN ELSE....") + // this.logger.log("IN ELSE....") try { let response = await this.aiToolsService.detectLanguage(userInput, userId, sessionId) prompt.inputLanguage = response["language"] as Language } catch (error) { - await this.telemetryService.capture({ - eventName: "Detect language error", - eventType: "DETECT_LANGUAGE", - producer: { - channel: "Bot", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - detectLanguageStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: promptDto.text, - comparisonScore: 0, - answer: prompt.inputLanguage, - logData: undefined, - errorData: { - input: userInput, - error: error, - }, - }, - errorType: "DETECT_LANGUAGE", - tags: ["bot", "detect_language", "error"], - }); } - // console.log("LANGUAGE DETECTED...") + // this.logger.log("LANGUAGE DETECTED...") //@ts-ignore if (prompt.inputLanguage == "unk") { prompt.inputLanguage = prompt.input.inputLanguage as Language; } // verboseLogger("Detected Language =", prompt.inputLanguage) } - // console.log("TELEMETRYYYYY") - await this.telemetryService.capture({ - eventName: "Detect language", - eventType: "DETECT_LANGUAGE", - producer: { - channel: "Bot", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - detectLanguageStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: promptDto.text, - comparisonScore: 0, - answer: prompt.inputLanguage, - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "detect_language"], - }); } else if (promptDto.media) { let audioStartTime = Date.now(); if (promptDto.media.category == "base64audio" && promptDto.media.text) { @@ -331,39 +253,6 @@ export class AppController { response = await this.aiToolsService.speechToText(promptDto.media.text,prompt.inputLanguage,userId,sessionId) if (response.error) { - await this.telemetryService.capture({ - eventName: "Speech to text error", - eventType: "SPEECH_TO_TEXT_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: promptDto.text, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - language: prompt.inputLanguage, - error: response.error, - }, - }, - errorType: "SPEECH_TO_TEXT", - tags: ["bot", "speech_to_text", "error"], - }); errorLogger(response.error); this.monitoringService.incrementTotalFailureSessionsCount(); this.monitoringService.incrementSomethingWentWrongTryAgainCount(); @@ -374,37 +263,6 @@ export class AppController { }; } userInput = response["text"]; - // verboseLogger("speech to text =",userInput) - await this.telemetryService.capture({ - eventName: "Speech to text", - eventType: "SPEECH_TO_TEXT", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: userInput, - questionSubmitted: userInput, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: null, - }, - errorType: "SPEECH_TO_TEXT", - tags: ["bot", "speech_to_text"], - }); } else { this.monitoringService.incrementUnsupportedMediaCount(); errorLogger("Unsupported media"); @@ -419,7 +277,7 @@ export class AppController { } conversation.inputType = type; - // console.log("CP 1...") + // this.logger.log("CP 1...") //get flow let botFlowMachine; switch (configid) { @@ -446,7 +304,7 @@ export class AppController { type == "Text" ) { let hashedAadhaar = await argon2.hash(promptDto.text); - console.log("you have entered aadhaar", hashedAadhaar); + this.logger.log("you have entered aadhaar", hashedAadhaar); await this.prismaService.message.create({ data: { text: hashedAadhaar, @@ -459,7 +317,7 @@ export class AppController { } }) }else { - // console.log("creating a new message in Message table...") + // this.logger.log("creating a new message in Message table...") await this.prismaService.message.create({ data: { text: type == "Text" ? promptDto.text : null, @@ -487,78 +345,13 @@ export class AppController { } res['audio'] = await this.aiToolsService.textToSpeech(res.text,prompt.inputLanguage,promptDto.audioGender,userId,sessionId) if(res['audio']['error']){ - await this.telemetryService.capture({ - eventName: "Text to speech error", - eventType: "TEXT_TO_SPEECH_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: res.text, - comparisonScore: 0, - answer: prompt.inputLanguage, - logData: undefined, - errorData: { - input: res.text, - language: prompt.inputLanguage, - error: res["audio"]["error"], - }, - }, - errorType: "TEXT_TO_SPEECH", - tags: ["bot", "text_to_speech", "error"], - }); - } else { - await this.telemetryService.capture({ - eventName: "Text to speech", - eventType: "TEXT_TO_SPEECH", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: res.text, - comparisonScore: 0, - answer: prompt.inputLanguage, - logData: undefined, - errorData: undefined, - }, - errorType: "TEXT_TO_SPEECH", - tags: ["bot", "text_to_speech"], - }); } res["messageId"] = uuid.v4(); res["conversationId"] = conversation?.id; return res; } else { //translate to english - // console.log("Translating to English...") + // this.logger.log("Translating to English...") let translateStartTime = Date.now(); if (userInput == "resend OTP") { this.monitoringService.incrementResentOTPCount(); @@ -573,40 +366,6 @@ export class AppController { sessionId ) if(!response['text']) { - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: userInput, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: response["error"], - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger( "Sorry, We are unable to translate given input, please try again" ); @@ -619,72 +378,7 @@ export class AppController { }; } prompt.inputTextInEnglish = response["text"]; - // verboseLogger("translated english text =", prompt.inputTextInEnglish) - await this.telemetryService.capture({ - eventName: "Translate", - eventType: "TRANSLATE", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: userInput, - comparisonScore: 0, - answer: prompt.inputTextInEnglish, - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "translate"], - }); } catch (error) { - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: userInput, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: error, - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger( "Sorry, We are unable to translate given input, please try again" ); @@ -734,13 +428,6 @@ export class AppController { prompt.inputTextInEnglish = isOTP ? "1111" : "AB123123123"; isNumber = true; } - // let number = prompt.inputTextInEnglish.replace(/\s/g, '') - - // if(/\d/.test(number)){ - // isNumber = true - // prompt.inputTextInEnglish = number.toUpperCase() - // verboseLogger("english text to numbers conversion =",prompt.inputTextInEnglish) - // } } } @@ -763,7 +450,7 @@ export class AppController { currentState: state.value, }; botFlowService.state.context = updatedContext; - // console.log('Current context:', state.context); + // this.logger.log('Current context:', state.context); if (state.context.type == "pause") { // verboseLogger("paused state", state.value) resolve(state); @@ -818,9 +505,7 @@ export class AppController { placeholder = engMessage["label.popUpTitle.short"]; } if (prompt.inputLanguage != Language.en && !isNumber) { - let translateStartTime = Date.now(); try { - let inp = result.text; let response = await this.aiToolsService.translate( Language.en, prompt.inputLanguage as Language, @@ -829,40 +514,6 @@ export class AppController { sessionId ) if(!response['text']){ - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: result.text, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: response["error"], - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger( "Sorry, We are unable to translate given input, please try again" ); @@ -872,72 +523,7 @@ export class AppController { "Sorry, We are unable to translate given input, please try again"; } result.text = response["text"]; - // verboseLogger("input language translated text =",result.text) - await this.telemetryService.capture({ - eventName: "Translate", - eventType: "TRANSLATE", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: inp, - comparisonScore: 0, - answer: result.text, - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "translate"], - }); } catch (error) { - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: userInput, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: error, - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger(error); this.monitoringService.incrementTotalFailureSessionsCount(); this.monitoringService.incrementUnableToTranslateCount(); @@ -959,40 +545,6 @@ export class AppController { sessionId ) if(!response['text']){ - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: placeholder, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: response["error"], - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger( "Sorry, We are unable to translate given input, please try again" ); @@ -1002,71 +554,7 @@ export class AppController { "Sorry, We are unable to translate given input, please try again"; } result["placeholder"] = response["text"]; - await this.telemetryService.capture({ - eventName: "Translate", - eventType: "TRANSLATE", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: placeholder, - comparisonScore: 0, - answer: result["placeholder"], - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "translate"], - }); } catch (error) { - await this.telemetryService.capture({ - eventName: "Translate error", - eventType: "TRANSLATE_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - translateStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: placeholder, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: userInput, - language: prompt.inputLanguage, - error: error, - }, - }, - errorType: "TRANSLATE", - tags: ["bot", "translate", "error"], - }); errorLogger(error); this.monitoringService.incrementTotalFailureSessionsCount(); this.monitoringService.incrementUnableToTranslateCount(); @@ -1131,78 +619,11 @@ export class AppController { let audioStartTime = Date.now(); textToaudio = removeLinks(textToaudio) result['audio'] = await this.aiToolsService.textToSpeech(textToaudio,isNumber ? Language.en : prompt.inputLanguage,promptDto.audioGender,userId,sessionId) - if(result['audio']['error']){ - await this.telemetryService.capture({ - eventName: "Text to speech error", - eventType: "TEXT_TO_SPEECH_ERROR", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: textToaudio, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: { - input: textToaudio, - language: prompt.inputLanguage, - error: result["audio"]["error"], - }, - }, - errorType: "TEXT_TO_SPEECH", - tags: ["bot", "text_to_speech", "error"], - }); - } else { - await this.telemetryService.capture({ - eventName: "Text to speech", - eventType: "TEXT_TO_SPEECH", - producer: { - channel: "Bhashini", - deviceID: null, - producerID: userId, - platform: "nodejs", - }, - platform: "nodejs", - sessionId: userId, - context: { - userID: userId, - conversationID: userId, - pageID: null, - rollup: undefined, - }, - eventData: { - duration: `${Date.now() - audioStartTime}`, - audioURL: null, - questionGenerated: null, - questionSubmitted: textToaudio, - comparisonScore: 0, - answer: null, - logData: undefined, - errorData: undefined, - }, - errorType: null, - tags: ["bot", "text_to_speech"], - }); - } } catch (error) { result["audio"] = { text: "", error: error.message }; } } - // console.log("Saving conversation..") + // this.logger.log("Saving conversation..") conversation = await this.conversationService.saveConversation( sessionId, userId, diff --git a/src/app.module.ts b/src/app.module.ts index a12d0e7..731a890 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -5,7 +5,6 @@ import { PrismaService } from "./global-services/prisma.service"; import { ConfigModule, ConfigService } from "@nestjs/config"; import { UserModule } from "./modules/user/user.module"; import { APP_GUARD, APP_PIPE } from "@nestjs/core"; -import { CustomLogger } from "./common/logger"; import { ConversationService } from "./modules/conversation/conversation.service"; import { ConversationModule } from "./modules/conversation/conversation.module"; import { PrometheusModule } from "@willsoto/nestjs-prometheus"; @@ -13,13 +12,17 @@ import { RateLimiterGuard } from './rate-limiter.guard'; import { ThrottlerModule } from '@nestjs/throttler'; import { MonitoringModule } from "./modules/monitoring/monitoring.module"; import { PromptModule } from "./xstate/prompt/prompt.module"; -import { TelemetryModule } from "./modules/telemetry/telemetry.module"; -import { TelemetryService } from "./modules/telemetry/telemetry.service"; import { MonitoringController } from "./modules/monitoring/monitoring.controller"; +import { CacheProvider } from "./modules/cache/cache.provider"; +import { LokiLoggerModule } from "./modules/loki-logger/loki-logger.module"; +import { HttpModule } from "@nestjs/axios"; @Module({ imports: [ - ConfigModule.forRoot(), + HttpModule, + ConfigModule.forRoot({ + isGlobal: true, + }), UserModule, ConversationModule, MonitoringModule, @@ -29,12 +32,12 @@ import { MonitoringController } from "./modules/monitoring/monitoring.controller } }), PromptModule, - TelemetryModule, ThrottlerModule.forRoot({ ttl: 60, // Time in seconds for the window (e.g., 60 seconds) limit: 10, // Maximum requests per window }), - CacheModule.register() + CacheModule.register(), + LokiLoggerModule ], controllers: [AppController], providers: [ @@ -42,17 +45,17 @@ import { MonitoringController } from "./modules/monitoring/monitoring.controller PrismaService, ConfigService, ConversationService, - TelemetryService, MonitoringController, { provide: APP_PIPE, useClass: ValidationPipe, }, - CustomLogger, { provide: APP_GUARD, useClass: RateLimiterGuard, }, + CacheProvider ], + exports: [CacheProvider], }) export class AppModule {} \ No newline at end of file diff --git a/src/app.service.ts b/src/app.service.ts index dae9d5c..8c44a63 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -2,6 +2,9 @@ import { Injectable } from "@nestjs/common"; import { PromptDto } from "./app.controller"; import { Language } from "./language"; import { PrismaService } from "./global-services/prisma.service"; +import { LokiLogger } from "./modules/loki-logger/loki-logger.service"; +import { HttpService } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; // Overlap between LangchainAI and Prompt-Engine const cron = require('node-cron'); export interface Prompt { @@ -20,16 +23,26 @@ export interface Prompt { } @Injectable() export class AppService { + private logger: LokiLogger; + constructor( - private prismaService: PrismaService - ){} + private prismaService: PrismaService, + private httpService: HttpService, + private configService: ConfigService + ){ + this.logger = new LokiLogger( + AppService.name, + httpService, + configService, + ); + } getHello(): string { return "Hello World!"; } onApplicationBootstrap() { // Schedule the task to run every hour (adjust as needed) - console.log("scheduling cron for every 30min") + this.logger.log("scheduling cron for every 30min"); cron.schedule('*/30 * * * *', () => { this.clearAadhaarNumbers(); }); @@ -71,9 +84,9 @@ export class AppService { }); } - console.log('Cleared userAadhaarNumber in conversations older than 30 minutes.'); + this.logger.log('Cleared userAadhaarNumber in conversations older than 30 minutes.'); } catch (error) { - console.error('Error clearing Aadhaar numbers:', error); + this.logger.error('Error clearing Aadhaar numbers:', error); } } } diff --git a/src/common/fetch.ts b/src/common/fetch.ts index ce61f61..39fa677 100644 --- a/src/common/fetch.ts +++ b/src/common/fetch.ts @@ -1,3 +1,13 @@ +const { LokiLogger } = require('../modules/loki-logger/loki-logger.service'); +const { HttpService } = require('@nestjs/axios'); +const { ConfigService } = require('@nestjs/config'); + +const logger = new LokiLogger( + 'fetch', + new HttpService(), + new ConfigService() +); + module.exports = async (url, opts) => { let retry = opts && opts.retry || 3 @@ -23,15 +33,14 @@ module.exports = async (url, opts) => { } if (opts && opts.pause) { - if (opts && !opts.silent) console.log("pausing.."); + if (opts && !opts.silent) logger.log("pausing.."); await sleep(opts.pause); - if (opts && !opts.silent) console.log("done pausing..."); + if (opts && !opts.silent) logger.log("done pausing..."); } } } }; - function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } \ No newline at end of file diff --git a/src/common/logger.ts b/src/common/logger.ts deleted file mode 100644 index 06a25a5..0000000 --- a/src/common/logger.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { Logger, Injectable } from '@nestjs/common'; -import * as winston from 'winston'; -import { WinstonTransport as AxiomTransport } from '@axiomhq/axiom-node'; - -@Injectable() -export class CustomLogger extends Logger { - private static formatTimestamp(date: Date): string { - const hours = date.getHours(); - const hours12 = hours % 12 || 12; - const minutes = date.getMinutes().toString().padStart(2, "0"); - const seconds = date.getSeconds().toString().padStart(2, "0"); - const milliseconds = date.getMilliseconds().toString().padStart(3, "0"); - const amPm = hours >= 12 ? "PM" : "AM"; - - return `${hours12}:${minutes}:${seconds}.${milliseconds} ${amPm}`; - } - - private static combineLogs(params: any[]): string { - return params?.map(param => { - try { - param = JSON.stringify(param,null,2) - } catch { - param = param - } - return param - }).join(" ") - } - - private formatLog(level, params) { - const timestamp = CustomLogger.formatTimestamp(new Date()); - return { - level, - message: CustomLogger.combineLogs(params), - service: this.serviceName, - timestamp - } - } - - private readonly axiomLogger: winston.Logger; - private readonly serviceName: string; - - constructor(serviceName) { - super(); - const { combine, errors, json } = winston.format; - const axiomTransport = new AxiomTransport(); - this.axiomLogger = winston.createLogger({ - level: 'silly', - format: combine(errors({ stack: true }), json()), - transports: [axiomTransport], - exceptionHandlers: [axiomTransport], - rejectionHandlers: [axiomTransport] - }); - this.serviceName = serviceName; - } - - logToAxiomAndConsole(logData) { - switch(logData.level) { - case "info": - super.log(logData?.message, this.serviceName, logData?.timestamp); - break; - case "error": - super.error(logData?.message, this.serviceName, logData?.timestamp); - break; - case "warn": - super.warn(logData?.message, this.serviceName, logData?.timestamp); - break; - case "debug": - super.debug(logData?.message, this.serviceName, logData?.timestamp); - break; - case "verbose": - super.verbose(logData?.message, this.serviceName, logData?.timestamp); - break; - default: - super.log(logData?.message, this.serviceName, logData?.timestamp); - break; - } - if(process.env.ENVIRONMENT == 'Staging' || process.env.ENVIRONMENT == 'Production') - this.axiomLogger.log(logData) - } - - log(...params: any[]) { - this.logToAxiomAndConsole(this.formatLog('info',params)) - } - - error(...params: any[]) { - this.logToAxiomAndConsole(this.formatLog('error',params)) - } - - warn(...params: any[]) { - this.logToAxiomAndConsole(this.formatLog('warn',params)) - } - - debug(...params: any[]) { - this.logToAxiomAndConsole(this.formatLog('debug',params)) - } - - verbose(...params: any[]) { - this.logToAxiomAndConsole(this.formatLog('verbose',params)) - } - - logWithCustomFields(customFields, level="info") { - return (...params: any[]) => { - let logData = this.formatLog(level,params) - logData = { - ...customFields, - ...logData - } - this.logToAxiomAndConsole(logData) - } - } -} \ No newline at end of file diff --git a/src/common/utils.ts b/src/common/utils.ts index d09a114..c8daf37 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,5 +1,14 @@ -const fetch = require("node-fetch"); +const fetch = require("./fetch"); const { Headers } = fetch; +const { LokiLogger } = require('../modules/loki-logger/loki-logger.service'); +const { HttpService } = require('@nestjs/axios'); +const { ConfigService } = require('@nestjs/config'); + +const logger = new LokiLogger( + 'utils', + new HttpService(), + new ConfigService() +); export function isMostlyEnglish(text: string): boolean { const englishCharacterCount = ( @@ -148,7 +157,7 @@ export const encryptRequest = async (text: string) => { try { var myHeaders = new Headers(); myHeaders.append("Content-Type", "application/json"); - console.log("text: ", text); + logger.log("text: ", text); var raw = JSON.stringify({ EncryptedRequest: text, }); diff --git a/src/global-services/prisma.service.ts b/src/global-services/prisma.service.ts index c5f22b5..09c4dca 100644 --- a/src/global-services/prisma.service.ts +++ b/src/global-services/prisma.service.ts @@ -1,15 +1,22 @@ +import { HttpService } from "@nestjs/axios"; import { INestApplication, Injectable, OnModuleInit, } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { PrismaClient } from "@prisma/client"; -import { CustomLogger } from "../common/logger"; +import { LokiLogger } from "src/modules/loki-logger/loki-logger.service"; @Injectable() export class PrismaService extends PrismaClient implements OnModuleInit { - private readonly logger = new CustomLogger("DBService"); + private readonly configService = new ConfigService(); + private readonly logger = new LokiLogger( + 'main', + new HttpService(), + this.configService, + ); async onModuleInit() { this.logger.verbose("Initialized and Connected 🎉"); await this.$connect(); diff --git a/src/main.ts b/src/main.ts index d5eefb1..830d68b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,11 +10,12 @@ import helmet from "@fastify/helmet"; import multipart from "@fastify/multipart"; import compression from "@fastify/compress"; import { join } from "path"; -import { CustomLogger } from "./common/logger"; import { MonitoringService } from "./modules/monitoring/monitoring.service"; +import { LokiLogger } from "./modules/loki-logger/loki-logger.service"; +import { HttpService } from "@nestjs/axios"; +import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; async function bootstrap() { - const logger = new CustomLogger("Main"); process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0' /** Fastify Application */ @@ -29,6 +30,22 @@ async function bootstrap() { /** Global prefix: Will result in appending of keyword 'admin' at the start of all the request */ const configService = app.get(ConfigService); + const logger = new LokiLogger( + 'main', + new HttpService(), + configService, + ); + + // Setup Swagger + const config = new DocumentBuilder() + .setTitle('PM Kisan API Documentation') + .setDescription('The PM Kisan API description') + .setVersion('1.0') + .addBearerAuth() + .build(); + const document = SwaggerModule.createDocument(app, config); + SwaggerModule.setup('api', app, document); + app.register(helmet, { contentSecurityPolicy: { directives: { @@ -41,24 +58,24 @@ async function bootstrap() { }); process.on('exit', (code) => { - console.log(`Process is exiting with code: ${code}`); + logger.log(`Process is exiting with code: ${code}`); }) process.on('beforeExit', async () => { - console.log("process exit...") + logger.log("process exit...") const monitoringService = app.get(MonitoringService); await monitoringService.onExit(); }); process.on('SIGINT', async () => { - console.log('Received SIGINT signal. Gracefully shutting down...'); + logger.log('Received SIGINT signal. Gracefully shutting down...'); const monitoringService = app.get(MonitoringService); await monitoringService.onExit(); process.exit(0); }); process.on('SIGTERM', async () => { - console.log('Received SIGTERM signal. Gracefully shutting down...'); + logger.log('Received SIGTERM signal. Gracefully shutting down...'); const monitoringService = app.get(MonitoringService); await monitoringService.onExit(); process.exit(0); @@ -72,7 +89,7 @@ async function bootstrap() { await app.register(multipart); await app.register(compression, { encodings: ["gzip", "deflate"] }); app.useStaticAssets({ root: join(__dirname, "../../fileUploads") }); - await app.listen(3000, "0.0.0.0"); + await app.listen(3001, "0.0.0.0"); } bootstrap(); diff --git a/src/modules/aiTools/ai-tools.service.ts b/src/modules/aiTools/ai-tools.service.ts index 60a84fb..333eb4a 100644 --- a/src/modules/aiTools/ai-tools.service.ts +++ b/src/modules/aiTools/ai-tools.service.ts @@ -4,6 +4,8 @@ import { ConfigService } from "@nestjs/config"; import { Language } from "../../language"; import { isMostlyEnglish } from "src/common/utils"; import { MonitoringService } from "../monitoring/monitoring.service"; +import { LokiLogger } from "../loki-logger/loki-logger.service"; +import { HttpService } from '@nestjs/axios'; const fetch = require("../../common/fetch"); const nodefetch = require("node-fetch"); const { Headers } = require("node-fetch"); @@ -13,13 +15,22 @@ const engMessage = require(filePath); @Injectable() export class AiToolsService { + private logger: LokiLogger; + constructor( private configService: ConfigService, private readonly monitoringService: MonitoringService, + private httpService: HttpService, @Inject(CACHE_MANAGER) private readonly cacheManager: Cache - ) {} + ) { + this.logger = new LokiLogger( + AiToolsService.name, + httpService, + configService + ); + } + async detectLanguage(text: string, userId: string, sessionId: string): Promise { - // console.log("DETECTING LANGUAGE....") try { let input = { input: [ @@ -40,7 +51,7 @@ export class AiToolsService { sessionId ) if(response["error"]){ - console.log(response["error"]) + this.logger.error(response["error"]) throw new Error(response["error"]) } let language: Language; @@ -106,7 +117,6 @@ export class AiToolsService { { input: [ { - // "source": text?.replace("\n",".") "source": textArray[i] } ] @@ -115,7 +125,7 @@ export class AiToolsService { sessionId ) if(response["error"]){ - console.log(response["error"]) + this.logger.error(response["error"]) throw new Error(response["error"]) } textArray[i] = response?.pipelineResponse[0]?.output[0]?.target; @@ -128,7 +138,7 @@ export class AiToolsService { error: null, }; } catch (error) { - console.log(error); + this.logger.error(error); return { text: "", error: error, @@ -174,7 +184,7 @@ export class AiToolsService { sessionId ) if(response["error"]){ - console.log(response["error"]) + this.logger.error(response["error"]) throw new Error(response["error"]) } return { @@ -182,7 +192,7 @@ export class AiToolsService { error: null, }; } catch (error) { - console.log(error); + this.logger.error(error); return { text: "", error: error, @@ -226,7 +236,7 @@ export class AiToolsService { sessionId ) if(response["error"]){ - console.log(response["error"]) + this.logger.error(response["error"]) throw new Error(response["error"]) } return { @@ -234,7 +244,7 @@ export class AiToolsService { error: null, }; } catch (error) { - console.log(error); + this.logger.error(error); return { text: "", error: error, @@ -247,16 +257,6 @@ export class AiToolsService { var myHeaders = new Headers(); myHeaders.append("accept", "application/json"); myHeaders.append("X-API-Key", this.configService.get("WADHWANI_API_KEY")); - // let body = { - // text: text - // } - // let response: any = await fetch(`${this.configService.get("HUGGINGFACE_TEXT_CLASSIFICATION_BASE_URL")}`, { - // headers: myHeaders, - // "body": JSON.stringify(body), - // "method": "POST", - // "mode": "cors", - // "credentials": "omit" - // }); let response: any = await fetch( `${this.configService.get( "WADHWANI_BASE_URL" @@ -271,7 +271,7 @@ export class AiToolsService { response = await response.text(); return response; } catch (error) { - console.log(error); + this.logger.error(error); return { error, }; @@ -289,7 +289,7 @@ export class AiToolsService { myHeaders.append("accept", "application/json"); myHeaders.append("X-API-Key", this.configService.get("WADHWANI_API_KEY")); let startDate = new Date(); - console.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${this.configService.get("WADHWANI_BASE_URL")}/get_bot_response?query=${text}&user_id=${userId}&session_id=${sessionId}&scheme_name=${schemeName} to respond ...`) + this.logger.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${this.configService.get("WADHWANI_BASE_URL")}/get_bot_response?query=${text}&user_id=${userId}&session_id=${sessionId}&scheme_name=${schemeName} to respond ...`) let response: any = await fetch(`${this.configService.get("WADHWANI_BASE_URL")}/get_bot_response?query=${text}&user_id=${userId}&session_id=${sessionId}&scheme_name=${schemeName}`, { headers: myHeaders, "method": "GET", @@ -298,10 +298,10 @@ export class AiToolsService { }); let endDate = new Date(); response = await response.json() - console.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${this.configService.get("WADHWANI_BASE_URL")}/get_bot_response?query=${text}&user_id=${userId}&session_id=${sessionId} Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) + this.logger.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${this.configService.get("WADHWANI_BASE_URL")}/get_bot_response?query=${text}&user_id=${userId}&session_id=${sessionId} Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) return response } catch(error){ - console.log(error) + this.logger.error(error) return { error, }; @@ -348,27 +348,27 @@ export class AiToolsService { requestOptions.callback = function (retry) { const elapsed = Date.now() - this.startTime; - console.log(`userId: ${this.userId} sessionId: ${this.sessionId} URL: ${this.url} (config API) Re-Trying: ${retry}, Previous failed call Time Taken: ${elapsed}ms`); - }.bind(requestOptions); + this.logger.error(`userId: ${this.userId} sessionId: ${this.sessionId} URL: ${this.url} (config API) Re-Trying: ${retry}, Previous failed call Time Taken: ${elapsed}ms`); + }.bind({...requestOptions, logger: this.logger}); try{ this.monitoringService.incrementBhashiniCount() let startDate = new Date(); - console.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${this.configService.get("ULCA_CONFIG_URL")} (config API) to respond ...`) + this.logger.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${this.configService.get("ULCA_CONFIG_URL")} (config API) to respond ...`) let response = await fetch(this.configService.get("ULCA_CONFIG_URL"), requestOptions) if(response.status != 200){ - console.log(response) + this.logger.error(response) throw new Error(`${new Date()}: API call to '${this.configService.get("ULCA_CONFIG_URL")}' with config '${JSON.stringify(config,null,3)}' failed with status code ${response.status}`) } let endDate = new Date(); response = await response.json() - console.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${this.configService.get("ULCA_CONFIG_URL")} (config API) Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) + this.logger.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${this.configService.get("ULCA_CONFIG_URL")} (config API) Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) this.monitoringService.incrementBhashiniSuccessCount() await this.cacheManager.set(cacheKey, response, 86400); return response; } catch (error) { this.monitoringService.incrementBhashiniFailureCount(); - console.log(error); + this.logger.error(error); return { error, }; @@ -422,21 +422,21 @@ export class AiToolsService { requestOptions.callback = function (retry) { const elapsed = Date.now() - this.startTime; - console.log(`userId: ${this.userId} sessionId: ${this.sessionId} URL: ${this.url} for task (${this.task}) Re-Trying: ${retry}, Previous failed call Time Taken: ${elapsed}ms`); - }.bind(requestOptions); + this.logger.error(`userId: ${this.userId} sessionId: ${this.sessionId} URL: ${this.url} for task (${this.task}) Re-Trying: ${retry}, Previous failed call Time Taken: ${elapsed}ms`); + }.bind({...requestOptions, logger: this.logger}); try{ this.monitoringService.incrementBhashiniCount() let startDate = new Date(); - console.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${url} for task (${task}) to respond ...`) + this.logger.log(`${startDate}: userId: ${userId} sessionId: ${sessionId} Waiting for ${url} for task (${task}) to respond ...`) let response = await fetch(url, requestOptions) if(response.status != 200){ - console.log(response) + this.logger.error(response) throw new Error(`${new Date()}: API call to '${url}' with config '${JSON.stringify(config,null,3)}' failed with status code ${response.status}`) } let endDate = new Date(); response = await response.json() - console.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${url} for task (${task}) Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) + this.logger.log(`${endDate}: userId: ${userId} sessionId: ${sessionId} URL: ${url} for task (${task}) Responded succesfully in ${endDate.getTime()-startDate.getTime()} ms.`) this.monitoringService.incrementBhashiniSuccessCount() if(task != 'asr') { await this.cacheManager.set(cacheKey, response, 7200); @@ -444,7 +444,7 @@ export class AiToolsService { return response; } catch (error) { this.monitoringService.incrementBhashiniFailureCount(); - console.log(error); + this.logger.error(error); return { error, }; diff --git a/src/modules/cache/cache.provider.ts b/src/modules/cache/cache.provider.ts new file mode 100644 index 0000000..ae89dbb --- /dev/null +++ b/src/modules/cache/cache.provider.ts @@ -0,0 +1,38 @@ +import { Injectable } from '@nestjs/common'; +import * as redisStore from 'cache-manager-redis-store'; +import * as cacheManager from 'cache-manager'; +import { RedisClientType, createClient } from 'redis'; + +@Injectable() +export class CacheProvider { + private cache: cacheManager.Cache; + private redisClient: RedisClientType; + + constructor() { + this.redisClient = createClient({ url: 'redis://localhost:6379' }); + this.cache = cacheManager.caching({ + store: redisStore, + client: this.redisClient, + ttl: 60 * 5, // Time to live in seconds + }); + } + + // Method to get a value from cache + async get(key: string): Promise { + return this.cache.get(key); + } + + // Method to set a value in cache + async set(key: string, value: T): Promise { + return this.cache.set(key, value); + } + + // Optional: Method to delete a key from cache + async del(key: string): Promise { + return this.cache.del(key); + } + + async increment(key: string): Promise { + return this.cache.increment(key); + } +} \ No newline at end of file diff --git a/src/modules/conversation/conversation.controller.ts b/src/modules/conversation/conversation.controller.ts index 7f9fe9c..64a3061 100644 --- a/src/modules/conversation/conversation.controller.ts +++ b/src/modules/conversation/conversation.controller.ts @@ -1,9 +1,26 @@ import { Controller, Post, Body, Get } from '@nestjs/common'; import { ConversationService } from './conversation.service'; +import { ApiTags, ApiOperation, ApiBody, ApiResponse } from '@nestjs/swagger'; +@ApiTags('Conversation') @Controller('conversation') export class ConversationController { constructor(private conversationService: ConversationService) {} + + @ApiOperation({ summary: 'Create or update feedback' }) + @ApiBody({ + description: 'Feedback data', + schema: { + type: 'object', + properties: { + // Add properties based on your feedback structure + feedback: { type: 'string' }, + rating: { type: 'number' } + } + } + }) + @ApiResponse({ status: 200, description: 'Feedback successfully created/updated' }) + @ApiResponse({ status: 400, description: 'Bad request' }) @Post('/feedback') async createOrUpdateFeedback(@Body() body: any){ return this.conversationService.createOrUpdateFeedback(body) diff --git a/src/modules/conversation/conversation.service.ts b/src/modules/conversation/conversation.service.ts index 2bffcfb..bc6ccaa 100644 --- a/src/modules/conversation/conversation.service.ts +++ b/src/modules/conversation/conversation.service.ts @@ -1,18 +1,22 @@ import { Injectable } from "@nestjs/common"; import { PrismaService } from "../../global-services/prisma.service"; import { ConfigService } from "@nestjs/config"; -import { CustomLogger } from "../../common/logger"; import { feedback } from "@prisma/client"; - +import { LokiLogger } from "../loki-logger/loki-logger.service"; +import { HttpService } from "@nestjs/axios"; @Injectable() export class ConversationService { - private logger: CustomLogger; + private logger: LokiLogger; constructor( private prisma: PrismaService, private configService: ConfigService ) { - this.logger = new CustomLogger("ConversationService"); + this.logger = new LokiLogger( + 'main', + new HttpService(), + this.configService, + ); } async saveConversation( diff --git a/src/modules/loki-logger/loki-logger.module.ts b/src/modules/loki-logger/loki-logger.module.ts new file mode 100644 index 0000000..a5016ea --- /dev/null +++ b/src/modules/loki-logger/loki-logger.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { HttpModule, HttpService } from '@nestjs/axios'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { LokiLogger } from './loki-logger.service'; + +@Module({ + imports: [HttpModule, ConfigModule], + providers: [ + { + provide: LokiLogger, + useFactory: (httpService: HttpService, configService: ConfigService) => { + return new LokiLogger('YourAppName', httpService, configService); + }, + inject: [HttpService, ConfigService], + }, + ], + exports: [LokiLogger], +}) +export class LokiLoggerModule {} \ No newline at end of file diff --git a/src/modules/loki-logger/loki-logger.service.ts b/src/modules/loki-logger/loki-logger.service.ts new file mode 100644 index 0000000..ddd327e --- /dev/null +++ b/src/modules/loki-logger/loki-logger.service.ts @@ -0,0 +1,191 @@ +// loki-logger.service.ts + +import { Logger, LoggerService } from '@nestjs/common'; +import { HttpService } from '@nestjs/axios'; +import { ConfigService } from '@nestjs/config'; +import { catchError, firstValueFrom } from 'rxjs'; +import { AxiosError } from 'axios'; + +export class LokiLogger extends Logger implements LoggerService { + private readonly httpService: HttpService; + private readonly configService: ConfigService; + private readonly serviceName: string; + + constructor( + context: string, + httpService: HttpService, + configService: ConfigService, + ) { + super(context); + this.httpService = httpService; + this.configService = configService; + this.serviceName = context; + } + + private static formatTimestamp(date: Date): string { + const hours = date.getHours(); + const hours12 = hours % 12 || 12; + const minutes = date.getMinutes().toString().padStart(2, "0"); + const seconds = date.getSeconds().toString().padStart(2, "0"); + const milliseconds = date.getMilliseconds().toString().padStart(3, "0"); + const amPm = hours >= 12 ? "PM" : "AM"; + + return `${hours12}:${minutes}:${seconds}.${milliseconds} ${amPm}`; + } + + private static combineLogs(params: any[]): string { + return params?.map(param => { + try { + param = JSON.stringify(param,null,2) + } catch { + param = param + } + return param + }).join(" ") + } + + private formatLog(level, params) { + const timestamp = LokiLogger.formatTimestamp(new Date()); + return { + level, + message: LokiLogger.combineLogs(params), + service: this.serviceName, + timestamp + } + } + + log( + message: any, + trace?: string, + context?: string, + ) { + super.log(message); + this.pushToLoki('info', message, context, trace); + } + + error( + message: any, + trace?: string, + context?: string, + ) { + super.error(message, trace, context); + this.pushToLoki('error', message, context, trace); + } + + warn( + message: any, + trace?: string, + context?: string, + ) { + super.warn(message, context); + this.pushToLoki('warn', message, context, trace); + } + + debug( + message: any, + trace?: string, + context?: string + ) { + super.debug(message, context); + this.pushToLoki('debug', message, context, trace); + } + + verbose( + message: any, + trace?: string, + context?: string + ) { + super.verbose(message, context); + this.pushToLoki('verbose', message, context, trace); + } + + private async pushToLoki( + level: string, + message: any, + context?: string, + trace?: string, + ) { + const timestamp = Date.now() * 1e6; // Convert to nanoseconds + const logEntry = { + level, + message: typeof message === 'object' ? JSON.stringify(message) : message, + context: context || this.context, + trace, + env: process.env.NODE_ENV + }; + + const logs = { + streams: [ + { + stream: { + level, + app: this.serviceName + }, + values: [[timestamp.toString(), JSON.stringify(logEntry)]], + }, + ], + }; + const LokiURL = this.configService.get('GRAFANA_URL'); + try { + await firstValueFrom( + this.httpService + .post(LokiURL, logs, { + headers: { + 'Content-Type': 'application/json', + }, + }) + .pipe( + catchError((error: AxiosError) => { + console.error( + 'Error pushing logs to Loki:', + error.response?.data, + ); + throw error; + }), + ), + ); + } catch (error) { + console.error('Failed to push logs to Loki:', error); + } + } + + logToLokiAndConsole(logData) { + let customFieldsString = ''; + if (logData.customFields) { + customFieldsString = Object.entries(logData.customFields) + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + } + switch(logData.level) { + case "info": + this.log(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + case "error": + this.error(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + case "warn": + this.warn(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + case "debug": + this.debug(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + case "verbose": + this.verbose(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + default: + this.log(`${customFieldsString} ${logData?.message}`, this.serviceName, logData?.timestamp); + break; + } + } + + logWithCustomFields(customFields, level="info") { + return (...params: any[]) => { + let logData = this.formatLog(level,params) + logData = { + ...customFields, + ...logData + } + this.logToLokiAndConsole(logData) + } + } +} diff --git a/src/modules/monitoring/monitoring.controller.ts b/src/modules/monitoring/monitoring.controller.ts index 69fe01a..26dde3a 100644 --- a/src/modules/monitoring/monitoring.controller.ts +++ b/src/modules/monitoring/monitoring.controller.ts @@ -184,10 +184,4 @@ export class MonitoringController { return 'metrics set' } -} - - - - - - +} \ No newline at end of file diff --git a/src/modules/monitoring/monitoring.module.ts b/src/modules/monitoring/monitoring.module.ts index 7a7a35e..1058341 100644 --- a/src/modules/monitoring/monitoring.module.ts +++ b/src/modules/monitoring/monitoring.module.ts @@ -3,10 +3,13 @@ import { Module, OnModuleInit, Global, DynamicModule } from '@nestjs/common'; import { MonitoringService } from './monitoring.service'; import { PrismaService } from 'src/global-services/prisma.service'; import { MonitoringController } from './monitoring.controller'; +import { CacheProvider } from '../cache/cache.provider'; +import { HttpModule } from '@nestjs/axios'; @Global() @Module({ - providers: [MonitoringService,PrismaService], + imports: [HttpModule], + providers: [MonitoringService, PrismaService, CacheProvider], exports: [MonitoringService], controllers: [MonitoringController], }) diff --git a/src/modules/monitoring/monitoring.service.ts b/src/modules/monitoring/monitoring.service.ts index 93c62b3..d848740 100644 --- a/src/modules/monitoring/monitoring.service.ts +++ b/src/modules/monitoring/monitoring.service.ts @@ -1,12 +1,13 @@ import { Injectable } from '@nestjs/common'; import { Counter } from 'prom-client'; import { PrismaService } from '../../global-services/prisma.service'; +import { CacheProvider } from '../cache/cache.provider'; @Injectable() export class MonitoringService { - constructor(private prismaService: PrismaService){} + constructor(private prismaService: PrismaService, private cache: CacheProvider) {} - async initializeAsync(){ + async initializeAsync() { const metricsToUpsert: any = [ { name: 'bhashiniCount' }, { name: 'bhashiniSuccessCount' }, @@ -42,130 +43,130 @@ export class MonitoringService { { name: "untrainedQueryCount" }, { name: "resentOTPCount" }, { name: "stage1Count" }, - { name: "stage2Count" }, + { name: "stage2Count" }, { name: "stage3Count" }, { name: "stage4Count" }, { name: "stage5Count" }, ]; - for (const metric of metricsToUpsert){ + for (const metric of metricsToUpsert) { const existingMetric: any = await this.prismaService.metrics.findUnique({ where: { name: metric.name }, }); - if(existingMetric){ - switch(existingMetric.name){ + if (existingMetric) { + switch (existingMetric.name) { case 'bhashiniCount': - this.bhashiniCounter.inc(parseInt(existingMetric.value)); + this.bhashiniCounter.inc(parseInt((await this.cache.get('bhashiniCount')) || '0')); break; case 'bhashiniSuccessCount': - this.bhashiniSuccessCounter.inc(parseInt(existingMetric.value)); + this.bhashiniSuccessCounter.inc(parseInt((await this.cache.get('bhashiniSuccessCount')) || '0')); break; case 'bhashiniFailureCount': - this.bhashiniFailureCounter.inc(parseInt(existingMetric.value)); + this.bhashiniFailureCounter.inc(parseInt((await this.cache.get('bhashiniFailureCount')) || '0')); break; case 'totalSessions': - this.totalSessionsCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsCounter.inc(parseInt((await this.cache.get('totalSessions')) || '0')); break; case 'totalSuccessfullSessions': - this.totalSuccessfullSessionsCounter.inc(parseInt(existingMetric.value)); + this.totalSuccessfullSessionsCounter.inc(parseInt((await this.cache.get('totalSuccessfullSessions')) || '0')); break; - case 'totalFailureSessions' : - this.totalFailureSessionsCounter.inc(parseInt(existingMetric.value)); + case 'totalFailureSessions': + this.totalFailureSessionsCounter.inc(parseInt((await this.cache.get('totalFailureSessions')) || '0')); break; case 'totalIncompleteSessions': - this.totalIncompleteSessionsCounter.inc(parseInt(existingMetric.value)); + this.totalIncompleteSessionsCounter.inc(parseInt((await this.cache.get('totalIncompleteSessions')) || '0')); break case 'totalSessionsInHindi': - this.totalSessionsInHindiCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInHindiCounter.inc(parseInt((await this.cache.get('totalSessionsInHindi')) || '0')); break; case 'totalSessionsInTamil': - this.totalSessionsInTamilCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInTamilCounter.inc(parseInt((await this.cache.get('totalSessionsInTamil')) || '0')); break; case 'totalSessionsInOdia': - this.totalSessionsInOdiaCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInOdiaCounter.inc(parseInt((await this.cache.get('totalSessionsInOdia')) || '0')); break; case 'totalSessionsInTelugu': - this.totalSessionsInTeluguCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInTeluguCounter.inc(parseInt((await this.cache.get('totalSessionsInTelugu')) || '0')); break; case 'totalSessionsInMarathi': - this.totalSessionsInMarathiCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInMarathiCounter.inc(parseInt((await this.cache.get('totalSessionsInMarathi')) || '0')); break; case 'totalSessionsInBangla': - this.totalSessionsInBanglaCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInBanglaCounter.inc(parseInt((await this.cache.get('totalSessionsInBangla')) || '0')); break; case 'totalSessionsInEnglish': - this.totalSessionsInEnglishCounter.inc(parseInt(existingMetric.value)); + this.totalSessionsInEnglishCounter.inc(parseInt((await this.cache.get('totalSessionsInEnglish')) || '0')); break; case "aadhaarCount": - this.aadhaarCounter.inc(parseInt(existingMetric.value)); + this.aadhaarCounter.inc(parseInt((await this.cache.get('aadhaarCount')) || '0')); break; case "registrationIdCount": - this.registrationIdCounter.inc(parseInt(existingMetric.value)); + this.registrationIdCounter.inc(parseInt((await this.cache.get('registrationIdCount')) || '0')); break; case "mobileNumberCount": - this.mobileNumberCounter.inc(parseInt(existingMetric.value)); + this.mobileNumberCounter.inc(parseInt((await this.cache.get('mobileNumberCount')) || '0')); break; case "positveFeedbackCount": - this.positveFeedbackCounter.inc(parseInt(existingMetric.value)); + this.positveFeedbackCounter.inc(parseInt((await this.cache.get('positveFeedbackCount')) || '0')); break; case "negativeFeedbackCount": - this.negativeFeedbackCounter.inc(parseInt(existingMetric.value)); + this.negativeFeedbackCounter.inc(parseInt((await this.cache.get('negativeFeedbackCount')) || '0')); break; case "micUsedCount": - this.micUsedCounter.inc(parseInt(existingMetric.value)); + this.micUsedCounter.inc(parseInt((await this.cache.get('micUsedCount')) || '0')); break; case "directMessageTypedCount": - this.directMessageTypedCounter.inc(parseInt(existingMetric.value)); + this.directMessageTypedCounter.inc(parseInt((await this.cache.get('directMessageTypedCount')) || '0')); break; case "sampleQueryUsedCount": - this.sampleQueryUsedCounter.inc(parseInt(existingMetric.value)); + this.sampleQueryUsedCounter.inc(parseInt((await this.cache.get('sampleQueryUsedCount')) || '0')); break; case "internalServerErrorCount": - this.internalServerErrorCounter.inc(parseInt(existingMetric.value)); + this.internalServerErrorCounter.inc(parseInt((await this.cache.get('internalServerErrorCount')) || '0')); break; case "badGatewayCount": - this.badGatewayCounter.inc(parseInt(existingMetric.value)); + this.badGatewayCounter.inc(parseInt((await this.cache.get('badGatewayCount')) || '0')); break; case "gatewayTimeoutCount": - this.gatewayTimeoutCounter.inc(parseInt(existingMetric.value)); + this.gatewayTimeoutCounter.inc(parseInt((await this.cache.get('gatewayTimeoutCount')) || '0')); break; case "somethingWentWrongCount": - this.somethingWentWrongCounter.inc(parseInt(existingMetric.value)); + this.somethingWentWrongCounter.inc(parseInt((await this.cache.get('somethingWentWrongCount')) || '0')); break; case "unsupportedMediaCount": - this.unsupportedMediaCounter.inc(parseInt(existingMetric.value)); + this.unsupportedMediaCounter.inc(parseInt((await this.cache.get('unsupportedMediaCount')) || '0')); break; case "unableToTranslateCount": - this.unableToTranslateCounter.inc(parseInt(existingMetric.value)); + this.unableToTranslateCounter.inc(parseInt((await this.cache.get('unableToTranslateCount')) || '0')); break; case "somethingWentWrongTryAgainCount": - this.somethingWentWrongTryAgainCounter.inc(parseInt(existingMetric.value)); + this.somethingWentWrongTryAgainCounter.inc(parseInt((await this.cache.get('somethingWentWrongTryAgainCount')) || '0')); break; case "unableToGetUserDetailsCount": - this.unableToGetUserDetailsCounter.inc(parseInt(existingMetric.value)); + this.unableToGetUserDetailsCounter.inc(parseInt((await this.cache.get('unableToGetUserDetailsCount')) || '0')); break; - case "noUserRecordsFoundCount": - this.noUserRecordsFoundCounter.inc(parseInt(existingMetric.value)); + case "noUserRecordsFoundCount": + this.noUserRecordsFoundCounter.inc(parseInt((await this.cache.get('noUserRecordsFoundCount')) || '0')); break; case "untrainedQueryCount": - this.untrainedQueryCounter.inc(parseInt(existingMetric.value)); + this.untrainedQueryCounter.inc(parseInt((await this.cache.get('untrainedQueryCount')) || '0')); break; case "resentOTPCount": - this.resentOTPCounter.inc(parseInt(existingMetric.value)); + this.resentOTPCounter.inc(parseInt((await this.cache.get('resentOTPCount')) || '0')); break; case "stage1Count": - this.stage1Counter.inc(parseInt(existingMetric.value)); + this.stage1Counter.inc(parseInt((await this.cache.get('stage1Count')) || '0')); break; case "stage2Count": - this.stage2Counter.inc(parseInt(existingMetric.value)); + this.stage2Counter.inc(parseInt((await this.cache.get('stage2Count')) || '0')); break; case "stage3Count": - this.stage3Counter.inc(parseInt(existingMetric.value)); + this.stage3Counter.inc(parseInt((await this.cache.get('stage3Count')) || '0')); break; case "stage4Count": - this.stage4Counter.inc(parseInt(existingMetric.value)); + this.stage4Counter.inc(parseInt((await this.cache.get('stage4Count')) || '0')); break; case "stage5Count": - this.stage5Counter.inc(parseInt(existingMetric.value)); + this.stage5Counter.inc(parseInt((await this.cache.get('stage5Count')) || '0')); break; default: break; @@ -236,7 +237,7 @@ export class MonitoringService { name: 'total_sessions_in_bangla_count', help: 'Counts the API requests of /prompt API', }); - + private totalSessionsInEnglishCounter: Counter = new Counter({ name: 'total_sessions_in_english_count', help: 'Counts the API requests of /prompt API', @@ -381,7 +382,7 @@ export class MonitoringService { let count = await this.totalSessionsCounter.get(); return count.values[0].value; } - + public async getTotalSuccessfullSessionsCount() { let count = await this.totalSuccessfullSessionsCounter.get(); return count.values[0].value; @@ -441,7 +442,7 @@ export class MonitoringService { let count = await this.registrationIdCounter.get(); return count.values[0].value; } - + public async getMobileNumberCount() { let count = await this.mobileNumberCounter.get(); return count.values[0].value; @@ -554,10 +555,12 @@ export class MonitoringService { public incrementBhashiniCount(): void { this.bhashiniCounter.inc(); + this.cache.increment('bhashiniCount'); } public incrementBhashiniSuccessCount(): void { this.bhashiniSuccessCounter.inc(); + this.cache.increment('bhashiniSuccessCount'); } public incrementBhashiniFailureCount(): void { @@ -566,149 +569,184 @@ export class MonitoringService { public incrementTotalSessionsCount() { this.totalSessionsCounter.inc(); + this.cache.increment('totalSessions'); } public incrementTotalSuccessfullSessionsCount() { this.totalSuccessfullSessionsCounter.inc(); + this.cache.increment('totalSuccessfullSessions'); } public incrementTotalFailureSessionsCount() { this.totalFailureSessionsCounter.inc(); + this.cache.increment('totalFailureSessions'); } public incrementTotalIncompleteSessionsCount() { this.totalIncompleteSessionsCounter.inc(); + this.cache.increment('totalIncompleteSessions'); } public incrementTotalSessionsInHindiCount() { this.totalSessionsInHindiCounter.inc(); + this.cache.increment('totalSessionsInHindi'); } public incrementTotalSessionsInTamilCount() { this.totalSessionsInTamilCounter.inc(); + this.cache.increment('totalSessionsInTamil'); } public incrementTotalSessionsInOdiaCount() { this.totalSessionsInOdiaCounter.inc(); + this.cache.increment('totalSessionsInOdia'); } public incrementTotalSessionsInTeluguCount() { this.totalSessionsInTeluguCounter.inc(); + this.cache.increment('totalSessionsInTelugu'); } public incrementTotalSessionsInMarathiCount() { this.totalSessionsInMarathiCounter.inc(); + this.cache.increment('totalSessionsInMarathi'); } public incrementTotalSessionsInBanglaCount() { this.totalSessionsInBanglaCounter.inc(); + this.cache.increment('totalSessionsInBangla'); } public incrementTotalSessionsInEnglishCount() { this.totalSessionsInEnglishCounter.inc(); + this.cache.increment('totalSessionsInEnglish'); } public incrementAadhaarCount() { this.aadhaarCounter.inc(); + this.cache.increment('aadhaarCount'); } public incrementRegistrationIdCount() { this.registrationIdCounter.inc(); + this.cache.increment('registrationIdCount'); } - + public incrementMobileNumberCount() { this.mobileNumberCounter.inc(); + this.cache.increment('mobileNumberCount'); } public incrementPositveFeedbackCount() { this.positveFeedbackCounter.inc(); + this.cache.increment('positveFeedbackCount'); } public incrementNegativeFeedbackCount() { this.negativeFeedbackCounter.inc(); + this.cache.increment('negativeFeedbackCount'); } public incrementMicUsedCount() { this.micUsedCounter.inc(); + this.cache.increment('micUsedCount'); } public incrementDirectMessageTypedCount() { this.directMessageTypedCounter.inc(); + this.cache.increment('directMessageTypedCount'); } public incrementSampleQueryUsedCount() { this.sampleQueryUsedCounter.inc(); + this.cache.increment('sampleQueryUsedCount'); } public incrementInternalServerErrorCount() { this.internalServerErrorCounter.inc(); + this.cache.increment('internalServerErrorCount'); } public incrementBadGatewayCount() { this.badGatewayCounter.inc(); + this.cache.increment('badGatewayCount'); } public incrementGatewayTimeoutCount() { this.gatewayTimeoutCounter.inc(); + this.cache.increment('gatewayTimeoutCount'); } public incrementSomethingWentWrongCount() { this.somethingWentWrongCounter.inc(); + this.cache.increment('somethingWentWrongCount'); } public incrementUnsupportedMediaCount() { this.unsupportedMediaCounter.inc(); + this.cache.increment('unsupportedMediaCount'); } public incrementUnableToTranslateCount() { this.unableToTranslateCounter.inc(); + this.cache.increment('unableToTranslateCount'); } public incrementSomethingWentWrongTryAgainCount() { this.somethingWentWrongTryAgainCounter.inc(); + this.cache.increment('somethingWentWrongTryAgainCount'); } public incrementUnableToGetUserDetailsCount() { this.unableToGetUserDetailsCounter.inc(); + this.cache.increment('unableToGetUserDetailsCount'); } public incrementNoUserRecordsFoundCount() { this.noUserRecordsFoundCounter.inc(); + this.cache.increment('noUserRecordsFoundCount'); } public incrementUntrainedQueryCount() { this.untrainedQueryCounter.inc(); + this.cache.increment('untrainedQueryCount'); } public incrementResentOTPCount() { this.resentOTPCounter.inc(); + this.cache.increment('resentOTPCount'); } public incrementStage1Count() { this.stage1Counter.inc(); + this.cache.increment('stage1Count'); } public incrementStage2Count() { this.stage2Counter.inc(); + this.cache.increment('stage2Count'); } public incrementStage3Count() { this.stage3Counter.inc(); + this.cache.increment('stage3Count'); } public incrementStage4Count() { this.stage4Counter.inc(); + this.cache.increment('stage4Count'); } public incrementStage5Count() { this.stage5Counter.inc(); + this.cache.increment('stage5Count'); } public async onExit(): Promise { const metricsToUpsert: any = [ - { name: 'bhashiniCount', value: `${await this.getBhashiniCount()}`}, - { name: 'bhashiniSuccessCount', value: `${await this.getBhashiniSuccessCount()}`}, - { name: 'bhashiniFailureCount', value: `${await this.getBhashiniFailureCount()}`}, + { name: 'bhashiniCount', value: `${await this.getBhashiniCount()}` }, + { name: 'bhashiniSuccessCount', value: `${await this.getBhashiniSuccessCount()}` }, + { name: 'bhashiniFailureCount', value: `${await this.getBhashiniFailureCount()}` }, { name: 'totalSessions', value: `${await this.getTotalSessionsCount()}` }, { name: 'totalSuccessfullSessions', value: `${await this.getTotalSuccessfullSessionsCount()}` }, { name: 'totalFailureSessions', value: `${await this.getTotalFailureSessionsCount()}` }, @@ -746,12 +784,12 @@ export class MonitoringService { { name: "stage5Count", value: `${await this.getStage5Count()}` }, ]; const upsertedMetrics = []; - try{ + try { for (const metric of metricsToUpsert) { const existingMetric: any = await this.prismaService.metrics.findUnique({ where: { name: metric.name }, }); - + if (existingMetric) { const updatedMetric = await this.prismaService.metrics.update({ where: { id: existingMetric.id }, @@ -765,19 +803,19 @@ export class MonitoringService { upsertedMetrics.push(createdMetric); } } - } catch(err){ + } catch (err) { console.log(err) } } public async setMetrics(metricsToUpsert): Promise { const upsertedMetrics = []; - try{ + try { for (const metric of metricsToUpsert) { const existingMetric: any = await this.prismaService.metrics.findUnique({ where: { name: metric.name }, }); - + if (existingMetric) { const updatedMetric = await this.prismaService.metrics.update({ where: { id: existingMetric.id }, @@ -791,9 +829,8 @@ export class MonitoringService { upsertedMetrics.push(createdMetric); } } - } catch(err){ + } catch (err) { console.log(err) } } - } \ No newline at end of file diff --git a/src/modules/telemetry/telemetry.controller.ts b/src/modules/telemetry/telemetry.controller.ts deleted file mode 100644 index 659df49..0000000 --- a/src/modules/telemetry/telemetry.controller.ts +++ /dev/null @@ -1,39 +0,0 @@ - -import { Controller, Get, Post, Query, Body, ParseIntPipe } from '@nestjs/common'; -import { TelemetryService } from './telemetry.service'; -import { telemetry_logs } from '@prisma/client'; - -@Controller('telemetry') -export class TelemetryController { - constructor(private telemetryLogsService: TelemetryService) {} - - @Post('/capture') - async createLog(@Body() data: telemetry_logs): Promise { - return this.telemetryLogsService.capture(data); - } - - @Get('/events') - async getEvents( - @Query('errorType') errorType?: string, - @Query('sessionId') sessionId?: string, - @Query('eid') eid?: string, - @Query('platform') platform?: string, - @Query('page') page: string = '1', - @Query('pageSize') pageSize: string = '10', - ): Promise { - const filters = { - errorType, - sessionId, - eid, - platform - }; - - return this.telemetryLogsService.getEvents(filters, parseInt(page), parseInt(pageSize)); - } -} - - - - - - diff --git a/src/modules/telemetry/telemetry.module.ts b/src/modules/telemetry/telemetry.module.ts deleted file mode 100644 index 7f33532..0000000 --- a/src/modules/telemetry/telemetry.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TelemetryService } from './telemetry.service'; -import { TelemetryController } from './telemetry.controller'; -import { PrismaService } from '../../global-services/prisma.service'; - -@Module({ - imports: [], - controllers: [TelemetryController], - providers: [TelemetryService, PrismaService], -}) -export class TelemetryModule {} \ No newline at end of file diff --git a/src/modules/telemetry/telemetry.service.ts b/src/modules/telemetry/telemetry.service.ts deleted file mode 100644 index 1d7dffe..0000000 --- a/src/modules/telemetry/telemetry.service.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { Prisma, telemetry_logs } from '@prisma/client'; -import { PrismaService } from '../../global-services/prisma.service'; - -@Injectable() -export class TelemetryService { - constructor(private prisma: PrismaService) {} - - async capture(data: Prisma.telemetry_logsCreateInput): Promise { - return this.prisma.telemetry_logs.create({ data }); - } - - async getEvents( - filters: { - errorType?: string; - sessionId?: string; - eid?: string; - platform?: string - }, - page: number, - pageSize: number, - ): Promise { - const { errorType, sessionId, eid, platform } = filters; - - return this.prisma.telemetry_logs.findMany({ - where: { - errorType: errorType || undefined, - sessionId: sessionId || undefined, - eid: eid || undefined, - platform: platform || undefined - }, - skip: (page - 1) * pageSize, - take: pageSize, - }); - } -} \ No newline at end of file diff --git a/src/modules/user/user.controller.ts b/src/modules/user/user.controller.ts index eb45b36..b5eb5dd 100644 --- a/src/modules/user/user.controller.ts +++ b/src/modules/user/user.controller.ts @@ -1,132 +1,57 @@ import { Body, Controller, Get, Param, Post, Headers } from '@nestjs/common'; import { UserService } from './user.service'; -import { CustomLogger } from 'src/common/logger'; import { PrismaService } from 'src/global-services/prisma.service'; import { Message } from '@prisma/client'; +import { ConfigService } from '@nestjs/config'; +import { LokiLogger } from '../loki-logger/loki-logger.service'; +import { HttpService } from '@nestjs/axios'; +import { ApiTags, ApiOperation, ApiResponse, ApiParam, ApiBody, ApiHeader } from '@nestjs/swagger'; const { v5: uuidv5 } = require('uuid'); +@ApiTags('User') @Controller('user') export class UserController { - private logger: CustomLogger; + private logger: LokiLogger; constructor( private readonly userService: UserService, - private readonly prismaService: PrismaService + private readonly prismaService: PrismaService, + private readonly configService: ConfigService ) { - this.logger = new CustomLogger("UserService"); - this.prismaService = new PrismaService(); - } - - @Get("/sendotp/:identifier") - async getOtp(@Param("identifier") identifier: string) { - if(/^[6-9]\d{9}$/.test(identifier)) { - return this.userService.sendOTP(identifier,"Mobile") - } else if(identifier.length==14 && /^[6-9]\d{9}$/.test(identifier.substring(0,10))){ - return this.userService.sendOTP(identifier,"MobileAadhar") - } else if(identifier.length==12 && /^\d+$/.test(identifier)){ - return this.userService.sendOTP(identifier,"Aadhar") - } else if(identifier.length == 11) { - return this.userService.sendOTP(identifier,"Ben_id") - } else { - return { - "status": "NOT_OK", - "error": "Please enter a valid Beneficiary ID/Aadhaar Number/Phone number" - } - } - } - - @Post("/verifyotp") - async verifyOtp(@Body() body: any ) { - if(/^[6-9]\d{9}$/.test(body.identifier)) { - return this.userService.verifyOTP(body.identifier,body.otp,"Mobile") - } else if(body.identifier.length==14 && /^[6-9]\d{9}$/.test(body.identifier.substring(0,10))){ - return this.userService.verifyOTP(body.identifier,body.otp,"MobileAadhar") - } else if(body.identifier.length==12 && /^\d+$/.test(body.identifier)){ - return this.userService.verifyOTP(body.identifier,body.otp,"Aadhar") - } else if(body.identifier.length == 11) { - return this.userService.verifyOTP(body.identifier,body.otp,"Ben_id") - }else { - return { - "status": "NOT_OK", - "error": "Please enter a valid Beneficiary ID/Aadhaar Number/Phone number" - } - } - } - - @Get("/getUserData/:identifier") - async getUserData(@Param("identifier") identifier: string) { - if(/^[6-9]\d{9}$/.test(identifier)) { - return this.userService.getUserData(identifier,"Mobile") - } else if(identifier.length==14 && /^[6-9]\d{9}$/.test(identifier.substring(0,10))){ - return this.userService.getUserData(identifier,"MobileAadhar") - } else if(identifier.length==12 && /^\d+$/.test(identifier)){ - return this.userService.getUserData(identifier,"Aadhar") - } else if(identifier.length == 11) { - return this.userService.getUserData(identifier,"Ben_id") - }else { - return { - "status": "NOT_OK", - "error": "Please enter a valid Beneficiary ID/Aadhaar Number/Phone number" - } - } + this.logger = new LokiLogger( + 'main', + new HttpService(), + this.configService, + ); } + @ApiOperation({ summary: 'Generate user ID' }) + @ApiParam({ name: 'identifier', description: 'Unique identifier to generate UUID' }) + @ApiResponse({ status: 200, description: 'User ID generated successfully' }) @Post('/generateUserId/:identifier') async generateUserId(@Param("identifier") identifier: string) { const uuid = uuidv5(identifier, uuidv5.DNS); return uuid } - @Get("/history/:flowId") - async prompt(@Headers() headers,@Param("flowId") flowId: string): Promise { - const userId = headers["user-id"] - if(!userId){ - return { - "status": "NOT_OK", - "error": "Invalid userId." - } - } - let user; - try{ - user = await this.prismaService.user.findUnique({ - where:{ - id: userId - }, - select: { - messages: { - where: { - flowId: flowId || '3' - } - } - } - }) - }catch{ - return { - "status": "NOT_OK", - "error": "Invalid userId." - } - } - if(!user) { - return { - "status": "NOT_OK", - "error": "Invalid userId." - } - } - return { - "status": "OK", - "data": user.messages - } - } - + @ApiOperation({ summary: 'Like a message' }) + @ApiParam({ name: 'id', description: 'Message ID to like' }) + @ApiResponse({ status: 200, description: 'Message liked successfully' }) @Get("message/like/:id") async likeQuery(@Param('id') id: string): Promise { return this.userService.likeQuery(id); } + @ApiOperation({ summary: 'Dislike a message' }) + @ApiParam({ name: 'id', description: 'Message ID to dislike' }) + @ApiResponse({ status: 200, description: 'Message disliked successfully' }) @Get("message/dislike/:id") async dislikeQuery(@Param('id') id: string): Promise { return this.userService.dislikeQuery(id); } + @ApiOperation({ summary: 'Remove reaction from message' }) + @ApiParam({ name: 'id', description: 'Message ID to remove reaction from' }) + @ApiResponse({ status: 200, description: 'Reaction removed successfully' }) @Get("message/removelike/:id") async removeLike(@Param('id') id: string): Promise { return this.userService.removeReactionOnQuery(id); diff --git a/src/modules/user/user.service.ts b/src/modules/user/user.service.ts index a4dca6b..47e7229 100644 --- a/src/modules/user/user.service.ts +++ b/src/modules/user/user.service.ts @@ -1,21 +1,26 @@ import { Injectable } from "@nestjs/common"; import { PrismaService } from "../../global-services/prisma.service"; import { ConfigService } from "@nestjs/config"; -import { CustomLogger } from "../../common/logger"; +import { LokiLogger } from '../loki-logger/loki-logger.service'; import axios from "axios"; import { decryptRequest, encryptRequest } from "../../common/utils"; import { Message } from "@prisma/client"; import { MonitoringService } from "../monitoring/monitoring.service"; +import { HttpService } from "@nestjs/axios"; @Injectable() export class UserService { - private logger: CustomLogger; + private logger: LokiLogger; constructor( private prisma: PrismaService, private configService: ConfigService, private monitoringService: MonitoringService ) { - this.logger = new CustomLogger("UserService"); + this.logger = new LokiLogger( + 'main', + new HttpService(), + this.configService, + ); } async sendOTP(mobileNumber: string, type: string = "Mobile"): Promise { @@ -25,7 +30,7 @@ export class UserService { "PM_KISSAN_TOKEN" )}\"}` ); - console.log("encrypted data: ", encryptedData); + this.logger.log("encrypted data: ", encryptedData); let data = JSON.stringify({ EncryptedRequest: `${encryptedData.d.encryptedvalu}@${encryptedData.d.token}`, }); @@ -41,7 +46,7 @@ export class UserService { }; let response: any = await axios.request(config); - console.log("sendOTP", response.status); + this.logger.log("sendOTP", response.status); if (response.status >= 200 && response.status < 300) { response = await response.data; let decryptedData: any = await decryptRequest( @@ -63,7 +68,7 @@ export class UserService { }; } } catch (error) { - console.log(error); + this.logger.error(error); return { d: { output: { @@ -103,14 +108,14 @@ export class UserService { }; let response: any = await axios.request(config); - console.log("verifyOTP", response.status); + this.logger.log("verifyOTP", response.status); if (response.status >= 200 && response.status < 300) { response = await response.data; let decryptedData: any = await decryptRequest( response.d.output, encryptedData.d.token ); - console.log(decryptedData); + this.logger.log(decryptedData); response.d.output = JSON.parse(decryptedData.d.decryptedvalue); response["status"] = response.d.output.Rsponce != "False" ? "OK" : "NOT_OK"; @@ -126,7 +131,7 @@ export class UserService { }; } } catch (error) { - console.log(error); + this.logger.error(error); return { d: { output: { @@ -165,7 +170,7 @@ export class UserService { data: data, }; res = await axios.request(config); - console.log("getUserData", res.status); + this.logger.log("getUserData", res.status); if (res.status >= 200 && res.status < 300) { res = await res.data; let decryptedData: any = await decryptRequest( diff --git a/src/xstate/prompt/prompt.machine.ts b/src/xstate/prompt/prompt.machine.ts index e66c1b9..77c4899 100644 --- a/src/xstate/prompt/prompt.machine.ts +++ b/src/xstate/prompt/prompt.machine.ts @@ -1,9 +1,18 @@ // @ts-nocheck +import { HttpService } from "@nestjs/axios"; +import { ConfigService } from "@nestjs/config"; +import { LokiLogger } from "src/modules/loki-logger/loki-logger.service"; import { assign } from "xstate"; const path = require("path"); const filePath = path.resolve(__dirname, "../../common/en.json"); const engMessage = require(filePath); +const logger = new LokiLogger( + 'prompt', + new HttpService(), + new ConfigService() +); + export const botFlowMachine1: any = { /** @xstate-layout N4IgpgJg5mDOIC5QCMD2AXAYgG1QdwDoZ0BVWMAJwEUBXOdAS1QDsBiEgZQFEAlAfQCSAOQAKJACoBtAAwBdRKAAOqWA0YsFIAB6IAjAA4AzAQAsukwDZDAVhMAma3f26ANCACeiALR2DBXQCc0sHWAboWugDsutIWAL5xbmhYuIQAjnSw6swAwtgAhrCqAGYMlKwQLGAEDMwAbqgA1tXJOPgEGfRMuQVFDKWUCLUNAMb52TKyk5rKqtmaOgi61rqmJtb61tbS+hYm0tFunghmBNJ2VvoHAfrRJobmCUkYbemZ2XmFJWUUrJQUqAoBEUBXQxUBAFsCK1Uh13t1Pn0BhQhvVUGMJnJpkgQLM1N0Ft5ItYCE57BtDBZNtYLNYjt59PoCLtrJFdmELoZDHZIk8QDD2oVGrUoABBfIQAAW+XyFCENAhyHKnF4glEEmxShU+I0OMWUSZ4UsFgcPOCAXW9IQdhMkVJkQdhluFxWtmsfIFhDq+WwDAg4xF4qlMrlCqVv0qzGqwyaLResO9vv9jGYYol0tl8sVgxjGO6k01uO18z13kcFgIAQsARWkUpGyc+itjmkBGs3K5gQdkRr+g98faib9AdTQYzoezEaqNTRzWhA69PuHKbTwczYZzaLzLALunkOLxJdA+p2qx25cZ5gCDmbdlb7bsnYC3d7-ZSg6XycD6ZDWfDFWnGM509Aghy-Ucf3XSdUVGAMdyxOx9y1OYCVLBAaQCStpF0QxIhtW1OxMW97w7B5nwdV9En5BdQM-EdV3HP9ykjaNZzjd9FyTeix1-DcUVzODmALQwkKLFDdWPRAwlMXCbTsfDNktDxEBbNtSK7CiNjfV5aK4lceKg-9-kBYFQXBCgoRAsDuMgidwxg9FBILOQZmLVDJIQHxpBJQwax2fQa0MWILB2K1fLsNtIhNFZOWrbD3SokChRFAB5cQRHYbh+GEMQpBcg83Ik7Q9CMUxzBpHYnF0AxQuUhArmMJ0NnbbzOwsXlEpo6yVzSjKWJnBpgK6uievShztyErF8uQnVmEJE4HUrZ8LV8ewHBwq1qpsNSTHuRxNP0ExtITEbUvSgCowG2N5w43TlzOkRxqcrE91c8S5rQ6qIp5fZuWwmJIh2CxNuWJkVmkKsAhrKlGWOj89Iev4KABIEQXGczLOGhHU16p7MSmaaxNm+avHCJk72JJ0q0iXblhB7b212mweR7DYjs627ijAdARklEUyEoAARcZ8gu1jBvYnSuZ5vnUwFihhfQfI8fzKbRMPdziqWELTA2NkWw2KlpEMTbLGMVkeQMB0bQpOHCGl3n+fIBWRaRlHTPRyEbql7nHbl53FeVgT8cLDWisWUmosrfZiUh58IjsTaOwIWOQqCdtNgCO2CGMqdLqAyXYVzlX4IJ9XCo+jzSdZAhCKBu8HlpOk6uWaSNmWW1duiXCEio5hUAgOBNE9N7ibQrxWRMStq1ret9EbK0vCdCstlZXQLnuExr1sbPiHl2guiKsPK61o1TAdcILSCAGLkXuwobOLY8MfGtAaMeIOZ0zosgRXpvkoUeR5T64UrEFBwZsqyxUXkEZkAVfB4QtDYB02dkoQTXHZABBV3rzXMBFK4YR448j2GyTa7Z-BmBrL4K41VwjZ26t+dBTEKCAM1vqPwJhGSbHng8YKPIwrVjONeAi7UQoxGfCg2AwocbpRYeHPQ15a7tR7NyHkVZHCJzqoDYwsQjAWkcFvYIhg6GnWkSIWRJ99TbFMBvKIyx6xBGBnVZ8xhnC3ACpnahdhs4O1llAeWgdzE4MsNHB42xcJVmwgEJOvgU6UhwsSdq88jDZ1zoE8eURWyHUBpvSwDYmwt0sKsWwQi8K7HsMSFJzAICvDSR5AwRSoi4XuNFf6m1aStgiEFTha1QgJQSEAA */ id: "botFlow", @@ -475,7 +484,7 @@ export const botFlowMachine2: any = { assign({ response: () => engMessage["label.popUpTitle"], queryType: (_, event) => { - console.log(`assigning queryType = ${event.data}`); + logger.log(`assigning queryType = ${event.data}`); return event.data; }, type: "pause", @@ -740,7 +749,7 @@ export const botFlowMachine2: any = { assign({ query: (_, event) => event.data.query, otp: (_context, event) => { - console.log("setting user otp"); + logger.log("setting user otp"); return `${event.data.query}`; }, type: "", @@ -1384,7 +1393,7 @@ export const botFlowMachine3: any = { assign({ query: (_, event) => event.data.query, otp: (_context, event) => { - console.log("setting user otp"); + logger.log("setting user otp"); return `${event.data.query}`; }, type: "", diff --git a/src/xstate/prompt/prompt.module.ts b/src/xstate/prompt/prompt.module.ts index ef61f43..855977c 100644 --- a/src/xstate/prompt/prompt.module.ts +++ b/src/xstate/prompt/prompt.module.ts @@ -1,3 +1,4 @@ +import { HttpModule } from "@nestjs/axios"; import { CacheModule, Module } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { PrismaService } from "src/global-services/prisma.service"; @@ -5,7 +6,7 @@ import { AiToolsService } from "src/modules/aiTools/ai-tools.service"; import { UserService } from "src/modules/user/user.service"; @Module({ - imports: [CacheModule.register()], + imports: [CacheModule.register(),HttpModule], providers: [ PrismaService, ConfigService, diff --git a/src/xstate/prompt/prompt.service.ts b/src/xstate/prompt/prompt.service.ts index 6dad23e..867454f 100644 --- a/src/xstate/prompt/prompt.service.ts +++ b/src/xstate/prompt/prompt.service.ts @@ -15,6 +15,8 @@ import { createMachine } from "xstate"; import { promptActions } from "./prompt.actions"; import { promptGuards } from "./prompt.gaurds"; import { MonitoringService } from "src/modules/monitoring/monitoring.service"; +import { LokiLogger } from "src/modules/loki-logger/loki-logger.service"; +import { HttpService } from "@nestjs/axios"; const path = require("path"); const filePath = path.resolve(__dirname, "../../common/kisanPortalErrors.json"); const PMKissanProtalErrors = require(filePath); @@ -23,17 +25,21 @@ import * as moment from "moment"; @Injectable() export class PromptServices { private userService: UserService; + private logger: LokiLogger; + constructor( private prismaService: PrismaService, private configService: ConfigService, private aiToolsService: AiToolsService, - private monitoringService: MonitoringService + private monitoringService: MonitoringService, + private httpService: HttpService ) { this.userService = new UserService( this.prismaService, this.configService, this.monitoringService ); + this.logger = new LokiLogger('prompt', this.httpService, this.configService); } async getInput(context) { @@ -41,18 +47,10 @@ export class PromptServices { } async questionClassifier (context) { - // console.log("IN questionclassifier") + this.logger.log("IN questionclassifier"); try{ let response: any = await this.aiToolsService.getResponseViaWadhwani(context.sessionId, context.userId, context.query, context.schemeName) if (response.error) throw new Error(`${response.error}, please try again.`) - // { - // "user_id": "19877818", - // "session_id": "123456", - // "query": "Installment not received", - // "query_intent": "Installment not received", - // "language": "English", - // "response": "Dear Beneficiary, You can check your status using Know Your Status (KYS) module at https://pmkisan.gov.in/BeneficiaryStatus_New.aspx. \nIf you are not satisfied with the status, please contact the PM Kisan officer Shri ABC on 9809898989 or you can also visit the Officer at PM Kisan Officer, 193310 village, 868 block, 965 sub-district, 123 district, 9, Pincode: . \nFor further assistant, please contact on the PM Kisan Samman Nidhi helpline number: 155261 / 011-24300606. The helpline is available on all working days from 9:30 AM to 6:00 PM." - // } let intent; if (response.query_intent == "Invalid") intent = "convo" if (response.query_intent == "convo_starter") intent = "convo" @@ -71,13 +69,13 @@ export class PromptServices { } async logError(_, event) { - console.log("logError"); - console.log(event.data); + this.logger.log("logError"); + this.logger.log(event.data); return event.data; } async validateAadhaarNumber(context, event) { - console.log("validate aadhar"); + this.logger.log("validate aadhar"); try { const userIdentifier = `${context.userAadhaarNumber}${context.lastAadhaarDigits}`; let res; @@ -112,13 +110,13 @@ export class PromptServices { this.monitoringService.incrementSomethingWentWrongCount(); throw new Error("Something went wrong."); } catch (error) { - console.log(error); + this.logger.error(error); return Promise.reject(new Error("Something went wrong.")); } } async validateOTP(context, event) { - console.log("Validate OTP"); + this.logger.log("Validate OTP"); const userIdentifier = `${context.userAadhaarNumber}${context.lastAadhaarDigits}`; const otp = context.otp; let res; @@ -157,7 +155,7 @@ export class PromptServices { } async fetchUserData(context, event) { - console.log("Fetch user data"); + this.logger.log("Fetch user data"); const userIdentifier = `${context.userAadhaarNumber}${context.lastAadhaarDigits}`; let res; let type = "Mobile"; @@ -201,8 +199,8 @@ export class PromptServices { res.d.output["eKYC_Status"] ); - console.log("ChatbotBeneficiaryStatus"); - console.log("using...", userIdentifier, type); + this.logger.log("ChatbotBeneficiaryStatus"); + this.logger.log("using...", userIdentifier, type); let userErrors = []; try { let encryptedData = await encryptRequest( @@ -213,7 +211,7 @@ export class PromptServices { let data = JSON.stringify({ EncryptedRequest: `${encryptedData.d.encryptedvalu}@${encryptedData.d.token}`, }); - console.log("body", data); + this.logger.log("body", data); let config = { method: "post", @@ -229,7 +227,7 @@ export class PromptServices { let errors: any = await axios.request(config); errors = await errors.data; - console.log("related issues", errors); + this.logger.log("related issues", errors); let decryptedData: any = await decryptRequest( errors.d.output, encryptedData.d.token @@ -245,7 +243,7 @@ export class PromptServices { context.queryType ) != -1 ) { - console.log(`ERRORVALUE: ${key} ${value}`); + this.logger.log(`ERRORVALUE: ${key} ${value}`); userErrors.push( PMKissanProtalErrors[`${value}`]["text"].replace( "{{farmer_name}}", @@ -274,14 +272,14 @@ export class PromptServices { ); } } catch (error) { - console.log("ChatbotBeneficiaryStatus error"); - console.log(error); + this.logger.error("ChatbotBeneficiaryStatus error"); + this.logger.error(error); } return `${userDetails}${userErrors.join("\n")}`; } async wadhwaniClassifier(context) { - console.log("Wadhwani Classifierrr"); + this.logger.log("Wadhwani Classifierrr"); try { let response: any = await this.aiToolsService.getResponseViaWadhwani( context.sessionId,