Skip to content

Commit

Permalink
DPO3DPKRT-864/Notification Emails (#633)
Browse files Browse the repository at this point in the history 
(new) send email routines via notifyEmail
(new) support for multiple recipients
(new) support for html body with text fallback
(new) support for embedded image/icon from message type (base64 or URL)
(new) robust server response handling
(new) email rate management with adaptive burst rates
(new) asynchronous calling for Logger + Notify
(new) metrics for average rates and counts

(fix) explicit calls to Logger vs. dynamic this. resolves 'this' binding
(fix) cleaner error handling and bubbling up of status
(fix) commented console handling in legacy Logger for cleaner debugging.
  • Loading branch information
@EMaslowskiQ
EMaslowskiQ authored Oct 11, 2024
1 parent 4779a9f commit 4d720cb
Show file tree
Hide file tree
Showing 9 changed files with 1,225 additions and 357 deletions.
11 changes: 7 additions & 4 deletions server/http/routes/sandbox.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@ import { RecordKeeper as RK } from '../../records/recordKeeper';

export const play = async (_req: Request, res: Response): Promise<void> => {

RK.configure(); // 'D:\\Temp\\PackratTemp\\Logs'
await RK.configure();

// test our logging
const numLogs: number = 1000;
const result = await RK.logTest(numLogs);
// const numLogs: number = 1000;
// const result = await RK.logTest(numLogs);

// test email notifications
const result = await RK.emailTest(5);

// return our results
res.status(200).send(H.Helpers.JSONStringify({ message: result.error }));
res.status(200).send(H.Helpers.JSONStringify(result));
};
225 changes: 116 additions & 109 deletions server/records/logger/log.ts
View file

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions server/records/notify/notify.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { NotifyResult, NotifyPackage, NotifyChannel, NotifyType } from './notifyShared';
import { NotifyEmail } from './notifyEmail';

export class Notify {

//#region EMAIL
public static configureEmail(env: 'prod' | 'dev', targetRate?: number, burstRate?: number, burstThreshold?: number): NotifyResult {
return NotifyEmail.configure(env,targetRate,burstRate,burstThreshold);
}

// cast the returns to NotifyResult so it's consistent with what is exported
public static sendEmailMessage = NotifyEmail.sendMessage as (params: NotifyPackage) => Promise<NotifyResult>;
public static sendEmailMessageRaw = NotifyEmail.sendMessageRaw as (type: NotifyType, sendTo: string[], subject: string, textBody: string, htmlBody?: string) => Promise<NotifyResult>;

// testing emails
public static testEmail = NotifyEmail.testEmails as (numEmails: number) => Promise<NotifyResult>;
//#endregion

//#region SLACK
//#endregio
}

// export shared types so they can be accessed via Notify
export { NotifyPackage, NotifyChannel, NotifyType };
472 changes: 472 additions & 0 deletions server/records/notify/notifyEmail.ts
View file

Large diffs are not rendered by default.

54 changes: 54 additions & 0 deletions server/records/notify/notifyShared.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
export enum NotifyType {
UNDEFINED = -1,
SYSTEM_ERROR = 1,
SYSTEM_NOTICE = 2,
JOB_FAILED = 10,
JOB_PASSED = 11,
JOB_STARTED = 12,
SECURITY_NOTICE = 20,
}
export enum NotifyChannel {
SLACK_OPS = 'slack-ops',
SLACK_DEV = 'slack-dev',
EMAIL_ADMIN = 'email-admin',
EMAIL_USER = 'email-user',
EMAIL_ALL = 'email-all'
}
export interface NotifyResult {
success: boolean,
message: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data?: any
}
export interface NotifyPackage {
type: NotifyType, // what type of message is this
message: string, // main message for the user
startDate: Date, // when did this job/event happen or start
endDate?: Date, // when did this job/event end (optional)
detailsLink?: { url: string, label: string }, // is there somewhere the user can get more info
detailsMessage?: string, // error message, scene name, full message, etc.
sendTo?: string[], // where are we sending the message. for Slack it is a user slack ID. for email it is the 'to' email address
}

export const getMessageIconUrlByType = (type: NotifyType, system: 'email' | 'slack'): string => {
// build our filename based on the type
const host: string = 'https://egofarms.com/packrat';
const typeStr: string = NotifyType[type];
const filename: string = `${host}/icon_${system}_${typeStr.toLowerCase()}.png`;
return filename;
};
export const getMessagePrefixByType = (type: NotifyType): string => {

switch(type) {
case NotifyType.SYSTEM_ERROR: return 'Error';
case NotifyType.SYSTEM_NOTICE: return 'Notice';

case NotifyType.JOB_FAILED: return 'Failed';
case NotifyType.JOB_PASSED: return 'Success';
case NotifyType.JOB_STARTED: return 'Start';

case NotifyType.SECURITY_NOTICE: return 'Notice';

default: return 'N/A';
}
};
183 changes: 149 additions & 34 deletions server/records/recordKeeper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,52 +2,94 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { Config } from '../config';
import { ASL, LocalStore } from '../utils/localStore';
import * as H from '../utils/helpers';
import { Logger as LOG, LogSection } from './logger/log';
import { Notify as NOTIFY, NotifyChannel, NotifyType, NotifyPackage } from './notify/notify';

/** TODO:
* - change H.IOResults.error to message and make a requirement
* - change IOResults.error to message and make a requirement
*/

// utils
const convertToIOResults = ( result: { success: boolean, message: string }): H.IOResults => {
// converts standard response from logging and notification systems into Packrat's current
// 'results' structure, which has 'error' instead of message.
return { success: result.success, error: result.message };
// const convertToIOResults = ( result: { success: boolean, message: string, data?: any }): IOResults => {
// // converts standard response from logging and notification systems into Packrat's current
// // 'results' structure, which has 'error' instead of message.
// let error: string | undefined = undefined;
// if(result.success===false && result.data) {
// if(result.data.error)
// error = result.data.error;
// else
// error = JSON.stringify(result.data);
// }

// const msg: string = result.message + ((error!=undefined) ? ` ${error}` : '');
// return { success: result.success, error: msg };
// };

// temp definition for where IOResults will be
type IOResults = {
success: boolean,
message: string,
data?: any
};
type ChannelConfig = {
[NotifyChannel.SLACK_DEV]: string,
[NotifyChannel.SLACK_OPS]: string,
[NotifyChannel.EMAIL_ADMIN]: string[],
[NotifyChannel.EMAIL_ALL]: string[],
};

export class RecordKeeper {

static LogSection = LogSection;
static NotifyChannel = NotifyChannel;
static NotifyType = NotifyType;

private static defaultEmail: string[] = ['[email protected]'];
private static notifyChannelConfig: ChannelConfig = {
[NotifyChannel.EMAIL_ADMIN]: [],
[NotifyChannel.EMAIL_ALL]: [],
[NotifyChannel.SLACK_DEV]: 'C07MKBKGNTZ', // packrat-dev
[NotifyChannel.SLACK_OPS]: 'C07NCJE9FJM', // packrat-ops
};

static configure(): H.IOResults {
static async configure(): Promise<IOResults> {

//#region CONFIG:LOGGER
// get our log path from the config
const environment: 'prod' | 'dev' = 'dev';
const logPath: string = Config.log.root ? Config.log.root : /* istanbul ignore next */ './var/logs';

// our default Logger configuration options
// base it on the environment variable for our base/target rate
const useRateManager: boolean = true; // do we manage our log output for greater consistency
const useRateManager: boolean = true; // do we manage our log output for greater consistency
const targetRate: number = Config.log.targetRate; // targeted logs per second (100-500)
const burstRate: number = targetRate * 5; // target rate when in burst mode and playing catchup
const burstThreshold: number = burstRate * 5; // when queue is bigger than this size, trigger 'burst mode'
const staggerLogs: boolean = true; // do we spread the logs out during each interval or all at once
const burstThreshold: number = burstRate; // when queue is bigger than this size, trigger 'burst mode'

// initialize logger sub-system
const logResults = LOG.configure(logPath, 'dev', useRateManager, targetRate, burstRate, burstThreshold, staggerLogs);
const logResults = LOG.configure(logPath, environment, useRateManager, targetRate, burstRate, burstThreshold);
if(logResults.success===false)
return convertToIOResults(logResults);
this.logInfo(LogSection.eSYS, logResults.message,{ path: logPath, useRateManager, targetRate, burstRate, burstThreshold, staggerLogs });
return logResults;
this.logInfo(LogSection.eSYS, logResults.message, { environment, path: logPath, useRateManager, targetRate, burstRate, burstThreshold }, 'Recordkeeper');
//#endregion

// initialize notify sub-system
// ...
//#region CONFIG:NOTIFY
const notifyResults = NOTIFY.configureEmail(environment);
if(notifyResults.success===false)
return notifyResults;
this.logInfo(LogSection.eSYS, notifyResults.message, { environment }, 'Recordkeeper');

return { success: true };
// get our email addresses from the system. these can be cached because they will be
// the same for all users and sessions. Uses defaults of 1 email/sec.
this.notifyChannelConfig[NotifyChannel.EMAIL_ADMIN] = await this.getEmailsFromChannel(NotifyChannel.EMAIL_ADMIN) ?? this.defaultEmail;
this.notifyChannelConfig[NotifyChannel.EMAIL_ALL] = await this.getEmailsFromChannel(NotifyChannel.EMAIL_ALL) ?? this.defaultEmail;
//#endregion

return { success: true, message: 'configured record keeper' };
}
static cleanup(): H.IOResults {
return { success: true };
static cleanup(): IOResults {
return { success: true, message: 'record keeper cleaned up' };
}

private static getContext(): { idUser: number, idRequest: number } {
// get our user and request ids from the local store
// TEST: does it maintain store context since static and not async
Expand All @@ -61,45 +103,118 @@ export class RecordKeeper {

//#region LOG
// Log routines for specific levels
static logCritical(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): void {
static async logCritical(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
LOG.critical(sec,message,data,caller,audit,idUser,idRequest);
return LOG.critical(sec,message,data,caller,audit,idUser,idRequest);
}
static logError(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): void {
static async logError(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
LOG.error(sec,message,data,caller,audit,idUser,idRequest);
return LOG.error(sec,message,data,caller,audit,idUser,idRequest);
}
static logWarning(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): void {
static async logWarning(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
LOG.warning(sec,message,data,caller,audit,idUser,idRequest);
return LOG.warning(sec,message,data,caller,audit,idUser,idRequest);
}
static logInfo(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): void {
static async logInfo(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
LOG.info(sec,message,data,caller,audit,idUser,idRequest);
return LOG.info(sec,message,data,caller,audit,idUser,idRequest);
}
static logDebug(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): void {
static async logDebug(sec: LogSection, message: string, data?: any, caller?: string, audit: boolean = false): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
LOG.debug(sec,message,data,caller,audit,idUser,idRequest);
return LOG.debug(sec,message,data,caller,audit,idUser,idRequest);
}

// profiler functions
// Usage: call 'profile' with a unique label and any needed metadata. This creates/starts a timer.
// to stop the timer and log the result, call profileEnd with the same label used to create it.
static profile(label: string, sec: LogSection, message: string, data?: any, caller?: string): H.IOResults {
static async profile(label: string, sec: LogSection, message: string, data?: any, caller?: string): Promise<IOResults> {
const { idUser, idRequest } = this.getContext();
return LOG.profile(label, sec, message, data, caller, idUser, idRequest);
}
static profileEnd(label: string): H.IOResults {
static async profileEnd(label: string): Promise<IOResults> {
return LOG.profileEnd(label);
}

// stats and utilities
static logTotalCount(): number {
return LOG.getStats().counts.total;
}
static async logTest(numLogs: number): Promise<H.IOResults > {
const result = await LOG.testLogs(numLogs);
return convertToIOResults(result);
static async logTest(numLogs: number): Promise<IOResults> {
return LOG.testLogs(numLogs);
}
//#endregion

//#region NOTIFY
// emails
private static async getEmailsFromChannel(channel: NotifyChannel, forceUpdate: boolean = false): Promise<string[] | undefined> {

switch(channel) {

case NotifyChannel.EMAIL_ALL: {
// see if we already initialized this
if(this.notifyChannelConfig[NotifyChannel.EMAIL_ALL].length>0 && forceUpdate===true)
return this.notifyChannelConfig[NotifyChannel.EMAIL_ALL];

// otherwise, grab all active Users from DB and their emails
return ['[email protected]'];
}

case NotifyChannel.EMAIL_ADMIN: {
// see if we already initialized this
if(this.notifyChannelConfig[NotifyChannel.EMAIL_ADMIN].length>0 && forceUpdate===true)
return this.notifyChannelConfig[NotifyChannel.EMAIL_ADMIN];

// get ids from Config and then get their emails
return ['[email protected]','[email protected]'];
}

case NotifyChannel.EMAIL_USER: {
// const { idUser } = this.getContext();
// TODO: from current user id
return ['[email protected]'];
}
}

return undefined;
}
static async sendEmail(type: NotifyType, channel: NotifyChannel, subject: string, body: string, startDate?: Date): Promise<IOResults> {

// build our package
const params: NotifyPackage = {
type,
message: subject,
detailsMessage: body,
startDate: startDate ?? new Date(),
sendTo: await this.getEmailsFromChannel(channel)
};

// send our message out.
// we await the result so we can catch and audit the failure
this.logInfo(LogSection.eSYS,'sending email',{ sendTo: params.sendTo },'RecordKeeper.sendEmail',true);
const emailResult = await NOTIFY.sendEmailMessage(params);
if(emailResult.success===false)
this.logError(LogSection.eSYS,'failed to send email',{ sendTo: params.sendTo },'RecordKeeper.sendEmail',true);

// return the results
return emailResult;
}
static async sendEmailRaw(type: NotifyType, sendTo: string[], subject: string, textBody: string, htmlBody?: string): Promise<IOResults> {

// send our email but also log it for auditing
// we wait for results so we can log the failure
this.logInfo(LogSection.eSYS,'sending raw email',{ sendTo },'RecordKeeper.sendEmailRaw',true);
const emailResult = await NOTIFY.sendEmailMessageRaw(type, sendTo, subject, textBody, htmlBody);
if(emailResult.success===false)
this.logError(LogSection.eSYS,'failed to send raw email',{ sendTo },'RecordKeeper.sendEmailRaw',true);

return emailResult;
}
static async emailTest(numEmails: number): Promise<IOResults> {
return NOTIFY.testEmail(numEmails);
}

// slack
// sendSlackMessage(...)
// sendSlackMessageRaw(...)
//#endregion
}
Loading

0 comments on commit 4d720cb

Please sign in to comment.