Enable lambda functions to run daily fetch through secrets manager and SAM template

README.md

@@ -62,9 +62,13 @@ Nope. You can [import transactions from a CSV bank statement](./docs/README.md#m
 
 Nope. You can [export your account balances & transactions to a CSV file](./docs/README.md#on-your-local-machine--via-csv-files) exclusively on your local machine.
 
+**Do I have to use AWS Secrets Manager?**
+
+Nope. You can keep your config information in a file exclusively on your local machine.
+
 **Do I have to manually run this every time I want new transactions in my spreadsheet?**
 
-Nope. You can automate it for free using [BitBar](./docs/README.md#automatically-in-your-macs-menu-bar--via-bitbar), [`cron`](./docs/README.md#automatically-in-your-local-machines-terminal--via-cron), or [GitHub Actions](./docs/README.md#automatically-in-the-cloud--via-github-actions).
+Nope. You can automate it for free using [BitBar](./docs/README.md#automatically-in-your-macs-menu-bar--via-bitbar), [`cron`](./docs/README.md#automatically-in-your-local-machines-terminal--via-cron), [GitHub Actions](./docs/README.md#automatically-in-the-cloud--via-github-actions), or [AWS Lambdas](./docs/README.md#automatically-in-the-cloud--via-aws-lambda-functions).
 
 **It's not working!**
 

docs/README.md

@@ -2,26 +2,29 @@
 
 #### Table of Contents
 
-- [Overview](#overview)
-- [Installation](#installation)
-  - [Creating a Fresh Installation](#creating-a-fresh-installation)
-  - [Migrating from `v1.x.x`](#migrating-from-v1xx)
-- [Importing Account Balances & Transactions](#importing-account-balances--transactions)
-  - [Automatically – in the cloud – via Plaid](#automatically-in-the-cloud--via-plaid)
-  - [Manually – on your local machine – via CSV bank statements](#manually--on-your-local-machine--via-csv-bank-statements)
-- [Exporting Account Balances & Transactions](#exporting-account-balances--transactions)
-  - [In the cloud – via Google Sheets](#in-the-cloud-via-google-sheets)
-  - [On your local machine – via CSV files](#on-your-local-machine--via-csv-files)
-- [Updating Transactions/Accounts](#updating-transactionsaccounts)
-  - [Manually – in your local machine's terminal](#manually-in-your-local-machines-terminal)
-  - [Automatically – in your Mac's Menu Bar – via BitBar](#automatically-in-your-macs-menu-bar--via-bitbar)
-  - [Automatically – in your local machine's terminal – via `cron`](#automatically-in-your-local-machines-terminal--via-cron)
-  - [Automatically – in the cloud – via GitHub Actions](#automatically-in-the-cloud--via-github-actions)
-- [Transaction Rules](#transaction-rules)
-  - [Transaction `filter` Rules](#transaction-filter-rules)
-  - [Transaction `override` Rules](#transaction-override-rules)
-- [Development](#development)
-- [Contributing](#contributing)
+- [Documentation](#documentation)
+      - [Table of Contents](#table-of-contents)
+  - [Overview](#overview)
+  - [Installation](#installation)
+    - [Creating a Fresh Installation](#creating-a-fresh-installation)
+    - [Migrating from `v1.x.x`](#migrating-from-v1xx)
+  - [Importing Account Balances & Transactions](#importing-account-balances--transactions)
+    - [Automatically – in the cloud – via Plaid](#automatically-in-the-cloud--via-plaid)
+    - [Manually – on your local machine – via CSV bank statements](#manually--on-your-local-machine--via-csv-bank-statements)
+  - [Exporting Account Balances & Transactions](#exporting-account-balances--transactions)
+    - [In the cloud – via Google Sheets](#in-the-cloud-via-google-sheets)
+    - [On your local machine – via CSV files](#on-your-local-machine--via-csv-files)
+  - [Updating Transactions/Accounts](#updating-transactionsaccounts)
+    - [Manually – in your local machine's terminal](#manually-in-your-local-machines-terminal)
+    - [Automatically – in your Mac's Menu Bar – via BitBar](#automatically-in-your-macs-menu-bar--via-bitbar)
+    - [Automatically – in your local machine's terminal – via `cron`](#automatically-in-your-local-machines-terminal--via-cron)
+    - [Automatically – in the cloud – via GitHub Actions](#automatically-in-the-cloud--via-github-actions)
+    - [Automatically – in the cloud – via AWS Lambda Functions](#automatically-in-the-cloud--via-aws-lambda-functions)
+  - [Transaction Rules](#transaction-rules)
+    - [Transaction `filter` Rules](#transaction-filter-rules)
+    - [Transaction `override` Rules](#transaction-override-rules)
+  - [Development](#development)
+  - [Contributing](#contributing)
 
 ## Overview
 
@@ -239,6 +242,45 @@ In the **Actions** tab of your repo, the **Fetch** workflow will now update your
 
 > **Note:** The minimum interval supported by GitHub Actions is every 5 minutes.
 
+### Automatically – in the cloud – via AWS Lambda Functions
+
+You can use AWS Lambda functions to run Mintable automatically in the cloud:
+
+> **Note:** This requires an AWS account with billing enabled, but Lambda invocations and your secrets manager secret will be covered under the free tier.
+> **Note:** Some of these steps can be skipped if you have already setup an AWS account and are authenticated with the CLI.
+
+1. Install mintable normally and setup your accounts.
+2. Fork [this repo](https://github.com/kevinschaich/mintable) and open the directory.
+3. [Install AWS cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
+4. [Install AWS SAM](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html).
+5. [Create an AWS account](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/).
+6. Go to [AWS secrets manager](https://console.aws.amazon.com/secretsmanager/home). 
+   1. Select **Store a new secret** >> **Other type of secret** >> **Plaintext**
+   2. Open your mintable generated config file and copy the contents. Then, paste over the contents in the box on AWS.
+   3. Select **Next** and give your secret a name in **Secret name**. Select **Next** >> **Next** >> **Store**.
+   4. Select your new secret and copy the **Secret ARN**.
+   5. Paste your copied ARN into SECRET_MANAGER_ARN in the `template.yaml` file.
+7.  Go to [AWS IAM](https://console.aws.amazon.com/iamv2/home#/roles). 
+8.  Select **Create role** >> **AWS Service** >> **Lambda** >> **Next: Permissions**
+    1.  Search **SecretsManagerReadWrite** and select the option.
+    2.  Select **Next: Tags** >> **Next: Review** and give your role a name.
+    3.  Select **Create Role**.
+    4.  Select your new role from the list. Copy the **Role ARN** and paste it into `template.yaml` file next to role.
+9.  [Create access keys for your root user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_root-user.html#id_root-user_manage_add-key).
+10. Authenticate with the AWS cli and paste your keys: 
+   ```
+    $ aws configure
+    AWS Access Key ID []:
+    AWS Secret Access Key []: 
+    Default region name []: us-east-1
+    Default output format [json]: 
+
+   ```
+15. Open a terminal in the cloned repo.
+16. Run `sam deploy --guided` and follow the prompts accordingly.
+
+Done! Look for you lambda in AWS.
+
 ---
 
 ## Transaction Rules

package.json

@@ -19,6 +19,7 @@
         "tsconfig.json"
     ],
     "dependencies": {
+        "@aws-sdk/client-secrets-manager": "^3.46.0",
         "@types/body-parser": "^1.19.0",
         "@types/express": "^4.17.3",
         "@types/glob": "^7.1.2",

src/common/config.ts

@@ -10,6 +10,10 @@ import { Definition, CompilerOptions, PartialArgs, getProgramFromFiles, generate
 import Ajv from 'ajv'
 import { BalanceConfig } from '../types/balance'
 import { jsonc } from 'jsonc'
+import { SecretsManagerClient, PutSecretValueCommand, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager";
+
+const REGION = "us-east-1";
+const secretsManagerClient = new SecretsManagerClient({ region: REGION });
 
 const DEFAULT_CONFIG_FILE = '~/mintable.jsonc'
 const DEFAULT_CONFIG_VAR = 'MINTABLE_CONFIG'
@@ -37,7 +41,12 @@ export interface EnvironmentConfig {
     variable: string
 }
 
-export type ConfigSource = FileConfig | EnvironmentConfig
+export interface SecretManagerConfig {
+    type: 'secretManager'
+    secretName: string
+}
+
+export type ConfigSource = FileConfig | EnvironmentConfig | SecretManagerConfig
 
 export interface Config {
     integrations: { [id: string]: IntegrationConfig }
@@ -58,14 +67,19 @@ export const getConfigSource = (): ConfigSource => {
         return { type: 'environment', variable: DEFAULT_CONFIG_VAR }
     }
 
+    if (process.env.SECRET_MANAGER_ARN) {
+        logInfo(`Using Secrets Manager.`)
+        return { type: 'secretManager', secretName: process.env.SECRET_MANAGER_NAME }
+    }
+
     // Default to DEFAULT_CONFIG_FILE
     const path = DEFAULT_CONFIG_FILE.replace(/^~(?=$|\/|\\)/, os.homedir())
     logInfo(`Using default configuration file \`${path}.\``)
     logInfo(`You can supply either --config-file or --config-variable to specify a different configuration.`)
     return { type: 'file', path: path }
 }
 
-export const readConfig = (source: ConfigSource, checkExists?: boolean): string => {
+export const readConfig = async (source: ConfigSource, checkExists?: boolean): Promise<string> => {
     if (source.type === 'file') {
         try {
             const config = fs.readFileSync(source.path, 'utf8')
@@ -98,6 +112,28 @@ export const readConfig = (source: ConfigSource, checkExists?: boolean): string
             }
         }
     }
+    if (source.type === 'secretManager') {
+        try {
+
+            const secret = await secretsManagerClient.send(new GetSecretValueCommand({
+                SecretId: source.secretName
+            }));
+            const config = secret.SecretString;
+
+            if (config === undefined) {
+                throw `Unable to get config from Secrets Manager.`
+            }
+
+            logInfo('Successfully retrieved configuration variable.')
+            return config
+        } catch (e) {
+            if (!checkExists) {
+                logInfo('Unable to read config variable from secrets manager.')
+            } else {
+                logError('Unable to read config variable from secrets manager', e)
+            }
+        }
+    }
 }
 
 export const parseConfig = (configString: string): Object => {
@@ -159,15 +195,15 @@ export const validateConfig = (parsedConfig: Object): Config => {
     return validatedConfig
 }
 
-export const getConfig = (): Config => {
+export const getConfig = async (): Promise<Config> => {
     const configSource = getConfigSource()
-    const configString = readConfig(configSource)
+    const configString = await readConfig(configSource)
     const parsedConfig = parseConfig(configString)
     const validatedConfig = validateConfig(parsedConfig)
     return validatedConfig
 }
 
-export const writeConfig = (source: ConfigSource, config: Config): void => {
+export const writeConfig = async (source: ConfigSource, config: Config): Promise<void> => {
     if (source.type === 'file') {
         try {
             fs.writeFileSync(source.path, jsonc.stringify(config, null, 2))
@@ -181,23 +217,35 @@ export const writeConfig = (source: ConfigSource, config: Config): void => {
             'Node does not have permissions to modify global environment variables. Please use file-based configuration to make changes.'
         )
     }
+    if (source.type === 'secretManager') {
+        try {
+            const putSecretValue = new PutSecretValueCommand({
+                SecretId: source.secretName,
+                SecretString: jsonc.stringify(config, null, 2)
+            });
+            await secretsManagerClient.send(putSecretValue);
+            logInfo('Successfully wrote configuration file.')
+        } catch (e) {
+            logError('Unable to write configuration file.', e)
+        }
+    }
 }
 
 type ConfigTransformer = (oldConfig: Config) => Config
 
-export const updateConfig = (configTransformer: ConfigTransformer, initialize?: boolean): Config => {
+export const updateConfig = async (configTransformer: ConfigTransformer, initialize?: boolean): Promise<Config> => {
     let newConfig: Config
     const configSource = getConfigSource()
 
     if (initialize) {
         newConfig = configTransformer(DEFAULT_CONFIG)
     } else {
-        const configString = readConfig(configSource)
+        const configString = await readConfig(configSource)
         const oldConfig = parseConfig(configString) as Config
         newConfig = configTransformer(oldConfig)
     }
 
     const validatedConfig = validateConfig(newConfig)
-    writeConfig(configSource, validatedConfig)
+    await writeConfig(configSource, validatedConfig)
     return validatedConfig
 }

src/integrations/csv-export/setup.ts

@@ -45,7 +45,7 @@ export default async () => {
                 }
             ])
 
-            updateConfig(config => {
+            await updateConfig(config => {
                 let CSVExportConfig =
                     (config.integrations[IntegrationId.CSVExport] as CSVExportConfig) || defaultCSVExportConfig
 
@@ -62,7 +62,7 @@ export default async () => {
             })
 
             logInfo('Successfully set up CSV Export Integration.')
-            return resolve()
+            return resolve(1)
         } catch (e) {
             logError('Unable to set up CSV Export Integration.', e)
             return reject()

src/integrations/csv-import/setup.ts

@@ -61,7 +61,7 @@ export default async () => {
                 integration: IntegrationId.CSVImport
             }
 
-            updateConfig(config => {
+            await updateConfig(config => {
                 let CSVImportConfig =
                     (config.integrations[IntegrationId.CSVImport] as CSVImportConfig) || defaultCSVImportConfig
 
@@ -78,7 +78,7 @@ export default async () => {
             )
 
             logInfo('Successfully set up CSV Import Integration.')
-            return resolve()
+            return resolve(1)
         } catch (e) {
             logError('Unable to set up CSV Import Integration.', e)
             return reject()

src/integrations/google/setup.ts

@@ -47,7 +47,7 @@ export default async () => {
                 }
             ])
 
-            updateConfig(config => {
+            await updateConfig(config => {
                 let googleConfig = (config.integrations[IntegrationId.Google] as GoogleConfig) || defaultGoogleConfig
 
                 googleConfig.name = credentials.name
@@ -63,7 +63,8 @@ export default async () => {
                 return config
             })
 
-            const google = new GoogleIntegration(getConfig())
+            const newConfig = await getConfig();
+            const google = new GoogleIntegration(newConfig);
             open(google.getAuthURL())
 
             console.log('\n\t5. A link will open in your browser asking you to sign in')
@@ -87,7 +88,7 @@ export default async () => {
             await google.saveAccessTokens(tokens)
 
             logInfo('Successfully set up Google Integration.')
-            return resolve()
+            return resolve(1)
         } catch (e) {
             logError('Unable to set up Plaid Integration.', e)
             return reject()

src/integrations/plaid/accountSetup.ts

@@ -13,7 +13,7 @@ export default async () => {
             console.log('\t2. Sign in with your banking provider for each account you wish to link.')
             console.log("\t3. Click 'Done Linking Accounts' in your browser when you are finished.\n")
 
-            const config = getConfig()
+            const config = await getConfig()
             const plaidConfig = config.integrations[IntegrationId.Plaid] as PlaidConfig
             const plaid = new PlaidIntegration(config)
 
@@ -22,7 +22,7 @@ export default async () => {
             await plaid.accountSetup()
 
             logInfo('Successfully set up Plaid Account(s).')
-            return resolve()
+            return resolve(1)
         } catch (e) {
             logError('Unable to set up Plaid Account(s).', e)
             return reject()

src/integrations/plaid/setup.ts

@@ -60,7 +60,7 @@ export default async () => {
                 }
             ])
 
-            updateConfig(config => {
+            await updateConfig(config => {
                 let plaidConfig = (config.integrations[IntegrationId.Plaid] as PlaidConfig) || defaultPlaidConfig
 
                 plaidConfig.name = credentials.name
@@ -74,7 +74,7 @@ export default async () => {
             })
 
             logInfo('Successfully set up Plaid Integration.')
-            return resolve()
+            return resolve(1)
         } catch (e) {
             logError('Unable to set up Plaid Integration.', e)
             return reject()

src/scripts/cli.ts

@@ -63,7 +63,7 @@ import { logError } from '../common/logging'
                 logError('Config update cancelled by user.')
             }
         }
-        updateConfig(config => config, true)
+        await updateConfig(config => config, true)
         await plaid()
         await google()
         await accountSetup()

src/scripts/fetch.ts

@@ -10,7 +10,7 @@ import { CSVExportIntegration } from '../integrations/csv-export/csvExportIntegr
 import { Transaction, TransactionRuleCondition, TransactionRule } from '../types/transaction'
 
 export default async () => {
-    const config = getConfig()
+    const config = await getConfig()
 
     // Start date to fetch transactions, default to 2 months of history
     let startDate = config.transactions.startDate

src/scripts/lambda.ts

@@ -0,0 +1,7 @@
+import transact from './fetch'
+
+export const lambdaHandler = async (event: any, context: any) => {
+
+    await transact()
+
+};

src/scripts/migrate.ts

@@ -14,10 +14,10 @@ export const getOldConfig = (): ConfigSource => {
     logError('You need to specify the --old-config-file argument.')
 }
 
-export default () => {
+export default async () => {
     try {
         const oldConfigSource = getOldConfig()
-        const oldConfigString = readConfig(oldConfigSource)
+        const oldConfigString = await readConfig(oldConfigSource)
         let oldConfig = parseConfig(oldConfigString)
 
         const deprecatedProperties = ['HOST', 'PORT', 'CATEGORY_OVERRIDES', 'DEBUG', 'CREATE_BALANCES_SHEET', 'DEBUG']