Merge pull request #162 from CDCgov/prod-setup2

SFTP - Enable keys

.gitignore

@@ -33,7 +33,7 @@ terraform.tfstate*
 .terraform*
 
 # Local blob storage data
-/localdata
+/localdata/data
 
 
 # Items added by creating the Azure function:

.secrets.baseline

@@ -90,6 +90,10 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -122,6 +126,103 @@
       "path": "detect_secrets.filters.heuristic.is_templated_secret"
     }
   ],
-  "results": {},
-  "generated_at": "2024-05-15T17:31:59Z"
+  "results": {
+    ".github/workflows/prod-deploy.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".github/workflows/prod-deploy.yml",
+        "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
+        "is_verified": false,
+        "line_number": 12,
+        "is_secret": false
+      }
+    ],
+    "SECRETS.md": [
+      {
+        "type": "Secret Keyword",
+        "filename": "SECRETS.md",
+        "hashed_secret": "4ea048262c33345f67eb41df9f916812bf143cc4",
+        "is_verified": false,
+        "line_number": 11,
+        "is_secret": false
+      }
+    ],
+    "azure_functions/local.settings.json": [
+      {
+        "type": "Azure Storage Account access key",
+        "filename": "azure_functions/local.settings.json",
+        "hashed_secret": "5666459779d6a76bea73453137803fd27d8f79cd",
+        "is_verified": false,
+        "line_number": 7,
+        "is_secret": false
+      }
+    ],
+    "mock_credentials/ca-phl-reportstream-private-key-local": [
+      {
+        "type": "Private Key",
+        "filename": "mock_credentials/ca-phl-reportstream-private-key-local",
+        "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
+        "is_verified": false,
+        "line_number": 1,
+        "is_secret": false
+      }
+    ],
+    "mock_credentials/ca-phl-sftp-host-private-key-local": [
+      {
+        "type": "Private Key",
+        "filename": "mock_credentials/ca-phl-sftp-host-private-key-local",
+        "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
+        "is_verified": false,
+        "line_number": 1,
+        "is_secret": false
+      }
+    ],
+    "mock_credentials/ca-phl-sftp-user-credential-private-key-local": [
+      {
+        "type": "Private Key",
+        "filename": "mock_credentials/ca-phl-sftp-user-credential-private-key-local",
+        "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
+        "is_verified": false,
+        "line_number": 1,
+        "is_secret": false
+      }
+    ],
+    "src/orchestration/queue_test.go": [
+      {
+        "type": "Azure Storage Account access key",
+        "filename": "src/orchestration/queue_test.go",
+        "hashed_secret": "5666459779d6a76bea73453137803fd27d8f79cd",
+        "is_verified": false,
+        "line_number": 35,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "src/orchestration/queue_test.go",
+        "hashed_secret": "8f5ef7ea81fecd1995e8920912b87b232ab41d42",
+        "is_verified": false,
+        "line_number": 35,
+        "is_secret": false
+      }
+    ],
+    "src/senders/report_stream_sender_test.go": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/senders/report_stream_sender_test.go",
+        "hashed_secret": "493a7b3422cad32ee6eeec182085be8a776a27da",
+        "is_verified": false,
+        "line_number": 18,
+        "is_secret": false
+      },
+      {
+        "type": "JSON Web Token",
+        "filename": "src/senders/report_stream_sender_test.go",
+        "hashed_secret": "60cedfd5328355305997af2d7cb822adcd358490",
+        "is_verified": false,
+        "line_number": 96,
+        "is_secret": false
+      }
+    ]
+  },
+  "generated_at": "2024-09-16T15:04:44Z"
 }

README.md

@@ -81,7 +81,7 @@ match your newly-created file
    ```json
    {
    "topic": "/subscriptions/52203171-a2ed-4f6c-b5cf-9b368c43f15b/resourceGroups/csels-rsti-internal-moderate-rg/providers/Microsoft.Storage/storageAccounts/cdcrssftpinternal",
-   "subject": "/blobServices/default/containers/sftp/blobs/order_message.hl7",
+   "subject": "/blobServices/default/containers/sftp/blobs/import/order_message.hl7",
    "eventType": "Microsoft.Storage.BlobCreated",
    "id": "dac45448-001e-0031-7649-b8ad2c06c977",
    "data": {
@@ -92,7 +92,7 @@ match your newly-created file
    "contentType": "application/octet-stream",
    "contentLength": 1122,
    "blobType": "BlockBlob",
-   "url": "https://cdcrssftpinternal.blob.core.windows.net/sftp/order_message.hl7",
+   "url": "http://127.0.0.1:12000/devstoreaccount1/sftp/import/order_message.hl7",
    "sequencer": "00000000000000000000000000024DA1000000000006ab03",
    "storageDiagnostics": {
    "batchId": "6677b768-3006-0093-0049-b89735000000"
@@ -114,12 +114,31 @@ As of 7/3/24, when we copy a file from the local SFTP server, we try to unzip it
 files into the import folder, and if there are any errors, we upload an error file for the zip. If the original file is
 not a zip, we just copy it into the import folder.
 
-#### Manual cloud testing
-To trigger file ingestion in a deployed environment, go to the `cdcrssftp{env}` storage account in the Azure portal.
+#### Manual Cloud Testing
+
+##### Upload to Our Azure Container
+
+To trigger file ingestion in a deployed environment, go to the `cdcrssftp{env}` storage account in the Azure Portal.
 In the `sftp` container, upload a file to an `import` folder. If that folder doesn't already exist, you can create
 it by going to `Upload`, expanding `Advanced`, and putting `import` in the `Upload to folder` box!
 [upload_file.png](docs/upload_file.png)
 
+##### Upload to SFTP Server
+
+Log into CA's SFTP staging environment and drop a file into the `OUTPUT` folder.  You can either wait for one of our
+lower environments to trigger, or you can manually trigger the Azure function from the Azure Portal.
+
+The credentials and domain name for CA's SFTP environment can be found in Keybase under CA Info.
+
+To manually trigger the Azure function...
+
+1. If this is in an Azure Entra domain environment, you will need to log in as your -SU account.
+2. Go to the `polling-function-{env}` function app in the Azure Portal.
+3. Navigate to the CORS section, and add `https://portal.azure.com` as an allowed origin.  Click save.
+4. Navigate back to the Overview section, and click on the trigger function.
+5. Click on the Test/Run button, and then click on the Run button that pops-up.
+
+
 #### End-to-end Tests
 
 #### Load Testing
@@ -173,6 +192,10 @@ PR reviews before merge.
 The Production environment is the real deal.  It deploys to a CDC Azure Entra domain and subscription.  Deployments
 occur when a release is published.
 
+### Secrets
+
+See [SECRETS.md](./SECRETS.md) for a description of our secrets.
+
 ## Related documents
 
 * [Azure Functions and Typescript](/azure_functions/src/README.md)

SECRETS.md

@@ -0,0 +1,58 @@
+# Secrets
+
+Rotating secrets is tricky and tedious, the purpose of this document is to align contributors with the current state of
+secrets and ensure they continue to align as secrets are created/renamed or deleted.
+
+## Current Secrets
+
+Below are the secrets that currently exist in Azure KeyVault and what they represent. The `env` part represents the
+environment, such as `dev`, `stg`, etc.
+
+- ZIP password: `ca-phl-zip-password-env`.
+- SFTP starting directory: `ca-phl-sftp-starting-directory-env`.
+- SFTP server address: `ca-phl-sftp-server-address-env`.
+- SFTP username: `ca-phl-sftp-user-env`.
+- SFTP server host public key: `ca-phl-sftp-host-public-key-env`.
+- SFTP user credential private key: `ca-phl-sftp-user-credential-private-key-env`.
+- RS JWT signing key: `ca-phl-reportstream-private-key-env`.
+
+## Naming Convention
+
+The current naming convention for secrets is: [partner-name]-[associated-service]-[purpose]
+
+If we ever have private keys or public keys, include whether it is a private or public key in the [purpose]. See our
+existing secrets for inspiration.
+
+## Types
+
+Currently, there are two types of secrets to access our partners. Each secret associated with said service will contain
+one its name after the partner's name:
+
+- To access ReportStream: `reportstream` is included in the name.
+- To access an external SFTP site: `sftp` is included in the name.
+
+There are also two types of secrets when it comes to connecting to an SFTP server.
+
+### The User Credentials
+
+This is represented as `user-credential-private-key` in our secrets.
+
+This is a private key that the user (us) has and uses to authenticate to the SFTP server. The associated
+public key is given to the SFTP server administrator before we try to connect.
+
+### The Server's Host Key
+
+This is represented as `sftp-host-public-key` in our secrets.
+
+This is a public key that the user (us) has and uses to ensure we are connecting to the correct SFTP server. The
+associated private key is pre-created by the SFTP server administrator and installed on the SFTP server. We don't
+create the public key, but we can get the public key when we connect to the server or the SFTP server administrator can
+give it to us.
+
+## Local Secrets
+
+We put mock secrets into [mock_credentials](./mock_credentials) that are used when running the service locally. There
+are additional secrets in there than are used by our application. For example, we have a private key for our user
+credentials as one of our secrets, but we need an associated public key to be installed on the mock SFTP server for this
+to work. So, we have a public key that isn't used by our service but is used by our mock SFTP server to make the
+authentication work. A similar concept applies to the SFTP host key and could apply to other secrets.

docker-compose.yml

@@ -7,8 +7,7 @@ services:
       ENV: local
       # Uncomment the line below to call local report stream. Otherwise we'll use a mock response
       # REPORT_STREAM_URL_PREFIX: http://host.docker.internal:7071
-      FLEXION_PRIVATE_KEY_NAME: trusted-intermediary-private-key-local.pem #pragma: allowlist secret
-      FLEXION_CLIENT_NAME: flexion.simulated-hospital
+      CA_PHL_CLIENT_NAME: flexion.simulated-lab
       QUEUE_MAX_DELIVERY_ATTEMPTS: 5
       POLLING_TRIGGER_QUEUE_NAME: polling-trigger-queue
     volumes:
@@ -51,7 +50,7 @@ services:
       - -c
       - |
         az storage container create -n sftp
-        az storage blob upload --overwrite --account-name devstoreaccount1 --container-name sftp --name order_message.hl7 --file mock_data/order_message.hl7
+        az storage blob upload --overwrite --account-name devstoreaccount1 --container-name sftp --name import/order_message.hl7 --file mock_data/order_message.hl7
         az storage queue create -n message-import-queue
         az storage queue create -n message-import-dead-letter-queue
         az storage queue create -n polling-trigger-queue
@@ -96,9 +95,9 @@ services:
     environment:
       SFTP_USERS: ti_user:ti_password:::files
     volumes:
-#      - ./mock_credentials/sftp-public-key-local:/home/ti_user/.ssh/keys/id_rsa.pub:ro  # uncomment me when CA uses our public key for authentication
-      - ./mock_credentials/sftp-server-private-key-local:/etc/ssh/ssh_host_rsa_key
-#      - ./localdata/sftp_server_require_password_and_publickey.sh:/etc/sftp.d/sftp_server_require_password_and_publickey.sh  # uncomment me when CA uses our public key for authentication
+      - ./mock_credentials/ca-phl-sftp-user-credential-public-key-local:/home/ti_user/.ssh/keys/id_rsa.pub:ro
+      - ./mock_credentials/ca-phl-sftp-host-private-key-local:/etc/ssh/ssh_host_rsa_key
+      - ./localdata/sftp_server_require_publickey.sh:/etc/sftp.d/sftp_server_require_publickey.sh
       - ./localdata/data/sftp:/home/ti_user/files
     ports:
       - "2223:22"

localdata/sftp_server_require_publickey.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "AuthenticationMethods publickey" >> /etc/ssh/sshd_config

operations/template/app.tf

@@ -127,15 +127,14 @@ resource "azurerm_linux_web_app" "sftp" {
     ENV                             = var.environment
     AZURE_STORAGE_CONNECTION_STRING = azurerm_storage_account.storage.primary_blob_connection_string
     REPORT_STREAM_URL_PREFIX        = "https://${local.rs_domain_prefix}prime.cdc.gov"
-    FLEXION_PRIVATE_KEY_NAME        = azurerm_key_vault_secret.mock_public_health_lab_private_key.name
     AZURE_KEY_VAULT_URI             = azurerm_key_vault.key_storage.vault_uri
-    FLEXION_CLIENT_NAME             = "flexion.simulated-lab"
+    CA_PHL_CLIENT_NAME              = "flexion.simulated-lab"
     QUEUE_MAX_DELIVERY_ATTEMPTS     = azurerm_eventgrid_system_topic_event_subscription.topic_sub.retry_policy.0.max_delivery_attempts # making the Azure container <-> queue retry count be in sync with the queue <-> application retry count..
   }
 
   sticky_settings {
-    app_setting_names = ["AZURE_STORAGE_CONNECTION_STRING", "REPORT_STREAM_URL_PREFIX", "FLEXION_PRIVATE_KEY_NAME",
-    "AZURE_KEY_VAULT_URI", "FLEXION_CLIENT_NAME", "QUEUE_MAX_DELIVERY_ATTEMPTS"]
+    app_setting_names = ["AZURE_STORAGE_CONNECTION_STRING", "REPORT_STREAM_URL_PREFIX",
+    "AZURE_KEY_VAULT_URI", "CA_PHL_CLIENT_NAME", "QUEUE_MAX_DELIVERY_ATTEMPTS"]
   }
 
   identity {