From 448bf46522f808445f06403233924b2bb11a491e Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Tue, 19 Dec 2023 17:14:02 -0500 Subject: [PATCH 01/15] Changes for RHEL9 --- .dockerignore | 2 +- .nomad/apiary.nomad | 140 +++++++------- .nomad/conf/.env.tpl | 2 +- .nomad/conf/www.conf | 2 +- .nomad/scripts/prestart.sh | 5 - .nomad/scripts/web.sh | 1 + Dockerfile | 2 +- config/app.php | 2 - config/cas.php | 6 +- config/enlightn.php | 1 - .../vocabularies/RoboJackets/accept.txt | 6 + docs/admins/deployment.rst | 174 ++++++++++++++++++ docs/index.rst | 1 + 13 files changed, 254 insertions(+), 90 deletions(-) create mode 100644 docs/admins/deployment.rst diff --git a/.dockerignore b/.dockerignore index 24f6568d2..6d8f0ff46 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,6 @@ vendor/ node_modules/ public/js/ public/css/ -storage/*.key +storage/ docs/_build/ resources/test/ diff --git a/.nomad/apiary.nomad b/.nomad/apiary.nomad index ff7f93ea2..d0e5195aa 100644 --- a/.nomad/apiary.nomad +++ b/.nomad/apiary.nomad @@ -3,16 +3,6 @@ variable "image" { description = "The image to use for running the service" } -variable "persist_resumes" { - type = bool - description = "Whether to store resumes on a host volume, or just inside the container" -} - -variable "persist_docusign" { - type = bool - description = "Whether to store resumes on a host volume, or just inside the container" -} - variable "run_background_containers" { type = bool description = "Whether to start containers for horizon and scheduled tasks, or only the web task" @@ -23,9 +13,9 @@ variable "precompressed_assets" { description = "Whether assets in the image are pre-compressed" } -variable "environment_name" { +variable "web_shutdown_delay" { type = string - description = "The name of the environment being deployed" + description = "How long to delay shutting down the web task after the allocation is stopped" } locals { @@ -85,38 +75,11 @@ job "apiary" { type = "service" group "apiary" { - volume "assets" { - type = "host" - source = "assets" - } - volume "run" { type = "host" source = "run" } - dynamic "volume" { - for_each = var.persist_resumes ? ["resumes"] : [] - - labels = ["resumes"] - - content { - type = "host" - source = "apiary_production_resumes" - } - } - - dynamic "volume" { - for_each = var.persist_docusign ? ["docusign"] : [] - - labels = ["docusign"] - - content { - type = "host" - source = "apiary_${var.environment_name}_docusign" - } - } - task "prestart" { driver = "docker" @@ -136,6 +99,17 @@ job "apiary" { "-c", trimspace(file("scripts/prestart.sh")) ] + + mount { + type = "volume" + target = "/assets/" + source = "assets" + readonly = false + + volume_options { + no_copy = true + } + } } resources { @@ -144,11 +118,6 @@ job "apiary" { memory_max = 2048 } - volume_mount { - volume = "assets" - destination = "/assets/" - } - volume_mount { volume = "run" destination = "/var/opt/nomad/run/" @@ -158,17 +127,22 @@ job "apiary" { data = trimspace(file("conf/.env.tpl")) destination = "/secrets/.env" + env = true + + change_mode = "noop" } template { data = < env('APP_DEV_URL', 'https://github.com/RoboJackets/apiary'), - 'aliases' => Facade::defaultAliases()->merge([ 'Alert' => RealRashid\SweetAlert\Facades\Alert::class, 'Cas' => Subfission\Cas\Facades\Cas::class, diff --git a/config/cas.php b/config/cas.php index eb705a853..9a2d542ef 100644 --- a/config/cas.php +++ b/config/cas.php @@ -9,7 +9,7 @@ |-------------------------------------------------------------------------- | Example: 'cas.myuniv.edu'. */ - 'cas_hostname' => env('CAS_HOSTNAME', 'cas.myuniv.edu'), + 'cas_hostname' => env('CAS_HOSTNAME', null), /* |-------------------------------------------------------------------------- @@ -19,7 +19,7 @@ | recommended for protecting against DOS attacks. If using load | balanced hosts, then separate each with a comma. */ - 'cas_real_hosts' => env('CAS_REAL_HOSTS', 'cas.myuniv.edu'), + 'cas_real_hosts' => env('CAS_REAL_HOSTS', null), /* |-------------------------------------------------------------------------- @@ -112,7 +112,7 @@ | CAS Logout URI |-------------------------------------------------------------------------- */ - 'cas_logout_url' => env('CAS_LOGOUT_URL', 'https://cas.myuniv.edu/cas/logout'), + 'cas_logout_url' => env('CAS_LOGOUT_URL', null), /* |-------------------------------------------------------------------------- diff --git a/config/enlightn.php b/config/enlightn.php index ca6ede71d..194907869 100644 --- a/config/enlightn.php +++ b/config/enlightn.php @@ -205,7 +205,6 @@ 'apimatic/core', 'apimatic/core-interfaces', 'enlightn/enlightnpro', - 'oitnetworkservices/buzzapiclient', 'phpmyadmin/sql-parser', 'mwgg/airports', ], diff --git a/docs/.styles/config/vocabularies/RoboJackets/accept.txt b/docs/.styles/config/vocabularies/RoboJackets/accept.txt index 54cf2e9d7..9fa7d0726 100644 --- a/docs/.styles/config/vocabularies/RoboJackets/accept.txt +++ b/docs/.styles/config/vocabularies/RoboJackets/accept.txt @@ -16,3 +16,9 @@ expiration JWT RSVPs Qualtrics +Ansible +Meilisearch +Dockerfile +Nginx +Hostname +jobspec diff --git a/docs/admins/deployment.rst b/docs/admins/deployment.rst new file mode 100644 index 000000000..98a0ba9fc --- /dev/null +++ b/docs/admins/deployment.rst @@ -0,0 +1,174 @@ +:og:description: Apiary is a Laravel app packed into a Docker container deployed with HashiCorp Nomad and Consul. This section describes how to deploy a new production-grade instance of Apiary from scratch. + +.. vale write-good.E-Prime = NO +.. vale Google.Passive = NO +.. vale write-good.Passive = NO + +Deployment +========== + +Apiary is a `Laravel `_ app packed into a `Docker `_ container deployed with HashiCorp `Nomad `_ and `Consul `_. +This section describes how to deploy a new production-grade instance of Apiary from scratch. + +Server setup +------------ + +Initialize a Red Hat Enterprise Linux server using the `web-app-platform `_ Ansible playbook. + +In particular: + +.. vale Google.Acronyms = NO +.. vale Google.Parens = NO + +- The server must be Internet-accessible for inbound webhooks from vendor services to work, including Square, DocuSign, and Postmark +- The server must have a valid :abbr:`TLS (Transport Layer Security)` certificate +- MySQL, Redis, and Meilisearch must be installed + +.. vale Google.Acronyms = YES +.. vale Google.Parens = YES + +Build the app +------------- + +The entire build process is encapsulated into the Dockerfile at the root of the repository. +You can build a production image using the following command. + +.. code:: shell + + docker build --secret id=composer_auth,src=auth.json . + +Note that you must provide an ``auth.json`` file for Composer to authenticate to the Laravel Nova repository. +See the `Laravel Nova installation instructions `_ for more details. + +You should also tag the image so that it can be pushed to a registry. + +Push the image to a registry +---------------------------- + +.. important:: + While this repository itself is open source, this project uses **confidential and proprietary** components which are packed into Docker images produced by this process. + Images should **never** be pushed to a public registry. + +The Ansible playbook includes a ``registry`` role to host a private `CNCF Distribution Registry `_ instance for storing images. +The steps are similar for any other private registry. + +.. code:: shell + + docker login registry.example.robojackets.net + + docker push registry.example.robojackets.net/apiary + +Note the manifest digest printed at the end of the push. + +.. Vale doesn't like Consul being capitalized here +.. vale Google.Headings = NO + +Add configuration to Consul +--------------------------- + +.. vale Google.Headings = YES + +All environment-specific configuration options are stored in the Consul Key/Value Store and retrieved by Nomad when starting containers. +Apiary requires several keys to be configured. + +Hostname +~~~~~~~~ + +The ``nginx/hostnames`` key must be a JSON map with a key of the Nomad job name and a value of the fully qualified domain name of the Apiary instance. + +For example, if the Nomad job name is ``apiary-production`` and you're using ``apiary.robojackets.net`` as the domain name, the key should look like this. + +.. code:: yaml + + { + "apiary-production": "apiary.robojackets.net", + # other key-value pairs not shown + } + +Among other uses, this mapping is used to serve 503 error pages in the event the app isn't available. + +Redis session database +~~~~~~~~~~~~~~~~~~~~~~ + +The ``redis/session_database`` key must be a JSON map with a key of the Nomad job name and a value of the Redis database number that should be used for storing sessions. + +.. vale write-good.TooWordy = NO + +.. note:: + Redis database numbers should be used for a single purpose each - the system administrator must ensure there is no overlap. + For Apiary and other Laravel apps, you must allocate three separate databases for each environment. + +.. vale write-good.TooWordy = YES + +For example, if the Nomad job name is ``apiary-production`` and you're allocating Redis database ``0`` for sessions for this app, the key should look like this. + +.. code:: yaml + + { + "apiary-production": 0, + # other key-value pairs not shown + } + +App configuration +~~~~~~~~~~~~~~~~~ + +App-level configuration can be split across two keys, as needed. ``apiary/shared`` is loaded for all environments, and ``apiary/`` can be used for environment-specific configuration. + +The format for both keys is the same: a JSON map of key-value pairs. +The maps are transformed into environment variables, which are then read into the app configuration cache. + +For a comprehensive list of options, see the ``/config/`` directory in the root of the repository. + +Create a database and user +-------------------------- + +Apiary relies on a MySQL database for its primary data store. + +You must log in to the database server as ``root`` or another administrative user, then run the commands below to initialize an empty database. + +.. code:: sql + + create user apiary_example@localhost identified by 'supersecretpassword'; + + create database apiary_example; + + grant all privileges on apiary_example.* to apiary_example@localhost; + +The selected database name, user name, and password must be loaded in the environment-specific configuration key in Consul. + +.. vale Google.WordList = NO + +No other setup is required for the database. +Tables and other necessary data are initialized when the app is deployed. + +.. vale Google.WordList = YES + +.. Vale doesn't like Nomad being capitalized here +.. vale Google.Headings = NO + +Submit the Nomad job +-------------------- + +.. vale Google.Headings = YES + +Apiary uses Nomad as a lightweight orchestrator for Docker containers. +You must install Nomad on your machine to submit the job - see the `Nomad installation instructions `_ for more details. + +Before submitting the job to Nomad, ensure that the job name is unique and includes the environment name. +The job name `can't be modified at job submit time `_, so it must be done outside of the Nomad tooling. +Also ensure the region and data center match the Ansible inventory. + +.. code:: shell + + export NOMAD_ADDR=https://nomad.example.robojackets.net + # use a bootstrap token or secret id from `nomad login` + export NOMAD_TOKEN=00000000-0000-0000-0000-000000000000 + + nomad run \ + -var=image=registry.example.robojackets.net/apiary@ + -var=run_background_containers=true \ + -var=precompressed_assets=true \ + -var=web_shutdown_delay=30s \ + apiary.nomad + +See the jobspec file for variable descriptions. diff --git a/docs/index.rst b/docs/index.rst index c70501eb0..f883260e5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,5 +26,6 @@ Apiary admins/access-overrides admins/api/index + admins/deployment admins/membership-agreements admins/permissions-roles From e88638dbbf410c531ca88cbb03a9fc22606c3159 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Tue, 19 Dec 2023 17:30:24 -0500 Subject: [PATCH 02/15] Disable broken link check now that there are deliberate broken links --- docs/.proselintrc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.proselintrc.json b/docs/.proselintrc.json index 1ce6fdbea..6d07fea2e 100644 --- a/docs/.proselintrc.json +++ b/docs/.proselintrc.json @@ -19,7 +19,7 @@ "lexical_illusions.misc": true, "lgbtq.offensive_terms": true, "lgbtq.terms": true, - "links.broken": true, + "links.broken": false, "malapropisms.misc": true, "misc.apologizing": true, "misc.back_formations": true, From 049c90d0b86c92ff68bdcfbc7c92bb6d365729b0 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Wed, 20 Dec 2023 17:22:16 -0500 Subject: [PATCH 03/15] Add instructions for backup and restore --- .../vocabularies/RoboJackets/accept.txt | 4 ++ docs/admins/backup-and-restore.rst | 66 +++++++++++++++++++ docs/index.rst | 1 + 3 files changed, 71 insertions(+) create mode 100644 docs/admins/backup-and-restore.rst diff --git a/docs/.styles/config/vocabularies/RoboJackets/accept.txt b/docs/.styles/config/vocabularies/RoboJackets/accept.txt index 9fa7d0726..e3ebbcc7a 100644 --- a/docs/.styles/config/vocabularies/RoboJackets/accept.txt +++ b/docs/.styles/config/vocabularies/RoboJackets/accept.txt @@ -22,3 +22,7 @@ Dockerfile Nginx Hostname jobspec +failure +failed +execute +executed diff --git a/docs/admins/backup-and-restore.rst b/docs/admins/backup-and-restore.rst new file mode 100644 index 000000000..7d2846619 --- /dev/null +++ b/docs/admins/backup-and-restore.rst @@ -0,0 +1,66 @@ +:og:description: As with any production system, system administrators should configure backups for Apiary to ensure the app can be recovered in the event of a system failure. + +.. vale Google.Passive = NO +.. vale Google.Will = NO +.. vale write-good.E-Prime = NO +.. vale write-good.Passive = NO + +Backup and restore +================== + +As with any production system, system administrators should configure backups for Apiary to ensure the app can be recovered in the event of a system failure. + +Apiary maintains state in two primary locations: + +- The MySQL database contains most information that's displayed in the web UI +- The ``/storage/app/`` directory contains :ref:`uploaded resumes ` and DocuSign documents for both :ref:`membership agreements ` and travel + +Both the database and the storage directory must be backed up to fully restore an environment. + +Create a backup +--------------- + +It's not necessary to stop the app during backups, but you may optionally do so to ensure consistency. + +Disk storage +~~~~~~~~~~~~ + +The ``/storage/app/`` directory should be copied to a secure location. +If running under Nomad, this directory is within a Docker volume. + +Some subdirectories can be excluded to save space: + +- ``/storage/app/nova-exports/`` contains exports generated within Nova +- ``/storage/app/dompdf`` is no longer used + +Database +~~~~~~~~ + +The MySQL database can be dumped using the ``mysqldump`` command. +This will generate a SQL script that can be executed later to reproduce the state of the database. +**Note that if you back up more than one database in a single invocation, the databases will be combined into a single script.** + +Some tables can be excluded to save space: + +- ``action_events`` is a log of actions taken within Nova +- ``attendance_exports`` is no longer used +- ``failed_jobs`` contains failed job information +- ``jobs`` is no longer used +- ``notification_templates`` is no longer used +- ``nova_notifications`` contains notifications shown within the Nova notifications panel +- ``oauth_refresh_tokens`` contains OAuth refresh tokens +- ``oauth_access_tokens`` contains OAuth access tokens +- ``recruiting_campaign_recipients`` is no longer used +- ``recruiting_campaigns`` is no longer used +- ``recruiting_responses`` is no longer used +- ``recruiting_visits`` is no longer used +- ``remote_attendance_links`` contains time-limited remote attendance links +- ``webhook_calls`` contains inbound webhook requests from vendors - Square, DocuSign, Postmark + +Restore from a backup +--------------------- + +1. You must have a running, empty instance that's configured and working. See :doc:`/admins/deployment` for instructions. +2. Stop the Nomad job, or otherwise prevent user access to the app for the duration of the restore process. +3. Restore the contents of the ``/storage/app/`` directory. +4. Run the script generated by ``mysqldump`` against the target database. **Note that if you backed up more than one database in a single invocation, the databases were combined into a single script, and you must trim the file to just the statements you want to execute.** diff --git a/docs/index.rst b/docs/index.rst index f883260e5..b16e4c6b2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ Apiary admins/access-overrides admins/api/index + admins/backup-and-restore admins/deployment admins/membership-agreements admins/permissions-roles From 6a2e00e186710001a2d2d244cb05c372a13c1c49 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Wed, 20 Dec 2023 17:23:53 -0500 Subject: [PATCH 04/15] Add instructions for restoring Meilisearch --- docs/admins/backup-and-restore.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/admins/backup-and-restore.rst b/docs/admins/backup-and-restore.rst index 7d2846619..b2b0cacf4 100644 --- a/docs/admins/backup-and-restore.rst +++ b/docs/admins/backup-and-restore.rst @@ -64,3 +64,5 @@ Restore from a backup 2. Stop the Nomad job, or otherwise prevent user access to the app for the duration of the restore process. 3. Restore the contents of the ``/storage/app/`` directory. 4. Run the script generated by ``mysqldump`` against the target database. **Note that if you backed up more than one database in a single invocation, the databases were combined into a single script, and you must trim the file to just the statements you want to execute.** +5. Start the Nomad job. +6. Run ``php artisan scout:import-all`` to index the data in Meilisearch. From de57f4a4ae9675470d77620fbdba8b3de880a69d Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Fri, 22 Dec 2023 15:47:02 -0600 Subject: [PATCH 05/15] Add docs on external services --- .../vocabularies/RoboJackets/accept.txt | 8 + docs/admins/access-overrides.rst | 2 +- docs/admins/deployment.rst | 4 + docs/admins/external-services.rst | 197 ++++++++++++++++++ docs/index.rst | 1 + 5 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 docs/admins/external-services.rst diff --git a/docs/.styles/config/vocabularies/RoboJackets/accept.txt b/docs/.styles/config/vocabularies/RoboJackets/accept.txt index e3ebbcc7a..56a74330a 100644 --- a/docs/.styles/config/vocabularies/RoboJackets/accept.txt +++ b/docs/.styles/config/vocabularies/RoboJackets/accept.txt @@ -20,9 +20,17 @@ Ansible Meilisearch Dockerfile Nginx +hostname Hostname jobspec failure failed execute executed +Apereo +CAS +OIT +USERTrust +PEM +DSN +JEDI diff --git a/docs/admins/access-overrides.rst b/docs/admins/access-overrides.rst index 27a7ebc88..e2ba40cbf 100644 --- a/docs/admins/access-overrides.rst +++ b/docs/admins/access-overrides.rst @@ -18,7 +18,7 @@ Self-service ------------ .. seealso:: - There is a :doc:`separate member-facing page for self-service overrides`. + There is a :doc:`separate member-facing page for self-service overrides `. To prevent misuse, a user must meet several criteria before they become eligible for a self-service override. A user's eligibility for a self-service override is visible on their details page in Nova, in the System Access panel. diff --git a/docs/admins/deployment.rst b/docs/admins/deployment.rst index 98a0ba9fc..8ae864b5a 100644 --- a/docs/admins/deployment.rst +++ b/docs/admins/deployment.rst @@ -49,9 +49,13 @@ Push the image to a registry While this repository itself is open source, this project uses **confidential and proprietary** components which are packed into Docker images produced by this process. Images should **never** be pushed to a public registry. +.. vale Google.Acronyms = NO + The Ansible playbook includes a ``registry`` role to host a private `CNCF Distribution Registry `_ instance for storing images. The steps are similar for any other private registry. +.. vale Google.Acronyms = YES + .. code:: shell docker login registry.example.robojackets.net diff --git a/docs/admins/external-services.rst b/docs/admins/external-services.rst new file mode 100644 index 000000000..8efd9d404 --- /dev/null +++ b/docs/admins/external-services.rst @@ -0,0 +1,197 @@ +:og:description: Apiary integrates with several external services that require configuration outside of the app itself. + +External services +================= + +Apiary integrates with several external services that require configuration outside of the app itself. + +.. vale Google.Headings = NO +.. vale write-good.E-Prime = NO +.. vale write-good.Passive = NO +.. vale Google.Passive = NO + +Central Authentication Service +------------------------------ + +.. vale Google.Acronyms = NO +.. vale Google.Parens = NO +.. vale Google.WordList = NO + +Apereo :abbr:`CAS (Central Authentication Service)` is the :abbr:`OIT (Office of Information Technology)`-hosted and managed single sign-on service that allows members to authenticate to Apiary with their usual Georgia Tech username and password. +CAS access may be requested from OIT Identity and Access Management within `ServiceNow `_. + +.. vale Google.Acronyms = YES +.. vale Google.Parens = YES +.. vale Google.WordList = YES + +The following attributes must be returned: + +- ``gtGTID`` +- ``email_primary`` +- ``givenName`` +- ``sn`` + +The production CAS service requires the following configuration within Apiary: + +- ``CAS_HOSTNAME`` must be set to ``sso.gatech.edu`` +- ``CAS_REAL_HOSTS`` must be set to ``sso.gatech.edu`` +- ``CAS_PORT`` must be set to ``443`` +- ``CAS_URI`` must be set to ``/cas`` +- ``CAS_CLIENT_SERVICE`` must be set to the fully qualified URL for the Apiary instance +- ``CAS_VALIDATION`` must be set to ``ca`` +- ``CAS_CERT`` must be set to the USERTrust root certificate file location +- ``CAS_VALIDATE_CN`` must be set to ``true`` +- ``CAS_LOGOUT_URL`` must be set to ``https://sso.gatech.edu/cas/logout`` +- ``CAS_LOGOUT_REDIRECT`` must be ``null`` +- ``CAS_ENABLE_SAML`` must be set to ``false`` +- ``CAS_VERSION`` must be set to ``3.0`` + +BuzzAPI +------- + +BuzzAPI is an OIT-hosted and managed service that allows Apiary to look up individuals based on their GTID, among other uses. +BuzzAPI access may be requested from OIT Identity and Access Management within `ServiceNow `_. + +.. vale Google.Will = NO + +The BuzzAPI service account must have access to search ``central.iam.gted.accounts``. +Apiary will use either a GTID, username, or ``gtPersonDirectoryID`` depending on which is available. + +The following attributes must be returned: + +- ``gtGTID`` +- ``mail`` +- ``sn`` +- ``givenName`` +- ``gtPrimaryGTAccountUsername`` +- ``uid`` + +BuzzAPI requires the following configuration within Apiary: + +- ``BUZZAPI_HOST`` must be set to ``api.gatech.edu`` +- ``BUZZAPI_APP_ID`` must be set to the username used to access BuzzAPI +- ``BUZZAPI_APP_PASSWORD`` must be set to the password used to access BuzzAPI + +In some cases, it may not be desirable to use a real BuzzAPI server. +You can enable and use a mock endpoint by setting the following options: + +- ``FEATURE_SANDBOX_MODE`` must be set to ``true`` +- ``BUZZAPI_HOST`` must be set to the Apiary instance's hostname +- ``BUZZAPI_APP_ID`` must be a randomly generated secret value +- ``BUZZAPI_APP_PASSWORD`` must be a randomly generated secret value + +Note that the mock endpoint uses the app's internal database to look up users, and can return real data in some cases. + +DocuSign +-------- + +Apiary uses `DocuSign Embedded Signing `_ for :doc:`membership agreements `. + +For development and testing, a `DocuSign Developer account `_ can be used. + +To set up an app within Georgia Tech's DocuSign account, contact `OIT Enterprise Apps and Data Management `_. + +DocuSign requires the following configuration within Apiary: + +.. vale Google.Parens = NO +.. vale write-good.Weasel = NO + +- ``DOCUSIGN_CLIENT_ID`` must be set to the client ID, also known as the integration key +- ``DOCUSIGN_CLIENT_SECRET`` must be set to the client secret, also known as the secret key +- ``DOCUSIGN_API_BASE_PATH`` must be set to the base path for the DocuSign API server + - For the demo environment, this value is always ``https://demo.docusign.net/restapi`` + - For Georgia Tech's production environment, this value is ``https://na3.docusign.net/restapi`` +- ``DOCUSIGN_ACCOUNT_ID`` must be set to the account where the app is registered, also known as the API account ID +- ``DOCUSIGN_IMPERSONATE_USER_ID`` must be set to the user ID that will be impersonated for sending membership agreements +- ``DOCUSIGN_PRIVATE_KEY`` must be set to the RSA private key in :abbr:`PEM (Privacy-Enhanced Mail)` format +- ``DOCUSIGN_MEMBERSHIP_AGREEMENT_MEMBER_ONLY_TEMPLATE_ID`` must be set to the template ID for membership agreements where only the member must sign +- ``DOCUSIGN_MEMBERSHIP_AGREEMENT_MEMBER_AND_GUARDIAN_TEMPLATE_ID`` must be set to the template ID for membership agreements where both the member and a parent or guardian must sign + +.. vale Google.Parens = YES +.. vale write-good.Weasel = YES + +Postmark +-------- + +Apiary sends transactional emails to remind members about mandatory tasks, as well as receipts and DocuSign acknowledgement emails. +While Laravel supports a wide variety of email service providers, RoboJackets uses `Postmark `_. + +Postmark requires the following configuration within Apiary: + +- ``MAIL_MAILER`` must be set to ``postmark`` +- ``MAIL_FROM_ADDRESS`` must be set to the ``From`` address used to send emails + - This address must be either individually verified within Postmark or under a verified domain +- ``MAIL_FROM_NAME`` will be the display name shown to email recipients +- ``POSTMARK_TOKEN`` must be set to the server API token +- ``POSTMARK_MESSAGE_STREAM_ID`` must be set to the stream ID used to send emails +- ``POSTMARK_OUTBOUND_TOKEN`` must be set to a randomly generated secret value and used as the ``X-Postmark-Token`` header for webhooks + - This enables Postmark to notify Apiary of bounces and subscription changes, which are then persisted on user records to suppress further emails. + +Webhooks should be sent to ``/api/v1/postmark/outbound`` with a custom header of ``X-Postmark-Token`` with the value matching ``POSTMARK_OUTBOUND_TOKEN``. + +Laravel Nova +------------ + +Apiary uses `Laravel Nova `_ to build the administrator-facing web interface. +Nova is commercial software, and requires a license key to be provided in the ``NOVA_LICENSE_KEY`` environment variable to remove the red :guilabel:`UNREGISTERED` text in the navigation bar. + +Sentry +------ + +Apiary uses `Sentry `_ for monitoring errors and app performance. +While not strictly required, it's helpful for the development team to receive information about all deployed instances. + +Sentry requires the following configuration within Apiary: + +.. vale Google.Parens = NO + +- ``SENTRY_LARAVEL_DSN`` must be set to the :abbr:`DSN (data source name)` for the Sentry project +- ``CSP_REPORT_URI`` must be set to the Content Security Policy report URI for the Sentry project +- ``DOCKER_IMAGE_DIGEST`` must be set to an identifier for the release version - if running in a Docker container, use the image digest + +.. vale Google.Parens = YES + +GitHub +------ + +OAuth credentials must be provided to enable linking a `GitHub `_ account within Apiary. +See the `GitHub documentation `_ for more details on registering a GitHub App. + +- ``GITHUB_CLIENT_ID`` must be set to the client ID +- ``GITHUB_CLIENT_SECRET`` must be set to the client secret + +Google +------ + +OAuth credentials must be provided to enable linking a `Google Account `_ within Apiary. +See the `Google developer documentation `_ for more details. + +- ``GOOGLE_CLIENT_ID`` must be set to the client ID +- ``GOOGLE_CLIENT_SECRET`` must be set to the client secret + +Square +------ + +Apiary uses `Square `_ for collecting payments. +See the `Square developer documentation `_ for more details on registering an app. + +Square requires the following configuration within Apiary: + +- ``SQUARE_ACCESS_TOKEN`` must be set to the access token +- ``SQUARE_LOCATION_ID`` must be set to the location where payments should be attributed +- ``SQUARE_ENVIRONMENT`` must be set to either ``production`` or ``sandbox`` +- ``SQUARE_WEBHOOK_SIGNATURE_KEY`` must be set to the webhook signature key + +Webhooks should be sent to ``/api/v1/square`` for ``payment.created`` and ``payment.updated`` events. + +Full OAuth authentication with merchant accounts isn't supported. + +JEDI +---- + +Apiary can optionally integrate with `JEDI `_ to support propagating changes within Apiary to a variety of other services. + +JEDI requires the following configuration within Apiary: + +- ``JEDI_HOST`` must be the base URL for the JEDI server +- ``JEDI_TOKEN`` must be the token to use to authenticate to JEDI diff --git a/docs/index.rst b/docs/index.rst index b16e4c6b2..492893e81 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,5 +28,6 @@ Apiary admins/api/index admins/backup-and-restore admins/deployment + admins/external-services admins/membership-agreements admins/permissions-roles From 5e32c60abac20ad8f6322d007dec7f97e8dc67f6 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Sat, 23 Dec 2023 14:46:34 -0600 Subject: [PATCH 06/15] Reformat restore instructions --- docs/admins/backup-and-restore.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/admins/backup-and-restore.rst b/docs/admins/backup-and-restore.rst index b2b0cacf4..201509207 100644 --- a/docs/admins/backup-and-restore.rst +++ b/docs/admins/backup-and-restore.rst @@ -60,9 +60,11 @@ Some tables can be excluded to save space: Restore from a backup --------------------- -1. You must have a running, empty instance that's configured and working. See :doc:`/admins/deployment` for instructions. +1. You must have a running, empty instance that's configured and working. + See :doc:`/admins/deployment` for instructions. 2. Stop the Nomad job, or otherwise prevent user access to the app for the duration of the restore process. 3. Restore the contents of the ``/storage/app/`` directory. -4. Run the script generated by ``mysqldump`` against the target database. **Note that if you backed up more than one database in a single invocation, the databases were combined into a single script, and you must trim the file to just the statements you want to execute.** +4. Run the script generated by ``mysqldump`` against the target database. + **Note that if you backed up more than one database in a single invocation, the databases were combined into a single script, and you must trim the file to just the statements you want to execute.** 5. Start the Nomad job. 6. Run ``php artisan scout:import-all`` to index the data in Meilisearch. From 56764e4b714db18e91d85b08923d0a40d8278c20 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Sat, 23 Dec 2023 14:46:39 -0600 Subject: [PATCH 07/15] Add note about running scout import in Docker --- docs/admins/backup-and-restore.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/admins/backup-and-restore.rst b/docs/admins/backup-and-restore.rst index 201509207..b313d8277 100644 --- a/docs/admins/backup-and-restore.rst +++ b/docs/admins/backup-and-restore.rst @@ -68,3 +68,4 @@ Restore from a backup **Note that if you backed up more than one database in a single invocation, the databases were combined into a single script, and you must trim the file to just the statements you want to execute.** 5. Start the Nomad job. 6. Run ``php artisan scout:import-all`` to index the data in Meilisearch. + If running in Docker, this command must be run inside of the Docker container. From 70bce7f8a0b3905f9261c6f86546d0332f258d5a Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Fri, 14 Jun 2024 17:57:47 -0400 Subject: [PATCH 08/15] Allow both uppercase and lowercase hostname --- docs/.styles/config/vocabularies/RoboJackets/accept.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/.styles/config/vocabularies/RoboJackets/accept.txt b/docs/.styles/config/vocabularies/RoboJackets/accept.txt index 56a74330a..976ef90be 100644 --- a/docs/.styles/config/vocabularies/RoboJackets/accept.txt +++ b/docs/.styles/config/vocabularies/RoboJackets/accept.txt @@ -20,8 +20,7 @@ Ansible Meilisearch Dockerfile Nginx -hostname -Hostname +(h|H)ostname jobspec failure failed From 76b0649c9a25c4deb1d0aa90b6b5853e9cc3b220 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Fri, 14 Jun 2024 18:00:17 -0400 Subject: [PATCH 09/15] Fix doc lint issues --- docs/admins/deployment.rst | 1 + docs/admins/external-services.rst | 2 ++ docs/officers/dues/data-model.rst | 3 ++- docs/officers/payments/index.rst | 2 ++ docs/officers/travel/matrix.rst | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/admins/deployment.rst b/docs/admins/deployment.rst index 8ae864b5a..5a504a7a1 100644 --- a/docs/admins/deployment.rst +++ b/docs/admins/deployment.rst @@ -71,6 +71,7 @@ Add configuration to Consul --------------------------- .. vale Google.Headings = YES +.. vale write-good.Weasel = NO All environment-specific configuration options are stored in the Consul Key/Value Store and retrieved by Nomad when starting containers. Apiary requires several keys to be configured. diff --git a/docs/admins/external-services.rst b/docs/admins/external-services.rst index 8efd9d404..32a004067 100644 --- a/docs/admins/external-services.rst +++ b/docs/admins/external-services.rst @@ -3,6 +3,8 @@ External services ================= +.. vale write-good.Weasel = NO + Apiary integrates with several external services that require configuration outside of the app itself. .. vale Google.Headings = NO diff --git a/docs/officers/dues/data-model.rst b/docs/officers/dues/data-model.rst index a95edbcec..bd320ec8a 100644 --- a/docs/officers/dues/data-model.rst +++ b/docs/officers/dues/data-model.rst @@ -3,6 +3,8 @@ Data model ========== +.. vale write-good.Weasel = NO + Apiary tracks dues using several interrelated objects. Fiscal year @@ -17,7 +19,6 @@ Dues package .. vale Google.Passive = NO .. vale write-good.E-Prime = NO .. vale write-good.Passive = NO -.. vale write-good.Weasel = NO A **dues package** represents an option for paying dues. In the member-facing interface, packages are labeled "Dues Terms." diff --git a/docs/officers/payments/index.rst b/docs/officers/payments/index.rst index 414749702..f116ee562 100644 --- a/docs/officers/payments/index.rst +++ b/docs/officers/payments/index.rst @@ -4,6 +4,8 @@ Payments ======== +.. vale write-good.Weasel = NO + Apiary supports several payment methods for both :doc:`dues ` and :doc:`trips `. .. toctree:: diff --git a/docs/officers/travel/matrix.rst b/docs/officers/travel/matrix.rst index f97de5d91..318b25cbf 100644 --- a/docs/officers/travel/matrix.rst +++ b/docs/officers/travel/matrix.rst @@ -35,7 +35,7 @@ Search for flights #. Select :guilabel:`Matrix Airfare Search`. #. A popup will appear with options to configure your search. #. After filling out the popup, click the :guilabel:`Search` button at the bottom. - This will open Matrix in a new tab with several search criteria pre-populated. + This will open Matrix in a new tab with search criteria pre-populated. You can made adjustments on this page if desired, but the results might not meet the airfare policy for your trip. #. To submit your search, click the blue :guilabel:`Search` button in the bottom right in Matrix. This will show all flights that meet your criteria. From d2d4759f84db89c01d811a55c91a463b25998bd8 Mon Sep 17 00:00:00 2001 From: Zach Slaton Date: Sat, 22 Jun 2024 14:43:12 -0400 Subject: [PATCH 10/15] Change deploy.yml to use new, temporary nomad url --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 329974d05..2a839e889 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,13 +53,13 @@ jobs: - name: Exchange GitHub JWT for Nomad token uses: RoboJackets/nomad-jwt-auth@main with: - url: https://nomad.bcdc.robojackets.net + url: https://nomad2.bcdc.robojackets.net jwtGithubAudience: https://nomad.bcdc.robojackets.net methodName: GitHub - name: Run Nomad job env: - NOMAD_ADDR: https://nomad.bcdc.robojackets.net + NOMAD_ADDR: https://nomad2.bcdc.robojackets.net working-directory: ./.nomad/ run: | nomad run -var image=registry.bcdc.robojackets.net/apiary@${{ inputs.image-digest }} -var precompressed_assets=${{ inputs.precompressed-assets }} -var-file var-files/${{ inputs.environment }}.hcl apiary.nomad From de55cff1e96344844830671f8ed19bf7f99396b1 Mon Sep 17 00:00:00 2001 From: Zach Slaton Date: Sat, 22 Jun 2024 16:00:54 -0400 Subject: [PATCH 11/15] Undo nomad2 url thing just gonna change DNS --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2a839e889..329974d05 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,13 +53,13 @@ jobs: - name: Exchange GitHub JWT for Nomad token uses: RoboJackets/nomad-jwt-auth@main with: - url: https://nomad2.bcdc.robojackets.net + url: https://nomad.bcdc.robojackets.net jwtGithubAudience: https://nomad.bcdc.robojackets.net methodName: GitHub - name: Run Nomad job env: - NOMAD_ADDR: https://nomad2.bcdc.robojackets.net + NOMAD_ADDR: https://nomad.bcdc.robojackets.net working-directory: ./.nomad/ run: | nomad run -var image=registry.bcdc.robojackets.net/apiary@${{ inputs.image-digest }} -var precompressed_assets=${{ inputs.precompressed-assets }} -var-file var-files/${{ inputs.environment }}.hcl apiary.nomad From 0dc2a8f38e7a63fd186dc666dbcfbb760a4f3b4b Mon Sep 17 00:00:00 2001 From: Zach Slaton Date: Sat, 22 Jun 2024 16:34:28 -0400 Subject: [PATCH 12/15] Rename acl binding-rule name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 329974d05..eeb78e5e7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -55,7 +55,7 @@ jobs: with: url: https://nomad.bcdc.robojackets.net jwtGithubAudience: https://nomad.bcdc.robojackets.net - methodName: GitHub + methodName: GitHubActions - name: Run Nomad job env: From 1f531f7373755f30a8dedb5935a511b957118ed5 Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Sat, 22 Jun 2024 16:49:48 -0400 Subject: [PATCH 13/15] Update Nomad var files --- .nomad/var-files/production.hcl | 4 +--- .nomad/var-files/sandbox.hcl | 4 +--- .nomad/var-files/test.hcl | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.nomad/var-files/production.hcl b/.nomad/var-files/production.hcl index 65d45188f..e687f6a4a 100644 --- a/.nomad/var-files/production.hcl +++ b/.nomad/var-files/production.hcl @@ -1,4 +1,2 @@ -persist_resumes = true -persist_docusign = true run_background_containers = true -environment_name = "production" +web_shutdown_delay = "30s" diff --git a/.nomad/var-files/sandbox.hcl b/.nomad/var-files/sandbox.hcl index b053cc6c3..1340003d9 100644 --- a/.nomad/var-files/sandbox.hcl +++ b/.nomad/var-files/sandbox.hcl @@ -1,4 +1,2 @@ -persist_resumes = false -persist_docusign = false run_background_containers = false -environment_name = "sandbox" +web_shutdown_delay = "0s" diff --git a/.nomad/var-files/test.hcl b/.nomad/var-files/test.hcl index 5f709f1cd..cecbae054 100644 --- a/.nomad/var-files/test.hcl +++ b/.nomad/var-files/test.hcl @@ -1,4 +1,2 @@ -persist_resumes = false -persist_docusign = true run_background_containers = true -environment_name = "test" +web_shutdown_delay = "0s" From 8023f0d4a1d1aaebc3a5eeb2f5c435d04ff2ccae Mon Sep 17 00:00:00 2001 From: Kristaps Berzinch Date: Sat, 22 Jun 2024 18:28:32 -0400 Subject: [PATCH 14/15] Update Meilisearch to v1.9.0-rc.3 --- .nomad/conf/.env.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.nomad/conf/.env.tpl b/.nomad/conf/.env.tpl index abc2b1ad5..70d3d67a4 100644 --- a/.nomad/conf/.env.tpl +++ b/.nomad/conf/.env.tpl @@ -18,10 +18,10 @@ REDIS_PORT="-1" REDIS_HOST="{{- index .ServiceMeta "socket" | trimSpace -}}" {{ end }} REDIS_PASSWORD="{{- key "redis/password" | trimSpace -}}" -{{- range service "meilisearch-v1-5" }} +{{- range service "meilisearch-v1-9-0-rc-3" }} MEILISEARCH_HOST="http://127.0.0.1:{{- .Port -}}" {{ end }} -MEILISEARCH_KEY="{{- key "meilisearch/admin-key-v1.5" | trimSpace -}}" +MEILISEARCH_KEY="{{- key "meilisearch/admin-key-v1.9.0-rc.3" | trimSpace -}}" SESSION_SECURE_COOKIE="true" SESSION_COOKIE="__Host-apiary_session" {{ range $key, $value := (key (printf "apiary/%s" (slice (env "NOMAD_JOB_NAME") 7)) | parseJSON) -}} From 8b6292d9df985e7ce00c16efdd7241612f1f1ffd Mon Sep 17 00:00:00 2001 From: Zach Slaton Date: Sat, 22 Jun 2024 19:06:48 -0400 Subject: [PATCH 15/15] Fix changed directory in apiary.nomad Dicated by Kristaps --- .nomad/apiary.nomad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nomad/apiary.nomad b/.nomad/apiary.nomad index d0e5195aa..a32037a8e 100644 --- a/.nomad/apiary.nomad +++ b/.nomad/apiary.nomad @@ -164,7 +164,7 @@ EOF mount { type = "bind" - source = "local/fpm/" + source = "local/" target = "/etc/php/8.3/fpm/pool.d/" }