Merge pull request #106 from quantified-uncertainty/dockerize

Dockerize and clean up

.dockerignore

@@ -0,0 +1,4 @@
+/node_modules
+/.git
+/.github
+/ops

.github/workflows/ci.yaml

@@ -0,0 +1,77 @@
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  push:
+    branches:
+      - master
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: write
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,format=long
+            type=ref,event=branch
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ops/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built.
+      # It increases supply chain security for people who consume the image.
+      # For more information, see https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+
+      - name: Commit new image tag
+        run: |
+          sed -i "s/^  tag:.*/  tag: sha-${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}/" ./ops/chart/values.yaml
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Actions"
+          git add .
+          git commit -m 'Update image tag'
+          git push origin ${{ github.ref_name }}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "typescript.tsdk": "node_modules/typescript/lib"
+}

README.md

@@ -13,7 +13,7 @@ This repository includes the source code for both the website and the library th
 ```
 $ git clone https://github.com/quantified-uncertainty/metaforecast
 $ cd metaforecast
-$ npm install
+$ pnpm install
 ```
 
 ### 2. Set up a database and environment variables
@@ -26,7 +26,7 @@ See [./docs/configuration.md](./docs/configuration.md) for details.
 
 ### 3. Actually run
 
-After installing and building (`npm run build`) the application, `npm run cli` starts a local CLI which presents the user with choices. If you would like to skip that step, use the option name instead, e.g., `npm run cli wildeford`.
+After installing and building (`pnpm run build`) the application, `pnpm run cli` starts a local CLI which presents the user with choices. If you would like to skip that step, use the option name instead, e.g., `pnpm run cli wildeford`.
 
 ![](./public/screenshot-cli.png)
 
@@ -37,10 +37,19 @@ So overall this would look like
 ```
 $ git clone https://github.com/quantified-uncertainty/metaforecast
 $ cd metaforecast
-$ npm install
-$ npm run build
-$ npm run cli
-$ npm run next-dev
+$ pnpm install
+$ pnpm run build
+$ pnpm run cli
+$ pnpm run next-dev
+```
+
+### 4. Example: download the metaforecasts database
+
+```
+$ git clone https://github.com/quantified-uncertainty/metaforecast
+$ cd metaforecast
+$ pnpm install
+$ node src/backend/manual/manualDownload.js
 ```
 
 ## Integrations

ops/Dockerfile

@@ -0,0 +1,26 @@
+# Use the official Node.js 20 image for builds
+ARG DOCKER_NODE_VERSION=20-bookworm-slim
+
+FROM node:$DOCKER_NODE_VERSION AS build
+
+WORKDIR /app
+
+# Install OpenSSL - necessary for Prisma client
+# Mount caches via https://stackoverflow.com/a/72851168
+RUN --mount=type=cache,id=apt-lists,target=/var/lib/apt/lists,sharing=locked \
+    --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=locked \
+    rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update \
+    && apt-get install -y openssl
+
+# Install pnpm globally
+RUN npm install -g pnpm
+
+COPY . .
+
+# Install dependencies
+RUN --mount=type=cache,id=pnpm-store,target=/root/.local/share/pnpm/store/v3 pnpm install --frozen-lockfile
+
+ENV NODE_ENV=production
+
+RUN pnpm build

ops/chart/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: metaforecast
+version: "0.1.0"

ops/chart/README.md

@@ -0,0 +1,9 @@
+# Helm chart for Metaforecast
+
+## Notes
+
+The `image.tag` is updated automatically by the CI GitHub Action on each commit.
+
+The secret with the name matching the `envSecret` value, containing environment variables, should be created in the target Kubernetes cluster. See `env.example` at the root of the repository for the required variables.
+
+Elasticsearch is not deployed by default; you need to set up your own instance and provide its credentials in the secret. If you want to self-host Elastic in Kubernetes, you'll need to set up [ECK](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html) (that's what we do at QURI) or some other configuration.

ops/chart/templates/jobs.yaml

@@ -0,0 +1,29 @@
+{{- range $jobname, $job := .Values.jobs }}
+---  
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ $.Release.Name }}-{{ $jobname }}
+spec:
+  concurrencyPolicy: Forbid
+  schedule: {{ $job.schedule | required "schedule is required" }}
+  timeZone: Etc/UTC
+  jobTemplate:
+    metadata:
+      labels:
+        app: {{ $.Release.Name }}
+        cronjob: {{ $jobname }}
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: cronjob
+            image: "{{ required "image.name is required" $.Values.image.name }}:{{ required "image.tag is required" $.Values.image.tag }}"
+            imagePullPolicy: Always
+            envFrom:
+            - secretRef:
+                name: {{ $.Values.envSecret }}
+            command:
+              {{- $job.command | required "command is required" | toYaml | nindent 12 }}
+{{- end }}

ops/chart/values.yaml

@@ -0,0 +1,15 @@
+image:
+  name: ghcr.io/quantified-uncertainty/metaforecast
+  # do not edit; this tag will be updated by the CI GitHub Action on each commit
+  tag: sha-0a9379814257b763f37b300dd8be22aca5dbc420
+
+# Secret; you should create this in your k8s cluster.
+envSecret: metaforecast-env
+
+jobs:
+  scheduler:
+    command: ['./node_modules/.bin/ts-node', '-T', 'src/backend/flow/doEverythingForScheduler.ts']
+    schedule: '0 3 * * *' # every day at 3am
+  frontpage:
+    command: ['./node_modules/.bin/ts-node', '-T', 'src/backend/index.ts', 'frontpage']
+    schedule: '0 6 * * *' # every day at 6am

package.json

@@ -1,10 +1,11 @@
 {
   "name": "metaforecast",
   "version": "2.0.0",
+  "private": true,
   "description": "Get forecasts from various platforms",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/QURIresearch/metaforecast.git"
+    "url": "git+https://github.com/quantified-uncertainty/metaforecast.git"
   },
   "keywords": [
     "forecasts",
@@ -13,20 +14,17 @@
   "author": "Nuño Sempere",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/QURIresearch/metaforecast/"
+    "url": "https://github.com/quantified-uncertainty/metaforecast/"
   },
-  "homepage": "https://github.com/QURIresearch/metaforecast#readme",
+  "homepage": "https://github.com/quantified-uncertainty/metaforecast#readme",
   "scripts": {
     "cli": "ts-node -T src/backend/index.ts",
     "reload": "heroku run:detached ts-node -T src/backend/flow/doEverythingForScheduler.js",
     "setCookies": "./src/backend/manual/setCookies.sh",
-    "cookies": "./src/backend/manual/setCookies.sh",
-    "next-dev": "next dev",
-    "build": "prisma generate && next build",
-    "next-start": "next start",
-    "next-export": "next export",
-    "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES",
-    "upgrade-interactive": "yarn upgrade-interactive --latest"
+    "dev": "next dev",
+    "gen": "prisma generate",
+    "build": "pnpm run gen && next build",
+    "dbshell": ". .env && psql $DIGITALOCEAN_POSTGRES"
   },
   "dependencies": {
     "@elastic/elasticsearch": "^8.13.0",
@@ -51,16 +49,13 @@
     "airtable": "^0.11.5",
     "ajv": "^8.11.0",
     "api": "^5.0.7",
-    "autoprefixer": "10.4.5",
     "axios": "^1.1.0",
     "chroma-js": "^2.4.2",
-    "critters": "^0.0.16",
     "date-fns": "^2.29.3",
     "dom-to-image": "^2.6.0",
     "dotenv": "^16.0.3",
     "fetch": "^1.1.0",
     "fs": "^0.0.1-security",
-    "fuse.js": "^6.6.2",
     "google-spreadsheet": "^3.3.0",
     "graphql": "^16.6.0",
     "graphql-request": "^5.0.0",
@@ -69,17 +64,13 @@
     "https": "^1.0.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^19.0.0",
-    "json2csv": "^5.0.7",
-    "multiselect-react-dropdown": "^2.0.25",
     "next": "^12.3.1",
     "next-plausible": "^3.6.3",
     "next-urql": "^3.3.3",
     "nprogress": "^0.2.0",
     "open": "^7.4.2",
-    "papaparse": "^5.3.2",
     "pg": "^8.8.0",
     "postcss": "^8.4.18",
-    "postcss-flexbugs-fixes": "^5.0.2",
     "postcss-preset-env": "^7.8.2",
     "prisma": "^3.15.2",
     "query-string": "^7.1.1",
@@ -117,10 +108,11 @@
     "@graphql-codegen/typed-document-node": "^2.3.5",
     "@graphql-codegen/typescript": "^2.7.5",
     "@graphql-codegen/typescript-operations": "^2.5.5",
-    "@svgr/cli": "^6.5.0",
+    "@graphql-typed-document-node/core": "^3.2.0",
     "@types/pg": "^8.6.5",
     "eslint": "^8.25.0",
     "eslint-config-next": "^12.3.1",
     "typescript": "4.9.3"
-  }
+  },
+  "packageManager": "[email protected]+sha512.b051a32c7e695833b84926d3b29b8cca57254b589f0649d899c6e9d0edb670b91ec7e2a43459bae73759bb5ce619c3266f116bf931ce22d1ef1759a7e45aa96f"
 }