From 276c1505bed233fb4a4afb15bf33713eff3846f1 Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Fri, 2 Aug 2024 18:06:18 -0400 Subject: [PATCH 1/6] enhancement: Adds runbook for deployment of the refactor platform. --- .gitignore | 10 + Cargo.lock | 4 +- Cargo.toml | 4 +- Dockerfile | 77 +++++++ README.md | 4 +- docker-compose.yaml | 51 +++++ docs/runbooks/Container-README.md | 341 ++++++++++++++++++++++++++++++ docs/runbooks/deploy.md | 299 ++++++++++++++++++++++++++ entity_api/Cargo.toml | 9 +- migration/src/setup.sql | 10 + scripts/rebuild_db_container.sh | 76 +++++++ web/Cargo.toml | 2 +- 12 files changed, 874 insertions(+), 13 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yaml create mode 100644 docs/runbooks/Container-README.md create mode 100644 docs/runbooks/deploy.md create mode 100644 migration/src/setup.sql create mode 100644 scripts/rebuild_db_container.sh diff --git a/.gitignore b/.gitignore index b3041e5..87603fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ /target dbml-error.log + +# Mac OS files +.DS_Store + +# workspace files +*.code-workspace +.vscode/ + +# environment files +.env* diff --git a/Cargo.lock b/Cargo.lock index 6f42707..e742f23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3293,9 +3293,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index efbc309..960951b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,13 @@ members = [".", "entity_api", "entity", "migration", "service", "web"] [dependencies] service = { path = "service" } -entity_api = {path = "entity_api" } +entity_api = { path = "entity_api" } web = { path = "web" } clap = { version = "4.5.20", features = ["cargo", "derive", "env"] } log = "0.4.22" simplelog = { version = "0.12.2", features = ["paris"] } -tokio = "1.40" +tokio = "1.41.0" [[bin]] name = "seed_db" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c74b814 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +# Stage 1: Build Stage +FROM rust:latest AS builder + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Install necessary packages +RUN apt-get update && apt-get install -y \ + bash \ + build-essential \ + libssl-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Install the necessary Rust target for ARM64 (Raspberry Pi 5) +RUN rustup target add aarch64-unknown-linux-gnu + +# Copy the main workspace Cargo.toml and Cargo.lock to define workspace structure +COPY Cargo.toml Cargo.lock ./ + +# Copy each module's Cargo.toml to maintain the workspace structure +COPY ./entity/Cargo.toml ./entity/Cargo.toml +COPY ./entity_api/Cargo.toml ./entity_api/Cargo.toml +COPY ./migration/Cargo.toml ./migration/Cargo.toml +COPY ./service/Cargo.toml ./service/Cargo.toml +COPY ./web/Cargo.toml ./web/Cargo.toml + +# Copy the complete source code into the container's working directory +COPY . . + +# Set the target directory to ensure binaries are placed in a known location +#ENV CARGO_TARGET_DIR=/usr/src/app/target/aarch64-unknown-linux-gnu/release + +# Build the project +RUN cargo build --release --workspace + +# logs the contents of the /usr/src/app directory to the docker build log and outputs them to the console +RUN ls -la /usr/src/app/target/release/ + +RUN file /usr/src/app/target/release/* + +# Stage 2: Runtime Stage +FROM debian:stable-slim AS runtime + +# Install necessary runtime dependencies +RUN apt-get update && apt-get install -y \ + libssl3 \ + libpq5 \ + postgresql-client-15 \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /usr/src/app + +# Copy the compiled binaries from the builder stage +COPY --from=builder /usr/src/app/target/release/refactor_platform_rs /usr/local/bin/refactor_platform_rs +COPY --from=builder /usr/src/app/target/release/migration /usr/local/bin/migration +COPY --from=builder /usr/src/app/target/release/seed_db /usr/local/bin/seed_db + +# Create a non-root user for running the application +RUN useradd -m appuser && \ + chown -R appuser:appuser /usr/src/app /usr/local/bin && \ + chmod +x /usr/local/bin/* + +# Switch to the non-root user +USER appuser + +# Expose the necessary ports +EXPOSE 4000 + +# Default command starts an interactive bash shell +# Set ENTRYPOINT to default to run the Rust binary with arguments +ENTRYPOINT ["/bin/bash", "-c", "/usr/local/bin/refactor_platform_rs -l DEBUG -i \"$SERVICE_INTERFACE\" -p \"$SERVICE_PORT\" -d \"$DATABASE_URL\""] + +# Default CMD allows overriding with custom commands +CMD ["bash"] diff --git a/README.md b/README.md index 5e78c0f..80e3dc1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ [![Build & Tests (backend)](https://github.com/Jim-Hodapp-Coaching/refactor-platform-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/Jim-Hodapp-Coaching/refactor-platform-rs/actions/workflows/ci.yml) # Refactor Coaching & Mentoring Platform -### Backend + +## Backend ## Intro @@ -90,6 +91,7 @@ DATABASE_URL=postgres://refactor:password@localhost:5432/refactor_platform sea-o ``` ### Generate a new Entity from Database + Note that to generate a new Entity using the CLI you must ignore all other tables using the `--ignore-tables` option. You must add the option for _each_ table you are ignoring. ```bash diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..283a014 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,51 @@ +services: + # Local PostgreSQL container (used for local development when needed) + postgres: + image: postgres:17 # Use PostgreSQL version 17 + container_name: postgres # Name the container "postgres" + environment: + POSTGRES_USER: ${POSTGRES_USER} # Set PostgreSQL user from environment variable + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Set PostgreSQL password from environment variable + POSTGRES_DB: ${POSTGRES_DB} # Set PostgreSQL database name from environment variable + ports: + - "${POSTGRES_PORT}:5432" # Map host port to container's PostgreSQL port + volumes: + - postgres_data:/var/lib/postgresql/data # Persist PostgreSQL data + - ./migration/src/setup.sql:/docker-entrypoint-initdb.d/0-setup.sql # Initialize database with setup.sql + - ./migration/src/refactor_platform_rs.sql:/docker-entrypoint-initdb.d/1-refactor_plaform_rs.sql # Initialize with refactor_platform_rs.sql + networks: + - backend_network # Connect to backend_network + + # Rust application that connects to either local or remote PostgreSQL + rust-app: + image: rust-backend # Use the built image + build: + context: . # Build context is current directory + dockerfile: Dockerfile # Use specified Dockerfile + target: runtime # Use runtime target + platform: linux/arm64 # Specify the platform + container_name: rust-app # Name the container "rust-app" + environment: + POSTGRES_USER: ${POSTGRES_USER} # Set PostgreSQL user from environment variable + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Set PostgreSQL password from environment variable + POSTGRES_DB: ${POSTGRES_DB} # Set PostgreSQL database name from environment variable + POSTGRES_SCHEMA: ${POSTGRES_SCHEMA} # Set PostgreSQL schema from environment variable + POSTGRES_HOST: postgres # Set PostgreSQL host to "postgres" service + POSTGRES_PORT: ${POSTGRES_PORT} # Set PostgreSQL port from environment variable + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/${POSTGRES_DB} # Configure database URL + SERVICE_PORT: ${SERVICE_PORT} # Set service port from environment variable + SERVICE_INTERFACE: ${SERVICE_INTERFACE} # Set service interface from environment variable + ports: + - "${SERVICE_PORT}:${SERVICE_PORT}" # Map host port to container's service port + depends_on: + - postgres # Ensure postgres service starts before rust-app + networks: + - backend_network # Connect to backend_network + command: ["sh", "-c", "sleep 5 && /usr/local/bin/refactor_platform_rs"] # Wait for Postgres and run the app + +networks: + backend_network: + driver: bridge # Use bridge network driver + +volumes: + postgres_data: # Define postgres_data volume diff --git a/docs/runbooks/Container-README.md b/docs/runbooks/Container-README.md new file mode 100644 index 0000000..5694175 --- /dev/null +++ b/docs/runbooks/Container-README.md @@ -0,0 +1,341 @@ +# Refactor Coaching & Mentoring Platform with Docker & Docker Compose + +*This project is a Rust-based backend/web API that connects to a PostgreSQL database. It uses Docker and Docker Compose for easy local development and deployment, and includes utilities for database management, migrations, and more. You can choose to run PostgreSQL either locally (via Docker) or remotely by configuring the environment variables.* + +## Prerequisites + +Before you begin, ensure you have the following installed: + +- [Docker](https://www.docker.com/products/docker-desktop) (version 20+) +- [Docker Compose](https://docs.docker.com/compose/install/) (version 1.29+) + +## Project Setup + +1. **Clone the repository**: + + ```bash + git clone + cd + ``` + +2. **Environment Configuration**: + + You'll need to decide whether you're connecting to a **local PostgreSQL container** (using Docker) or a **remote PostgreSQL instance** (e.g., on a different host or in the cloud). This is configured using `.env` files. + + - **For local PostgreSQL** (default, Docker-based): + Create a `.env.local` file based on the template provided below and specify `POSTGRES_HOST=postgres`. + + - **For remote PostgreSQL**: + Create a `.env.remote-db` file and set `POSTGRES_HOST` to the external IP or hostname of the remote PostgreSQL instance. + + Example `.env.local` for local development: + + ```env + POSTGRES_USER=refactor + POSTGRES_PASSWORD=password + POSTGRES_DB=refactor_platform + POSTGRES_HOST=postgres + POSTGRES_PORT=5432 + WEB_PORT=4000 + USERNAME=appuser + USER_UID=1000 + USER_GID=1000 + ``` + + Example `.env.remote-db` for remote PostgreSQL: + + ```env + POSTGRES_USER=remote_user + POSTGRES_PASSWORD=remote_password + POSTGRES_DB=remote_db + POSTGRES_HOST=remote-db-host.com + POSTGRES_PORT=5432 + WEB_PORT=8080 + USERNAME=remote_appuser + USER_UID=1001 + USER_GID=1001 + ``` + +3. **Review the `docker-compose.yaml` file**: + + The Docker Compose file is configured to use environment variables defined in your `.env` files. The PostgreSQL container can either run locally (if specified in your environment file) or you can connect to a remote instance by setting the appropriate `POSTGRES_HOST`. + + ```yaml + services: + postgres: + image: postgres:17 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "${POSTGRES_PORT}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + rust-app: + build: . + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + ports: + - "${WEB_PORT}:4000" + depends_on: + - postgres + ``` + +## Building and Running the Application + +### **1. Building the rust-backend image** + +To build the Docker image for the Rust application, run the following command: + + ```bash + docker buildx build --platform linux/amd64,linux/arm64 -t rust-backend . + ``` + +- this will build the image for both amd64 and arm64 architectures or you can choose to build for a specific architecture by specifying the `--platform` flag. +- the image will be tagged as `rust-backend` and will be used in the `docker-compose.yaml` file. + +### **2. Running the Rust Backend Application** + + **Interactive Shell Access to Functions:** When you start the container the default behavior is to enter an interactive session, both the binaries and each of the functions from rebuild_db.sh are immediately available: + + ```bash + docker run -it rns-backend:latest + ``` + +- Inside the container shell, you can directly call any function from `rebuild_db.sh` directly by name: +- You can also run any of the binaries or functions from `rebuild_db.sh` directly by name as arguments to the `docker run` command: + +```bash +docker run -it rns-backend:latest seed_db +``` + +## Building and Running the Entire Application + +To build and run the entire application, including the Rust backend and PostgreSQL database, follow these steps: + +### **Step 1: Build the Docker images** + +From the project root, run the following command to build the Docker images for both the Rust application and PostgreSQL (if running locally). + +```bash +docker-compose --env-file .env.local build +``` + +or for remote PostgreSQL: + +```bash +docker-compose --env-file .env.remote-db build +``` + +### **Step 2: Start the application** + +Once the build is complete, start the containers using Docker Compose. This will start the PostgreSQL database (if local) and the Rust application. + +For local PostgreSQL: + +```bash +docker-compose --env-file .env.local up +``` + +For remote PostgreSQL: + +```bash +docker-compose --env-file .env.remote-db up +``` + +The Rust web API should now be running on `http://localhost:8080` and PostgreSQL should be available on port `5432` (or remotely if using the `.env.remote-db` setup). + +## Using the Database Utilities + +### **Rebuild the database** + +To rebuild the database (create a new database, user, and schema): + +```bash +docker-compose run rust-app rebuild-db +``` + +This will use the default settings from your environment variables (`POSTGRES_DB`, `POSTGRES_USER`, and `POSTGRES_SCHEMA`). + +### **Seed the database** + +To seed the database with test data: + +```bash +docker-compose run rust-app seed-db +``` + +This runs a Rust service that seeds the PostgreSQL database with predefined data. + +### **Convert DBML to SQL** + +If you have a DBML file (`schema.dbml`), you can convert it into an SQL schema: + +```bash +docker-compose run -v $(pwd)/sql:/app/sql -v $(pwd)/schema.dbml:/app/schema.dbml rust-app dbml2sql +``` + +This will output the generated SQL file in the `sql` directory. + +## Stopping the Containers + +To stop the containers, you can use the following command: + +```bash +docker-compose down +``` + +If you want to remove all the containers, networks, and volumes, including the PostgreSQL data: + +```bash +docker-compose down -v +``` + +## Troubleshooting and Gotchas + +### **1. Cannot connect to PostgreSQL** + +- **Problem**: The Rust application cannot connect to the PostgreSQL container. +- **Solution**: + 1. Ensure that PostgreSQL is running by checking the container status: + + ```bash + docker-compose ps + ``` + + 2. Check the PostgreSQL logs to see if it's running correctly: + + ```bash + docker-compose logs postgres + ``` + + 3. Ensure that the `POSTGRES_HOST` environment variable in the Rust app is set to `postgres` (for local) or to the correct remote hostname. + +### **2. Web API not accessible** + +- **Problem**: The web API is not accessible on `localhost:8080`. +- **Solution**: + 1. Verify that the container is running: + + ```bash + docker-compose ps + ``` + + 2. Check if the correct port is exposed: + + ```bash + docker-compose logs rust-app + ``` + + 3. If you have changed the port (e.g., `WEB_PORT=9090`), make sure you access the web API at `http://localhost:9090`. + +### **3. Port conflicts** + +- **Problem**: The default ports `8080` for the web API or `5432` for PostgreSQL are already in use by another service. +- **Solution**: Change the ports in the `.env` file or in `docker-compose.yaml` to different, unused ports: + + ```yaml + services: + postgres: + ports: + - "5433:5432" + + rust-app: + ports: + - "9090:8080" + ``` + +### **4. Rebuild the application after changes** + +- **Problem**: You made changes to the Rust code or the Dockerfile, but they are not reflected when you restart the container. +- **Solution**: Make sure to rebuild the Docker image after code changes: + + ```bash + docker-compose build + docker-compose up + ``` + +### **5. Database persistence** + +- **Problem**: The database resets every time the containers are stopped and restarted. +- **Solution**: Docker volumes are used to persist data between container restarts. Ensure that the volume `postgres_data` is properly configured in `docker-compose.yaml`: + + ```yaml + volumes: + postgres_data: + ``` + + If the data is still being wiped, make sure you are not running `docker-compose down -v` unless you want to remove the database volume. + +## Tips for Development + +- You can bring up the containers in detached mode by running: + + ```bash + docker-compose up -d + ``` + +- To access the running containers (e.g., for debugging): + + ```bash + docker exec -it bash + ``` + + Example to access the Rust app container: + + ```bash + docker exec -it rust-app bash + ``` + +- If you want to restart only one service (e.g., `rust-app`): + + ```bash + docker-compose restart rust-app + ``` + +- To stop all containers: + + ```bash + docker-compose down + ``` + +To stop and remove all containers, networks, and volumes: + + ```bash + docker-compose down -v + ``` + +--- + +## Working with the Rust application: + +- To run the container with `rebuild_db.sh` (or any other binary) interactively, you can launch the container with a specific `CMD` override or call it directly: + + ```bash + # Run the container and execute the shell script + docker run -it --entrypoint ./rebuild_db.sh my-image-name:latest + ``` + +## Testing and Debugging in Interactive Mode + +- To debug or test interactively with all scripts and binaries in place: + + ```bash + # Run the container interactively + docker run -it --entrypoint /bin/bash my-image-name:latest + ``` + +- Inside the container, you can manually check and execute: + + ```bash + # Manually run the shell script + ./rebuild_db.sh + # Manually run other binaries + ./target/release/seed_db + ./target/release/refactor_platform_rs + ``` diff --git a/docs/runbooks/deploy.md b/docs/runbooks/deploy.md new file mode 100644 index 0000000..22c653d --- /dev/null +++ b/docs/runbooks/deploy.md @@ -0,0 +1,299 @@ +

Refactor Platform Deployment Runbook

+ +
+ Ticket Contents
+ +> **Create the automation code and Terraform code to deploy the frontend and backend Rust app to a running Ubuntu 22.04 server, using local Terraform state, try deploying it with a container and without using Docker containers or something “native”. A Container that can run on the ContainerD Container Runtime, which should include Docker Containers.** + +**Idea:** *use Ansible for performing initial setup on a server target.* + +### Definition of done + +* Able to build and start the backend and frontend applications using Docker containers and docker-compose +* All needed Ansible entities (roles, resource groupings) and scripts exist that can deploy to any target Ubuntu 22.04 instance running Ansible + +
+ +
+ Summary
+ +*This runbook provides detailed steps to deploy Rust frontend and backend applications on an Ubuntu 22.04 server using Ansible for initial setup and Terraform for automation. It covers both containerized (using Docker) and native deployment methods.* + +
+ +
+ I. Initial Server Setup with Ansible
+ +1. **Define Ansible Inventory:** Create an Ansible inventory file containing the IP address or hostname of your target Ubuntu 22.04 server. + +```ini +[ubuntu_server] +192.168.1.10 ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/id_rsa +``` + +*Explanation*: The inventory file specifies the target servers for Ansible to manage. Here, we define the target server with its IP address and SSH details. + +1. **Create Ansible Playbook:** Develop an Ansible playbook to perform the following tasks on the server: + * **Install Required Packages:** Install necessary packages like `curl`, `git`, `unzip`, `sudo`, `python3`, `python3-pip`, `jq`, `make`, `gcc`, `g++`, and others required for Rust compilation and container runtime. + * **Install Containerd:** Install and configure Containerd as the container runtime. + * **Install Docker (Optional):** Install Docker if you plan to use it alongside Containerd. + * **Install Ansible (Optional):** Install Ansible on the server for future deployments and automation. + * **Configure Firewall:** Open necessary ports for your applications (e.g., HTTP/HTTPS). + +```yaml +--- +- hosts: ubuntu_server + become: yes + tasks: + - name: Install required packages + apt: + name: + - curl + - git + - unzip + - sudo + - python3 + - python3-pip + - jq + - make + - gcc + - g++ + state: present + update_cache: yes + + - name: Install Containerd + shell: | + sudo apt-get install -y containerd + + - name: Install Docker (Optional) + shell: | + sudo apt-get install -y docker.io + + - name: Install Ansible (Optional) + apt: + name: ansible + state: present + + - name: Configure firewall + ufw: + rule: allow + port: "{{ item }}" + proto: tcp + with_items: + - 80 + - 443 +``` + +*Explanation*: The playbook installs necessary packages, container runtime, and configures the firewall. + +1. **Run Ansible Playbook:** Execute the playbook against the server. + +```bash +ansible-playbook -i inventory.ini setup.yml +``` + +*Gotcha*: Ensure Ansible is installed on your local machine and accessible via `ansible-playbook` command. + +
+ +
+ II. Frontend and Backend Application Deployment
+ +### A. Containerized Approach (Docker-Compose) + +1. **Create Dockerfiles:** Write Dockerfiles for both the frontend and backend applications, defining the necessary dependencies, build commands, and entry points. + +```Dockerfile +# Backend Dockerfile +FROM rust:latest + +WORKDIR /app +COPY . . + +RUN cargo build --release + +CMD ["./target/release/backend_app"] +``` + +```Dockerfile +# Frontend Dockerfile +FROM rust:latest + +WORKDIR /app +COPY . . + +RUN cargo build --release + +CMD ["./target/release/frontend_app"] +``` + +*Gotcha*: Ensure the Dockerfile paths and commands match your application structure. + +1. **Create Docker-Compose.yml:** Define the service dependencies and configurations for both frontend and backend applications within a docker-compose.yml file. + +```yaml +version: '3' +services: + backend: + build: ./backend + ports: + - "8000:8000" + frontend: + build: ./frontend + ports: + - "3000:3000" +``` + +*Context*: This configuration binds the container ports to the host machine. + +1. **Build Images:** Build the Docker images for both applications. + +```bash +docker-compose build +``` + +*Gotcha*: Ensure Docker is running on your local machine. + +1. **Deploy with Docker-Compose:** Use `docker-compose up -d` to start the containers on the server based on the defined docker-compose configuration. + +```bash +docker-compose up -d +``` + +*Context*: The `-d` flag runs containers in the background. + +## B. Native Approach (Without Containers) + +1. **Build Frontend and Backend Applications:** Build the frontend and backend applications using Rust's build tools (e.g., `cargo build`). + +```bash +cd backend +cargo build --release + +cd ../frontend +cargo build --release +``` + +*Gotcha*: Ensure Rust and Cargo are installed on your local machine. + +1. **Copy Compiled Binaries to Server:** Transfer the compiled binaries of the frontend and backend applications to the server. + +```bash +scp backend/target/release/backend_app ubuntu@192.168.1.10:~/backend_app +scp frontend/target/release/frontend_app ubuntu@192.168.1.10:~/frontend_app +``` + +*Context*: Use `scp` to securely copy files over SSH. + +1. **Configure and Start Applications:** Configure and start the applications on the server using systemd services or other suitable methods. + +```ini +# backend_app.service +[Unit] +Description=Backend Application + +[Service] +ExecStart=/home/ubuntu/backend_app + +[Install] +WantedBy=multi-user.target +``` + +```ini +# frontend_app.service +[Unit] +Description=Frontend Application + +[Service] +ExecStart=/home/ubuntu/frontend_app + +[Install] +WantedBy=multi-user.target +``` + +Start and enable services: + +```bash +sudo systemctl daemon-reload +sudo systemctl start backend_app +sudo systemctl enable backend_app + +sudo systemctl start frontend_app +sudo systemctl enable frontend_app +``` + +*Gotcha*: Ensure the service files are correctly placed in `/etc/systemd/system/`. + +
+ +
+ III. Terraform Deployment (Local State)
+ +1. **Create Terraform Code:** Write Terraform code to manage the deployment process, including: + * **Create Remote-Exec Provider:** Configure the remote-exec provider for running commands on the server. + * **Provision Ansible:** If not already installed, use the Terraform provider to install Ansible on the server (optional). + * **Run Ansible Playbook:** Use the remote-exec provider to execute the Ansible playbook on the server for initial setup. + * **Manage Application Files:** Transfer the Dockerfiles, docker-compose.yml, compiled binaries (for the native approach), or application configuration files to the server. + * **Run Docker-Compose (Containerized Approach):** Use the remote-exec provider to run `docker-compose up -d` command on the server to start the containers. + * **Start Applications (Native Approach):** Use the remote-exec provider to start the application processes on the server. + +```hcl +provider "local" {} + +provider "null" {} + +resource "null_resource" "setup" { + connection { + type = "ssh" + user = "ubuntu" + private_key = file("~/.ssh/id_rsa") + host = "192.168.1.10" + } + + provisioner "remote-exec" { + inline = [ + "sudo apt-get update", + "sudo apt-get install -y ansible", + "ansible-playbook -i inventory.ini setup.yml" + ] + } +} +``` + +*Context*: This config uses the `null` provider to run remote commands via SSH. + +1. **Initialize Terraform:** Initialize Terraform in the working directory. + +```bash +terraform init +``` + +*Gotcha*: Ensure you are in the correct directory containing `main.tf`. + +1. **Apply Terraform Changes:** Use `terraform apply` to apply the infrastructure and configuration changes. + +```bash +terraform apply +``` + +*Context*: Review the changes before applying by typing `yes` when prompted. + +
+ +
+ IV. Definition of Done
+ +* Both the frontend and backend applications are successfully built and started. +* The applications are accessible and function as expected. +* The Ansible playbook, Dockerfiles, and Terraform code are written and documented. +* All necessary steps are automated and can be deployed to any Ubuntu 22.04 instance running Ansible. + +
+ +
+ Additional Considerations
+ +* **Logging and Monitoring:** Implement logging and monitoring solutions like Prometheus, Grafana, or ELK stack for application performance and health checks. +* **Resilience and Elasticity:** Consider using load balancers and auto-scaling groups to handle traffic spikes and ensure high availability. + +
diff --git a/entity_api/Cargo.toml b/entity_api/Cargo.toml index 554a8fb..338bd9c 100644 --- a/entity_api/Cargo.toml +++ b/entity_api/Cargo.toml @@ -19,16 +19,11 @@ sqlx-sqlite = { version = "0.8.2" } utoipa = { version = "4.2.0", features = ["axum_extras", "uuid"] } [dependencies.sea-orm] -version = "1.1.0" # sea-orm version -features = [ - "debug-print", - "runtime-tokio-native-tls", - "sqlx-postgres" -] +version = "1.1.0" # sea-orm version +features = ["debug-print", "runtime-tokio-native-tls", "sqlx-postgres"] [dev-dependencies] tokio = { version = "1.40", features = ["full"] } [features] mock = ["sea-orm/mock"] - diff --git a/migration/src/setup.sql b/migration/src/setup.sql new file mode 100644 index 0000000..f4add38 --- /dev/null +++ b/migration/src/setup.sql @@ -0,0 +1,10 @@ +-- TODO: Is it possible to make the DB name, DB user, schema name, etc all environment variables from the container? + +CREATE SCHEMA refactor_platform AUTHORIZATION refactor; + +GRANT ALL PRIVILEGES ON DATABASE refactor TO refactor; +GRANT ALL ON SCHEMA refactor_platform TO refactor; + +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON TABLES TO refactor; +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON SEQUENCES TO refactor; +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON FUNCTIONS TO refactor; \ No newline at end of file diff --git a/scripts/rebuild_db_container.sh b/scripts/rebuild_db_container.sh new file mode 100644 index 0000000..149b2c2 --- /dev/null +++ b/scripts/rebuild_db_container.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Define default variables +DB_NAME=${1:-"refactor_platform"} +DB_USER=${2:-"refactor"} +SCHEMA_NAME=${3:-"refactor_platform"} + +echo "Using the following configuration:" +echo "Database Name: $DB_NAME" +echo "Database User: $DB_USER" +echo "Schema Name: $SCHEMA_NAME" + +# Check if postgres is installed with its client CLI +[ -f $(which postgres) ] && +[ -f $(which pg_ctl ) ] && +[ -f $(which psql) ] > /dev/null 2>&1 || +{ echo "Postgres and psql are not completely installed. Please install postgres with your package manager or Postgres.app and try again"; exit 1; } + +# if [[ -z "${PGDATA}" ]]; then +# echo 'Environment variable PGDATA unset. See `pg_ctl --help for more information.' +# fi + +# Ensure postgres is running and start postgres with homebrew if it is not running +pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB +#pg_ctl status > /dev/null 2>&1 || { echo "Starting Postgres..."; pg_ctl -w -t 15 start; } + +# Check if the postgres database exists and create it if it doesn't +POSTGRES_DB_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_database WHERE datname='postgres'") +if [ "$POSTGRES_DB_EXISTS" != "1" ]; then + echo "Creating 'postgres' database..." + createdb -U postgres postgres || { echo "Failed to create 'postgres' database"; exit 1; } +fi + +# Check if the user exists and create it if it doesn't +USER_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'") +if [ "$USER_EXISTS" != "1" ]; then + echo "Creating user $DB_USER..." + psql -U postgres -c "CREATE USER $DB_USER;" || { echo "Failed to create user $DB_USER"; exit 1; } +fi + +# Check if the database exists +DB_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'") + +# If the database exists, drop it +if [ "$DB_EXISTS" = "1" ]; then + echo "Database $DB_NAME exists. Dropping the database..." + psql $POSTGRES_URL -c "DROP DATABASE IF EXISTS $DB_NAME;" || { echo "Failed to drop database $DB_NAME"; exit 1; } +fi + +echo "Creating the database $DB_NAME..." +psql $POSTGRES_URL -c "CREATE DATABASE $DB_NAME;" || { echo "Failed to create database $DB_NAME"; exit 1; } + +echo "Granting privileges to $DB_USER on $DB_NAME..." +psql $POSTGRES_URL -c "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;" + +# Check if the schema exists +SCHEMA_EXISTS=$(psql $POSTGRES_URL -d $DB_NAME -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name = '$SCHEMA_NAME'") + +# If the schema does not exist, create it +if [ "$SCHEMA_EXISTS" != "1" ]; then + echo "Creating schema $SCHEMA_NAME..." + psql $POSTGRES_URL -d $DB_NAME -c "CREATE SCHEMA $SCHEMA_NAME;" || { echo "Failed to create schema $SCHEMA_NAME"; exit 1; } +fi + +# Generating SQL for the migrations using dbml2sql +echo "Generating SQL for the migrations..." +dbml2sql docs/db/refactor_platform_rs.dbml -o migration/src/refactor_platform_rs.sql || { echo "Error generating SQL file"; exit 1; } + +# Remove the line to create a schema from the generated SQL file +echo "Modifying the generated SQL file..." +sed -i '' '/CREATE SCHEMA/d' migration/src/refactor_platform_rs.sql + +echo "Running the migrations..." +DATABASE_URL=$POSTGRES_URL sea-orm-cli migrate up -s $SCHEMA_NAME || { echo "Failed to run migrations"; exit 1; } + +echo "Database setup and migrations completed successfully" \ No newline at end of file diff --git a/web/Cargo.toml b/web/Cargo.toml index b376c6d..814c7aa 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -33,7 +33,7 @@ features = [ "runtime-tokio-native-tls", "sea-orm-internal", "sqlx-postgres", - "with-uuid" + "with-uuid", ] [features] From dd42512a52bab5e1b8909df5b3aacb501caedc70 Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Sun, 24 Nov 2024 13:23:07 -0500 Subject: [PATCH 2/6] Adds updated and current Container Readme --- Dockerfile | 3 - docker-compose.yaml | 4 +- docs/runbooks/Container-README.md | 401 ++++++++++++------------------ 3 files changed, 165 insertions(+), 243 deletions(-) diff --git a/Dockerfile b/Dockerfile index c74b814..6af9000 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,9 +28,6 @@ COPY ./web/Cargo.toml ./web/Cargo.toml # Copy the complete source code into the container's working directory COPY . . -# Set the target directory to ensure binaries are placed in a known location -#ENV CARGO_TARGET_DIR=/usr/src/app/target/aarch64-unknown-linux-gnu/release - # Build the project RUN cargo build --release --workspace diff --git a/docker-compose.yaml b/docker-compose.yaml index 283a014..e1a400c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,8 +23,8 @@ services: context: . # Build context is current directory dockerfile: Dockerfile # Use specified Dockerfile target: runtime # Use runtime target - platform: linux/arm64 # Specify the platform - container_name: rust-app # Name the container "rust-app" + platform: ${PLATFORM} # Specify the platform + container_name: ${CONTAINER_NAME} # Name the container, default is "rust-app" environment: POSTGRES_USER: ${POSTGRES_USER} # Set PostgreSQL user from environment variable POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Set PostgreSQL password from environment variable diff --git a/docs/runbooks/Container-README.md b/docs/runbooks/Container-README.md index 5694175..902ee5e 100644 --- a/docs/runbooks/Container-README.md +++ b/docs/runbooks/Container-README.md @@ -2,6 +2,8 @@ *This project is a Rust-based backend/web API that connects to a PostgreSQL database. It uses Docker and Docker Compose for easy local development and deployment, and includes utilities for database management, migrations, and more. You can choose to run PostgreSQL either locally (via Docker) or remotely by configuring the environment variables.* +--- + ## Prerequisites Before you begin, ensure you have the following installed: @@ -9,333 +11,256 @@ Before you begin, ensure you have the following installed: - [Docker](https://www.docker.com/products/docker-desktop) (version 20+) - [Docker Compose](https://docs.docker.com/compose/install/) (version 1.29+) -## Project Setup - -1. **Clone the repository**: - - ```bash - git clone - cd - ``` - -2. **Environment Configuration**: - - You'll need to decide whether you're connecting to a **local PostgreSQL container** (using Docker) or a **remote PostgreSQL instance** (e.g., on a different host or in the cloud). This is configured using `.env` files. - - - **For local PostgreSQL** (default, Docker-based): - Create a `.env.local` file based on the template provided below and specify `POSTGRES_HOST=postgres`. - - - **For remote PostgreSQL**: - Create a `.env.remote-db` file and set `POSTGRES_HOST` to the external IP or hostname of the remote PostgreSQL instance. - - Example `.env.local` for local development: - - ```env - POSTGRES_USER=refactor - POSTGRES_PASSWORD=password - POSTGRES_DB=refactor_platform - POSTGRES_HOST=postgres - POSTGRES_PORT=5432 - WEB_PORT=4000 - USERNAME=appuser - USER_UID=1000 - USER_GID=1000 - ``` - - Example `.env.remote-db` for remote PostgreSQL: - - ```env - POSTGRES_USER=remote_user - POSTGRES_PASSWORD=remote_password - POSTGRES_DB=remote_db - POSTGRES_HOST=remote-db-host.com - POSTGRES_PORT=5432 - WEB_PORT=8080 - USERNAME=remote_appuser - USER_UID=1001 - USER_GID=1001 - ``` - -3. **Review the `docker-compose.yaml` file**: - - The Docker Compose file is configured to use environment variables defined in your `.env` files. The PostgreSQL container can either run locally (if specified in your environment file) or you can connect to a remote instance by setting the appropriate `POSTGRES_HOST`. - - ```yaml - services: - postgres: - image: postgres:17 - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - ports: - - "${POSTGRES_PORT}:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - rust-app: - build: . - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - ports: - - "${WEB_PORT}:4000" - depends_on: - - postgres - ``` - -## Building and Running the Application - -### **1. Building the rust-backend image** - -To build the Docker image for the Rust application, run the following command: - - ```bash - docker buildx build --platform linux/amd64,linux/arm64 -t rust-backend . - ``` - -- this will build the image for both amd64 and arm64 architectures or you can choose to build for a specific architecture by specifying the `--platform` flag. -- the image will be tagged as `rust-backend` and will be used in the `docker-compose.yaml` file. - -### **2. Running the Rust Backend Application** - - **Interactive Shell Access to Functions:** When you start the container the default behavior is to enter an interactive session, both the binaries and each of the functions from rebuild_db.sh are immediately available: +--- - ```bash - docker run -it rns-backend:latest - ``` +## Project Setup -- Inside the container shell, you can directly call any function from `rebuild_db.sh` directly by name: -- You can also run any of the binaries or functions from `rebuild_db.sh` directly by name as arguments to the `docker run` command: +### 1. **Clone the Repository** ```bash -docker run -it rns-backend:latest seed_db +git clone +cd ``` -## Building and Running the Entire Application +### 2. **Environment Configuration** + +Decide whether you're connecting to a **local PostgreSQL container** (using Docker) or a **remote PostgreSQL instance**. Configure this using `.env` files. + +#### **For Local PostgreSQL (Docker-based)**: +- Create a `.env.local` file based on the template below and specify `POSTGRES_HOST=postgres`. + +#### **For Remote PostgreSQL**: +- Create a `.env.remote-db` file and set `POSTGRES_HOST` to the external IP or hostname of the remote PostgreSQL instance. + +Example `.env.local`: +```env +POSTGRES_USER=refactor +POSTGRES_PASSWORD=password +POSTGRES_DB=refactor +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_SCHEMA=refactor_platform +DATABASE_URL=postgres://refactor:password@postgres:5432/refactor_platform +SERVICE_PORT=4000 +SERVICE_INTERFACE=0.0.0.0 +USERNAME=appuser +USER_UID=1000 +USER_GID=1000 +PLATFORM=linux/arm64 +``` -To build and run the entire application, including the Rust backend and PostgreSQL database, follow these steps: +Example `.env.remote-db`: +```env +POSTGRES_USER=remote_refactor +POSTGRES_PASSWORD=remote_password +POSTGRES_DB=refactor +POSTGRES_HOST=postgres.example.com +POSTGRES_SCHEMA=refactor_platform +DATABASE_URL=postgres://remote_refactor:remote_password@postgres.example.com:5432/refactor_platform +POSTGRES_PORT=5432 +SERVICE_PORT=4000 +SERVICE_INTERFACE=0.0.0.0 +USERNAME=remote_appuser +USER_UID=1001 +USER_GID=1001 +PLATFORM=linux/arm64 +``` -### **Step 1: Build the Docker images** +### 3. **Review `docker-compose.yaml`** + +The `docker-compose.yaml` file uses environment variables defined in your `.env` files. + +```yaml +services: + postgres: + image: postgres:17 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "${POSTGRES_PORT}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + rust-app: + build: + context: . + dockerfile: Dockerfile + target: runtime + platform: ${PLATFORM} + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PORT: ${POSTGRES_PORT} + DATABASE_URL: ${POSTGRES_HOST}://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + SERVICE_PORT: ${SERVICE_PORT} + ports: + - "${SERVIE_PORT}:4000" + depends_on: + - postgres +volumes: + postgres_data: +``` -From the project root, run the following command to build the Docker images for both the Rust application and PostgreSQL (if running locally). +--- -```bash -docker-compose --env-file .env.local build -``` +## Building and Running the Application -or for remote PostgreSQL: +### **1. Build the Rust Backend Image** ```bash -docker-compose --env-file .env.remote-db build +docker buildx build --platform linux/amd64,linux/arm64 -t rust-backend . ``` -### **Step 2: Start the application** - -Once the build is complete, start the containers using Docker Compose. This will start the PostgreSQL database (if local) and the Rust application. +This builds the image for both `amd64` and `arm64` architectures. Use the `--platform` flag to build for a specific architecture. -For local PostgreSQL: +### **2. Build and Run with Docker Compose** +#### For Local PostgreSQL: ```bash -docker-compose --env-file .env.local up +docker-compose --env-file .env.local up --build ``` -For remote PostgreSQL: - +#### For Remote PostgreSQL: ```bash -docker-compose --env-file .env.remote-db up +docker-compose --env-file .env.remote-db up --build ``` -The Rust web API should now be running on `http://localhost:8080` and PostgreSQL should be available on port `5432` (or remotely if using the `.env.remote-db` setup). +The web API will be accessible at `http://localhost:` . -## Using the Database Utilities +--- -### **Rebuild the database** +## Database Utilities -To rebuild the database (create a new database, user, and schema): +### **Rebuild the Database** ```bash docker-compose run rust-app rebuild-db ``` -This will use the default settings from your environment variables (`POSTGRES_DB`, `POSTGRES_USER`, and `POSTGRES_SCHEMA`). - -### **Seed the database** - -To seed the database with test data: +### **Seed the Database** ```bash docker-compose run rust-app seed-db ``` -This runs a Rust service that seeds the PostgreSQL database with predefined data. - ### **Convert DBML to SQL** -If you have a DBML file (`schema.dbml`), you can convert it into an SQL schema: - +If you have a DBML file (`schema.dbml`), convert it to SQL: ```bash docker-compose run -v $(pwd)/sql:/app/sql -v $(pwd)/schema.dbml:/app/schema.dbml rust-app dbml2sql ``` -This will output the generated SQL file in the `sql` directory. +--- -## Stopping the Containers +## Managing Containers -To stop the containers, you can use the following command: +### **Stop Containers** ```bash docker-compose down ``` -If you want to remove all the containers, networks, and volumes, including the PostgreSQL data: +### **Remove Containers, Networks, and Volumes** ```bash docker-compose down -v ``` -## Troubleshooting and Gotchas - -### **1. Cannot connect to PostgreSQL** - -- **Problem**: The Rust application cannot connect to the PostgreSQL container. -- **Solution**: - 1. Ensure that PostgreSQL is running by checking the container status: - - ```bash - docker-compose ps - ``` - - 2. Check the PostgreSQL logs to see if it's running correctly: - - ```bash - docker-compose logs postgres - ``` - - 3. Ensure that the `POSTGRES_HOST` environment variable in the Rust app is set to `postgres` (for local) or to the correct remote hostname. - -### **2. Web API not accessible** - -- **Problem**: The web API is not accessible on `localhost:8080`. -- **Solution**: - 1. Verify that the container is running: +--- - ```bash - docker-compose ps - ``` +## Troubleshooting - 2. Check if the correct port is exposed: +### **Cannot Connect to PostgreSQL** - ```bash - docker-compose logs rust-app - ``` +1. Verify PostgreSQL is running: + ```bash + docker-compose ps + ``` - 3. If you have changed the port (e.g., `WEB_PORT=9090`), make sure you access the web API at `http://localhost:9090`. +2. Check logs for PostgreSQL: + ```bash + docker-compose logs postgres + ``` -### **3. Port conflicts** +### **Web API Not Accessible** -- **Problem**: The default ports `8080` for the web API or `5432` for PostgreSQL are already in use by another service. -- **Solution**: Change the ports in the `.env` file or in `docker-compose.yaml` to different, unused ports: +1. Verify the container is running: + ```bash + docker-compose ps + ``` - ```yaml - services: - postgres: - ports: - - "5433:5432" +2. Check logs for the Rust app: + ```bash + docker-compose logs rust-app + ``` - rust-app: - ports: - - "9090:8080" - ``` +3. Confirm the correct port in `.env`: + ```bash + SERVICE_PORT=4000 + ``` -### **4. Rebuild the application after changes** +### **Port Conflicts** -- **Problem**: You made changes to the Rust code or the Dockerfile, but they are not reflected when you restart the container. -- **Solution**: Make sure to rebuild the Docker image after code changes: +Change the ports in `.env` or `docker-compose.yaml`: +```yaml +services: + postgres: + ports: + - "5433:5432" - ```bash - docker-compose build - docker-compose up - ``` + rust-app: + ports: + - "9090:8080" +``` -### **5. Database persistence** +### **Rebuild After Changes** -- **Problem**: The database resets every time the containers are stopped and restarted. -- **Solution**: Docker volumes are used to persist data between container restarts. Ensure that the volume `postgres_data` is properly configured in `docker-compose.yaml`: +```bash +docker-compose build +docker-compose up +``` - ```yaml - volumes: - postgres_data: - ``` +### **Database Persistence** - If the data is still being wiped, make sure you are not running `docker-compose down -v` unless you want to remove the database volume. +Ensure volumes are configured in `docker-compose.yaml`: +```yaml +volumes: + postgres_data: +``` -## Tips for Development +--- -- You can bring up the containers in detached mode by running: +## Development Tips +- Run containers in detached mode: ```bash docker-compose up -d ``` -- To access the running containers (e.g., for debugging): - +- Access a running container: ```bash docker exec -it bash ``` - Example to access the Rust app container: - - ```bash - docker exec -it rust-app bash - ``` - -- If you want to restart only one service (e.g., `rust-app`): - +- Restart a single service: ```bash docker-compose restart rust-app ``` -- To stop all containers: - - ```bash - docker-compose down - ``` - -To stop and remove all containers, networks, and volumes: - - ```bash - docker-compose down -v - ``` - --- -## Working with the Rust application: - -- To run the container with `rebuild_db.sh` (or any other binary) interactively, you can launch the container with a specific `CMD` override or call it directly: - - ```bash - # Run the container and execute the shell script - docker run -it --entrypoint ./rebuild_db.sh my-image-name:latest - ``` - -## Testing and Debugging in Interactive Mode - -- To debug or test interactively with all scripts and binaries in place: +## Interactive Testing +- Test interactively: ```bash - # Run the container interactively - docker run -it --entrypoint /bin/bash my-image-name:latest + docker run -it rust-backend:latest ``` -- Inside the container, you can manually check and execute: - +- Debug inside the container: ```bash - # Manually run the shell script - ./rebuild_db.sh - # Manually run other binaries - ./target/release/seed_db - ./target/release/refactor_platform_rs + docker run -it --entrypoint /bin/bash rust-backend:latest ``` From 1dc4588f51a8ca078f3a39614f4204fcf52c6ab3 Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Tue, 26 Nov 2024 14:41:10 -0500 Subject: [PATCH 3/6] Addressing pr comments --- docs/runbooks/Container-README.md | 37 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/runbooks/Container-README.md b/docs/runbooks/Container-README.md index 902ee5e..a87cfee 100644 --- a/docs/runbooks/Container-README.md +++ b/docs/runbooks/Container-README.md @@ -26,13 +26,16 @@ cd Decide whether you're connecting to a **local PostgreSQL container** (using Docker) or a **remote PostgreSQL instance**. Configure this using `.env` files. -#### **For Local PostgreSQL (Docker-based)**: +#### **For Local PostgreSQL (Docker-based)** + - Create a `.env.local` file based on the template below and specify `POSTGRES_HOST=postgres`. -#### **For Remote PostgreSQL**: +#### **For Remote PostgreSQL** + - Create a `.env.remote-db` file and set `POSTGRES_HOST` to the external IP or hostname of the remote PostgreSQL instance. Example `.env.local`: + ```env POSTGRES_USER=refactor POSTGRES_PASSWORD=password @@ -40,7 +43,7 @@ POSTGRES_DB=refactor POSTGRES_HOST=postgres POSTGRES_PORT=5432 POSTGRES_SCHEMA=refactor_platform -DATABASE_URL=postgres://refactor:password@postgres:5432/refactor_platform +DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB SERVICE_PORT=4000 SERVICE_INTERFACE=0.0.0.0 USERNAME=appuser @@ -49,14 +52,15 @@ USER_GID=1000 PLATFORM=linux/arm64 ``` -Example `.env.remote-db`: +**Example** `.env.remote-db`: + ```env POSTGRES_USER=remote_refactor POSTGRES_PASSWORD=remote_password POSTGRES_DB=refactor POSTGRES_HOST=postgres.example.com POSTGRES_SCHEMA=refactor_platform -DATABASE_URL=postgres://remote_refactor:remote_password@postgres.example.com:5432/refactor_platform +DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB POSTGRES_PORT=5432 SERVICE_PORT=4000 SERVICE_INTERFACE=0.0.0.0 @@ -98,7 +102,7 @@ services: DATABASE_URL: ${POSTGRES_HOST}://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} SERVICE_PORT: ${SERVICE_PORT} ports: - - "${SERVIE_PORT}:4000" + - "${SERVICE_PORT}:4000" depends_on: - postgres volumes: @@ -120,16 +124,18 @@ This builds the image for both `amd64` and `arm64` architectures. Use the `--pla ### **2. Build and Run with Docker Compose** #### For Local PostgreSQL: + ```bash docker-compose --env-file .env.local up --build ``` #### For Remote PostgreSQL: + ```bash docker-compose --env-file .env.remote-db up --build ``` -The web API will be accessible at `http://localhost:` . +The web API will be accessible at `http://localhost:` --- @@ -150,6 +156,11 @@ docker-compose run rust-app seed-db ### **Convert DBML to SQL** If you have a DBML file (`schema.dbml`), convert it to SQL: + +```bash +docker-compose run -v $(pwd)/sql:/app/sql -v $(pwd)/schema.dbml:/app/schema.dbml rust-app dbml2sql +``` + ```bash docker-compose run -v $(pwd)/sql:/app/sql -v $(pwd)/schema.dbml:/app/schema.dbml rust-app dbml2sql ``` @@ -177,11 +188,13 @@ docker-compose down -v ### **Cannot Connect to PostgreSQL** 1. Verify PostgreSQL is running: + ```bash docker-compose ps ``` 2. Check logs for PostgreSQL: + ```bash docker-compose logs postgres ``` @@ -189,16 +202,19 @@ docker-compose down -v ### **Web API Not Accessible** 1. Verify the container is running: + ```bash docker-compose ps ``` 2. Check logs for the Rust app: + ```bash docker-compose logs rust-app ``` 3. Confirm the correct port in `.env`: + ```bash SERVICE_PORT=4000 ``` @@ -206,6 +222,7 @@ docker-compose down -v ### **Port Conflicts** Change the ports in `.env` or `docker-compose.yaml`: + ```yaml services: postgres: @@ -227,6 +244,7 @@ docker-compose up ### **Database Persistence** Ensure volumes are configured in `docker-compose.yaml`: + ```yaml volumes: postgres_data: @@ -237,16 +255,19 @@ volumes: ## Development Tips - Run containers in detached mode: + ```bash docker-compose up -d ``` - Access a running container: + ```bash docker exec -it bash ``` - Restart a single service: + ```bash docker-compose restart rust-app ``` @@ -256,11 +277,13 @@ volumes: ## Interactive Testing - Test interactively: + ```bash docker run -it rust-backend:latest ``` - Debug inside the container: + ```bash docker run -it --entrypoint /bin/bash rust-backend:latest ``` From 9b37055d8a4c265777d2f71aeeceaaa02fc61a2f Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Tue, 26 Nov 2024 15:13:54 -0500 Subject: [PATCH 4/6] docs: Update README.md with Docker setup and quickstart instructions for the Coaching & Mentoring Platform --- README.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/README.md b/README.md index 80e3dc1..cb3042d 100644 --- a/README.md +++ b/README.md @@ -125,3 +125,68 @@ This will start the backend with log level DEBUG and attempt to connect to a Pos `src` - contains a main function that initializes logging and calls all sub-services `web` - API endpoint definition, routing, handling of request/responses, controllers + +--- + +## Refactor Coaching & Mentoring Platform with Docker & Docker Compose + +_This Rust-based backend/web API connects to a PostgreSQL database. It uses Docker and Docker Compose for local development and deployment, including utilities for database management and migrations. You can run PostgreSQL locally (via Docker) or remotely by configuring environment variables._ + +--- + +### Quickstart + +1. **Install Prerequisites**: + - [Docker](https://www.docker.com/products/docker-desktop) (20+) + - [Docker Compose](https://docs.docker.com/compose/install/) (1.29+) + +2. **Clone the Repository**: + + ```bash + git clone + cd + ``` + +3. **Set Environment Variables**: + - For **local PostgreSQL**, create a `.env.local` file and set `POSTGRES_HOST=postgres`. + - For **remote PostgreSQL**, use a `.env.remote-db` file with `POSTGRES_HOST` pointing to the external database. + +4. **Build and Start the Platform**: + - Local PostgreSQL: + + ```bash + docker-compose --env-file .env.local up --build + ``` + + - Remote PostgreSQL: + + ```bash + docker-compose --env-file .env.remote-db up --build + ``` + +5. **Access the API**: + - Visit `http://localhost:` in your browser or API client. + +### Key Commands + +- **Stop all containers**: + + ```bash + docker-compose down + ``` + + **Note**: This will stop all containers, including the database. + +- **Rebuild and restart**: + + ```bash + docker-compose up --build + ``` + +- **View logs**: + + ```bash + docker-compose logs + ``` + +_For additional commands, database utilities, and debugging tips, check the [full README](./runbooks/Container-README.md)._ From f1e9e57246040749b6a4da35116a3df45830688e Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Tue, 26 Nov 2024 15:25:24 -0500 Subject: [PATCH 5/6] docs: Revise README.md to update project structure and enhance container setup section --- README.md | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index cb3042d..f34ccf4 100644 --- a/README.md +++ b/README.md @@ -108,27 +108,9 @@ cargo run -- -l DEBUG -d postgres://refactor:password@localhost:5432/refactor_p This will start the backend with log level DEBUG and attempt to connect to a Postgres DB server on the same machine with user `refactor` and password `password` on port `5432` and selecting the database named `refactor_platform`. -## Project Directory Structure - -`docs` - project documentation including architectural records, DB schema, API docs, etc - -`entity_api` - data operations on the various `Entity` models - -`entity` - shape of the data models and the relationships to each other - -`migration` - relational DB SQL migrations - -`scripts` - contains handy developer-related scripts that make working with this codebase more straightforward - -`service` - CLI flags, environment variables, config handling and backend daemon setup - -`src` - contains a main function that initializes logging and calls all sub-services - -`web` - API endpoint definition, routing, handling of request/responses, controllers - --- -## Refactor Coaching & Mentoring Platform with Docker & Docker Compose +## Basic Container DB Setup and Management _This Rust-based backend/web API connects to a PostgreSQL database. It uses Docker and Docker Compose for local development and deployment, including utilities for database management and migrations. You can run PostgreSQL locally (via Docker) or remotely by configuring environment variables._ @@ -189,4 +171,24 @@ _This Rust-based backend/web API connects to a PostgreSQL database. It uses Dock docker-compose logs ``` -_For additional commands, database utilities, and debugging tips, check the [full README](./runbooks/Container-README.md)._ +_For additional commands, database utilities, and debugging tips, check the [Container README](runbooks/Container-README.md)._ + +--- + +## Project Directory Structure + +`docs` - project documentation including architectural records, DB schema, API docs, etc + +`entity_api` - data operations on the various `Entity` models + +`entity` - shape of the data models and the relationships to each other + +`migration` - relational DB SQL migrations + +`scripts` - contains handy developer-related scripts that make working with this codebase more straightforward + +`service` - CLI flags, environment variables, config handling and backend daemon setup + +`src` - contains a main function that initializes logging and calls all sub-services + +`web` - API endpoint definition, routing, handling of request/responses, controllers From 105b34d100ed19c2ca95aeca6a9fe8cdfead77ab Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Tue, 26 Nov 2024 15:26:43 -0500 Subject: [PATCH 6/6] docs: Update link to Container README in README.md for improved navigation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f34ccf4..a48006e 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ _This Rust-based backend/web API connects to a PostgreSQL database. It uses Dock docker-compose logs ``` -_For additional commands, database utilities, and debugging tips, check the [Container README](runbooks/Container-README.md)._ +_For additional commands, database utilities, and debugging tips, check the [Container README](docs/runbooks/Container-README.md)._ ---