From 276c1505bed233fb4a4afb15bf33713eff3846f1 Mon Sep 17 00:00:00 2001 From: Levi McDonough Date: Fri, 2 Aug 2024 18:06:18 -0400 Subject: [PATCH] enhancement: Adds runbook for deployment of the refactor platform. --- .gitignore | 10 + Cargo.lock | 4 +- Cargo.toml | 4 +- Dockerfile | 77 +++++++ README.md | 4 +- docker-compose.yaml | 51 +++++ docs/runbooks/Container-README.md | 341 ++++++++++++++++++++++++++++++ docs/runbooks/deploy.md | 299 ++++++++++++++++++++++++++ entity_api/Cargo.toml | 9 +- migration/src/setup.sql | 10 + scripts/rebuild_db_container.sh | 76 +++++++ web/Cargo.toml | 2 +- 12 files changed, 874 insertions(+), 13 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yaml create mode 100644 docs/runbooks/Container-README.md create mode 100644 docs/runbooks/deploy.md create mode 100644 migration/src/setup.sql create mode 100644 scripts/rebuild_db_container.sh diff --git a/.gitignore b/.gitignore index b3041e5..87603fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ /target dbml-error.log + +# Mac OS files +.DS_Store + +# workspace files +*.code-workspace +.vscode/ + +# environment files +.env* diff --git a/Cargo.lock b/Cargo.lock index 6f42707..e742f23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3293,9 +3293,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index efbc309..960951b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,13 @@ members = [".", "entity_api", "entity", "migration", "service", "web"] [dependencies] service = { path = "service" } -entity_api = {path = "entity_api" } +entity_api = { path = "entity_api" } web = { path = "web" } clap = { version = "4.5.20", features = ["cargo", "derive", "env"] } log = "0.4.22" simplelog = { version = "0.12.2", features = ["paris"] } -tokio = "1.40" +tokio = "1.41.0" [[bin]] name = "seed_db" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c74b814 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +# Stage 1: Build Stage +FROM rust:latest AS builder + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Install necessary packages +RUN apt-get update && apt-get install -y \ + bash \ + build-essential \ + libssl-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Install the necessary Rust target for ARM64 (Raspberry Pi 5) +RUN rustup target add aarch64-unknown-linux-gnu + +# Copy the main workspace Cargo.toml and Cargo.lock to define workspace structure +COPY Cargo.toml Cargo.lock ./ + +# Copy each module's Cargo.toml to maintain the workspace structure +COPY ./entity/Cargo.toml ./entity/Cargo.toml +COPY ./entity_api/Cargo.toml ./entity_api/Cargo.toml +COPY ./migration/Cargo.toml ./migration/Cargo.toml +COPY ./service/Cargo.toml ./service/Cargo.toml +COPY ./web/Cargo.toml ./web/Cargo.toml + +# Copy the complete source code into the container's working directory +COPY . . + +# Set the target directory to ensure binaries are placed in a known location +#ENV CARGO_TARGET_DIR=/usr/src/app/target/aarch64-unknown-linux-gnu/release + +# Build the project +RUN cargo build --release --workspace + +# logs the contents of the /usr/src/app directory to the docker build log and outputs them to the console +RUN ls -la /usr/src/app/target/release/ + +RUN file /usr/src/app/target/release/* + +# Stage 2: Runtime Stage +FROM debian:stable-slim AS runtime + +# Install necessary runtime dependencies +RUN apt-get update && apt-get install -y \ + libssl3 \ + libpq5 \ + postgresql-client-15 \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory +WORKDIR /usr/src/app + +# Copy the compiled binaries from the builder stage +COPY --from=builder /usr/src/app/target/release/refactor_platform_rs /usr/local/bin/refactor_platform_rs +COPY --from=builder /usr/src/app/target/release/migration /usr/local/bin/migration +COPY --from=builder /usr/src/app/target/release/seed_db /usr/local/bin/seed_db + +# Create a non-root user for running the application +RUN useradd -m appuser && \ + chown -R appuser:appuser /usr/src/app /usr/local/bin && \ + chmod +x /usr/local/bin/* + +# Switch to the non-root user +USER appuser + +# Expose the necessary ports +EXPOSE 4000 + +# Default command starts an interactive bash shell +# Set ENTRYPOINT to default to run the Rust binary with arguments +ENTRYPOINT ["/bin/bash", "-c", "/usr/local/bin/refactor_platform_rs -l DEBUG -i \"$SERVICE_INTERFACE\" -p \"$SERVICE_PORT\" -d \"$DATABASE_URL\""] + +# Default CMD allows overriding with custom commands +CMD ["bash"] diff --git a/README.md b/README.md index 5e78c0f..80e3dc1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ [![Build & Tests (backend)](https://github.com/Jim-Hodapp-Coaching/refactor-platform-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/Jim-Hodapp-Coaching/refactor-platform-rs/actions/workflows/ci.yml) # Refactor Coaching & Mentoring Platform -### Backend + +## Backend ## Intro @@ -90,6 +91,7 @@ DATABASE_URL=postgres://refactor:password@localhost:5432/refactor_platform sea-o ``` ### Generate a new Entity from Database + Note that to generate a new Entity using the CLI you must ignore all other tables using the `--ignore-tables` option. You must add the option for _each_ table you are ignoring. ```bash diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..283a014 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,51 @@ +services: + # Local PostgreSQL container (used for local development when needed) + postgres: + image: postgres:17 # Use PostgreSQL version 17 + container_name: postgres # Name the container "postgres" + environment: + POSTGRES_USER: ${POSTGRES_USER} # Set PostgreSQL user from environment variable + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Set PostgreSQL password from environment variable + POSTGRES_DB: ${POSTGRES_DB} # Set PostgreSQL database name from environment variable + ports: + - "${POSTGRES_PORT}:5432" # Map host port to container's PostgreSQL port + volumes: + - postgres_data:/var/lib/postgresql/data # Persist PostgreSQL data + - ./migration/src/setup.sql:/docker-entrypoint-initdb.d/0-setup.sql # Initialize database with setup.sql + - ./migration/src/refactor_platform_rs.sql:/docker-entrypoint-initdb.d/1-refactor_plaform_rs.sql # Initialize with refactor_platform_rs.sql + networks: + - backend_network # Connect to backend_network + + # Rust application that connects to either local or remote PostgreSQL + rust-app: + image: rust-backend # Use the built image + build: + context: . # Build context is current directory + dockerfile: Dockerfile # Use specified Dockerfile + target: runtime # Use runtime target + platform: linux/arm64 # Specify the platform + container_name: rust-app # Name the container "rust-app" + environment: + POSTGRES_USER: ${POSTGRES_USER} # Set PostgreSQL user from environment variable + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Set PostgreSQL password from environment variable + POSTGRES_DB: ${POSTGRES_DB} # Set PostgreSQL database name from environment variable + POSTGRES_SCHEMA: ${POSTGRES_SCHEMA} # Set PostgreSQL schema from environment variable + POSTGRES_HOST: postgres # Set PostgreSQL host to "postgres" service + POSTGRES_PORT: ${POSTGRES_PORT} # Set PostgreSQL port from environment variable + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/${POSTGRES_DB} # Configure database URL + SERVICE_PORT: ${SERVICE_PORT} # Set service port from environment variable + SERVICE_INTERFACE: ${SERVICE_INTERFACE} # Set service interface from environment variable + ports: + - "${SERVICE_PORT}:${SERVICE_PORT}" # Map host port to container's service port + depends_on: + - postgres # Ensure postgres service starts before rust-app + networks: + - backend_network # Connect to backend_network + command: ["sh", "-c", "sleep 5 && /usr/local/bin/refactor_platform_rs"] # Wait for Postgres and run the app + +networks: + backend_network: + driver: bridge # Use bridge network driver + +volumes: + postgres_data: # Define postgres_data volume diff --git a/docs/runbooks/Container-README.md b/docs/runbooks/Container-README.md new file mode 100644 index 0000000..5694175 --- /dev/null +++ b/docs/runbooks/Container-README.md @@ -0,0 +1,341 @@ +# Refactor Coaching & Mentoring Platform with Docker & Docker Compose + +*This project is a Rust-based backend/web API that connects to a PostgreSQL database. It uses Docker and Docker Compose for easy local development and deployment, and includes utilities for database management, migrations, and more. You can choose to run PostgreSQL either locally (via Docker) or remotely by configuring the environment variables.* + +## Prerequisites + +Before you begin, ensure you have the following installed: + +- [Docker](https://www.docker.com/products/docker-desktop) (version 20+) +- [Docker Compose](https://docs.docker.com/compose/install/) (version 1.29+) + +## Project Setup + +1. **Clone the repository**: + + ```bash + git clone + cd + ``` + +2. **Environment Configuration**: + + You'll need to decide whether you're connecting to a **local PostgreSQL container** (using Docker) or a **remote PostgreSQL instance** (e.g., on a different host or in the cloud). This is configured using `.env` files. + + - **For local PostgreSQL** (default, Docker-based): + Create a `.env.local` file based on the template provided below and specify `POSTGRES_HOST=postgres`. + + - **For remote PostgreSQL**: + Create a `.env.remote-db` file and set `POSTGRES_HOST` to the external IP or hostname of the remote PostgreSQL instance. + + Example `.env.local` for local development: + + ```env + POSTGRES_USER=refactor + POSTGRES_PASSWORD=password + POSTGRES_DB=refactor_platform + POSTGRES_HOST=postgres + POSTGRES_PORT=5432 + WEB_PORT=4000 + USERNAME=appuser + USER_UID=1000 + USER_GID=1000 + ``` + + Example `.env.remote-db` for remote PostgreSQL: + + ```env + POSTGRES_USER=remote_user + POSTGRES_PASSWORD=remote_password + POSTGRES_DB=remote_db + POSTGRES_HOST=remote-db-host.com + POSTGRES_PORT=5432 + WEB_PORT=8080 + USERNAME=remote_appuser + USER_UID=1001 + USER_GID=1001 + ``` + +3. **Review the `docker-compose.yaml` file**: + + The Docker Compose file is configured to use environment variables defined in your `.env` files. The PostgreSQL container can either run locally (if specified in your environment file) or you can connect to a remote instance by setting the appropriate `POSTGRES_HOST`. + + ```yaml + services: + postgres: + image: postgres:17 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "${POSTGRES_PORT}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + rust-app: + build: . + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + ports: + - "${WEB_PORT}:4000" + depends_on: + - postgres + ``` + +## Building and Running the Application + +### **1. Building the rust-backend image** + +To build the Docker image for the Rust application, run the following command: + + ```bash + docker buildx build --platform linux/amd64,linux/arm64 -t rust-backend . + ``` + +- this will build the image for both amd64 and arm64 architectures or you can choose to build for a specific architecture by specifying the `--platform` flag. +- the image will be tagged as `rust-backend` and will be used in the `docker-compose.yaml` file. + +### **2. Running the Rust Backend Application** + + **Interactive Shell Access to Functions:** When you start the container the default behavior is to enter an interactive session, both the binaries and each of the functions from rebuild_db.sh are immediately available: + + ```bash + docker run -it rns-backend:latest + ``` + +- Inside the container shell, you can directly call any function from `rebuild_db.sh` directly by name: +- You can also run any of the binaries or functions from `rebuild_db.sh` directly by name as arguments to the `docker run` command: + +```bash +docker run -it rns-backend:latest seed_db +``` + +## Building and Running the Entire Application + +To build and run the entire application, including the Rust backend and PostgreSQL database, follow these steps: + +### **Step 1: Build the Docker images** + +From the project root, run the following command to build the Docker images for both the Rust application and PostgreSQL (if running locally). + +```bash +docker-compose --env-file .env.local build +``` + +or for remote PostgreSQL: + +```bash +docker-compose --env-file .env.remote-db build +``` + +### **Step 2: Start the application** + +Once the build is complete, start the containers using Docker Compose. This will start the PostgreSQL database (if local) and the Rust application. + +For local PostgreSQL: + +```bash +docker-compose --env-file .env.local up +``` + +For remote PostgreSQL: + +```bash +docker-compose --env-file .env.remote-db up +``` + +The Rust web API should now be running on `http://localhost:8080` and PostgreSQL should be available on port `5432` (or remotely if using the `.env.remote-db` setup). + +## Using the Database Utilities + +### **Rebuild the database** + +To rebuild the database (create a new database, user, and schema): + +```bash +docker-compose run rust-app rebuild-db +``` + +This will use the default settings from your environment variables (`POSTGRES_DB`, `POSTGRES_USER`, and `POSTGRES_SCHEMA`). + +### **Seed the database** + +To seed the database with test data: + +```bash +docker-compose run rust-app seed-db +``` + +This runs a Rust service that seeds the PostgreSQL database with predefined data. + +### **Convert DBML to SQL** + +If you have a DBML file (`schema.dbml`), you can convert it into an SQL schema: + +```bash +docker-compose run -v $(pwd)/sql:/app/sql -v $(pwd)/schema.dbml:/app/schema.dbml rust-app dbml2sql +``` + +This will output the generated SQL file in the `sql` directory. + +## Stopping the Containers + +To stop the containers, you can use the following command: + +```bash +docker-compose down +``` + +If you want to remove all the containers, networks, and volumes, including the PostgreSQL data: + +```bash +docker-compose down -v +``` + +## Troubleshooting and Gotchas + +### **1. Cannot connect to PostgreSQL** + +- **Problem**: The Rust application cannot connect to the PostgreSQL container. +- **Solution**: + 1. Ensure that PostgreSQL is running by checking the container status: + + ```bash + docker-compose ps + ``` + + 2. Check the PostgreSQL logs to see if it's running correctly: + + ```bash + docker-compose logs postgres + ``` + + 3. Ensure that the `POSTGRES_HOST` environment variable in the Rust app is set to `postgres` (for local) or to the correct remote hostname. + +### **2. Web API not accessible** + +- **Problem**: The web API is not accessible on `localhost:8080`. +- **Solution**: + 1. Verify that the container is running: + + ```bash + docker-compose ps + ``` + + 2. Check if the correct port is exposed: + + ```bash + docker-compose logs rust-app + ``` + + 3. If you have changed the port (e.g., `WEB_PORT=9090`), make sure you access the web API at `http://localhost:9090`. + +### **3. Port conflicts** + +- **Problem**: The default ports `8080` for the web API or `5432` for PostgreSQL are already in use by another service. +- **Solution**: Change the ports in the `.env` file or in `docker-compose.yaml` to different, unused ports: + + ```yaml + services: + postgres: + ports: + - "5433:5432" + + rust-app: + ports: + - "9090:8080" + ``` + +### **4. Rebuild the application after changes** + +- **Problem**: You made changes to the Rust code or the Dockerfile, but they are not reflected when you restart the container. +- **Solution**: Make sure to rebuild the Docker image after code changes: + + ```bash + docker-compose build + docker-compose up + ``` + +### **5. Database persistence** + +- **Problem**: The database resets every time the containers are stopped and restarted. +- **Solution**: Docker volumes are used to persist data between container restarts. Ensure that the volume `postgres_data` is properly configured in `docker-compose.yaml`: + + ```yaml + volumes: + postgres_data: + ``` + + If the data is still being wiped, make sure you are not running `docker-compose down -v` unless you want to remove the database volume. + +## Tips for Development + +- You can bring up the containers in detached mode by running: + + ```bash + docker-compose up -d + ``` + +- To access the running containers (e.g., for debugging): + + ```bash + docker exec -it bash + ``` + + Example to access the Rust app container: + + ```bash + docker exec -it rust-app bash + ``` + +- If you want to restart only one service (e.g., `rust-app`): + + ```bash + docker-compose restart rust-app + ``` + +- To stop all containers: + + ```bash + docker-compose down + ``` + +To stop and remove all containers, networks, and volumes: + + ```bash + docker-compose down -v + ``` + +--- + +## Working with the Rust application: + +- To run the container with `rebuild_db.sh` (or any other binary) interactively, you can launch the container with a specific `CMD` override or call it directly: + + ```bash + # Run the container and execute the shell script + docker run -it --entrypoint ./rebuild_db.sh my-image-name:latest + ``` + +## Testing and Debugging in Interactive Mode + +- To debug or test interactively with all scripts and binaries in place: + + ```bash + # Run the container interactively + docker run -it --entrypoint /bin/bash my-image-name:latest + ``` + +- Inside the container, you can manually check and execute: + + ```bash + # Manually run the shell script + ./rebuild_db.sh + # Manually run other binaries + ./target/release/seed_db + ./target/release/refactor_platform_rs + ``` diff --git a/docs/runbooks/deploy.md b/docs/runbooks/deploy.md new file mode 100644 index 0000000..22c653d --- /dev/null +++ b/docs/runbooks/deploy.md @@ -0,0 +1,299 @@ +

Refactor Platform Deployment Runbook

+ +
+ Ticket Contents
+ +> **Create the automation code and Terraform code to deploy the frontend and backend Rust app to a running Ubuntu 22.04 server, using local Terraform state, try deploying it with a container and without using Docker containers or something “native”. A Container that can run on the ContainerD Container Runtime, which should include Docker Containers.** + +**Idea:** *use Ansible for performing initial setup on a server target.* + +### Definition of done + +* Able to build and start the backend and frontend applications using Docker containers and docker-compose +* All needed Ansible entities (roles, resource groupings) and scripts exist that can deploy to any target Ubuntu 22.04 instance running Ansible + +
+ +
+ Summary
+ +*This runbook provides detailed steps to deploy Rust frontend and backend applications on an Ubuntu 22.04 server using Ansible for initial setup and Terraform for automation. It covers both containerized (using Docker) and native deployment methods.* + +
+ +
+ I. Initial Server Setup with Ansible
+ +1. **Define Ansible Inventory:** Create an Ansible inventory file containing the IP address or hostname of your target Ubuntu 22.04 server. + +```ini +[ubuntu_server] +192.168.1.10 ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/id_rsa +``` + +*Explanation*: The inventory file specifies the target servers for Ansible to manage. Here, we define the target server with its IP address and SSH details. + +1. **Create Ansible Playbook:** Develop an Ansible playbook to perform the following tasks on the server: + * **Install Required Packages:** Install necessary packages like `curl`, `git`, `unzip`, `sudo`, `python3`, `python3-pip`, `jq`, `make`, `gcc`, `g++`, and others required for Rust compilation and container runtime. + * **Install Containerd:** Install and configure Containerd as the container runtime. + * **Install Docker (Optional):** Install Docker if you plan to use it alongside Containerd. + * **Install Ansible (Optional):** Install Ansible on the server for future deployments and automation. + * **Configure Firewall:** Open necessary ports for your applications (e.g., HTTP/HTTPS). + +```yaml +--- +- hosts: ubuntu_server + become: yes + tasks: + - name: Install required packages + apt: + name: + - curl + - git + - unzip + - sudo + - python3 + - python3-pip + - jq + - make + - gcc + - g++ + state: present + update_cache: yes + + - name: Install Containerd + shell: | + sudo apt-get install -y containerd + + - name: Install Docker (Optional) + shell: | + sudo apt-get install -y docker.io + + - name: Install Ansible (Optional) + apt: + name: ansible + state: present + + - name: Configure firewall + ufw: + rule: allow + port: "{{ item }}" + proto: tcp + with_items: + - 80 + - 443 +``` + +*Explanation*: The playbook installs necessary packages, container runtime, and configures the firewall. + +1. **Run Ansible Playbook:** Execute the playbook against the server. + +```bash +ansible-playbook -i inventory.ini setup.yml +``` + +*Gotcha*: Ensure Ansible is installed on your local machine and accessible via `ansible-playbook` command. + +
+ +
+ II. Frontend and Backend Application Deployment
+ +### A. Containerized Approach (Docker-Compose) + +1. **Create Dockerfiles:** Write Dockerfiles for both the frontend and backend applications, defining the necessary dependencies, build commands, and entry points. + +```Dockerfile +# Backend Dockerfile +FROM rust:latest + +WORKDIR /app +COPY . . + +RUN cargo build --release + +CMD ["./target/release/backend_app"] +``` + +```Dockerfile +# Frontend Dockerfile +FROM rust:latest + +WORKDIR /app +COPY . . + +RUN cargo build --release + +CMD ["./target/release/frontend_app"] +``` + +*Gotcha*: Ensure the Dockerfile paths and commands match your application structure. + +1. **Create Docker-Compose.yml:** Define the service dependencies and configurations for both frontend and backend applications within a docker-compose.yml file. + +```yaml +version: '3' +services: + backend: + build: ./backend + ports: + - "8000:8000" + frontend: + build: ./frontend + ports: + - "3000:3000" +``` + +*Context*: This configuration binds the container ports to the host machine. + +1. **Build Images:** Build the Docker images for both applications. + +```bash +docker-compose build +``` + +*Gotcha*: Ensure Docker is running on your local machine. + +1. **Deploy with Docker-Compose:** Use `docker-compose up -d` to start the containers on the server based on the defined docker-compose configuration. + +```bash +docker-compose up -d +``` + +*Context*: The `-d` flag runs containers in the background. + +## B. Native Approach (Without Containers) + +1. **Build Frontend and Backend Applications:** Build the frontend and backend applications using Rust's build tools (e.g., `cargo build`). + +```bash +cd backend +cargo build --release + +cd ../frontend +cargo build --release +``` + +*Gotcha*: Ensure Rust and Cargo are installed on your local machine. + +1. **Copy Compiled Binaries to Server:** Transfer the compiled binaries of the frontend and backend applications to the server. + +```bash +scp backend/target/release/backend_app ubuntu@192.168.1.10:~/backend_app +scp frontend/target/release/frontend_app ubuntu@192.168.1.10:~/frontend_app +``` + +*Context*: Use `scp` to securely copy files over SSH. + +1. **Configure and Start Applications:** Configure and start the applications on the server using systemd services or other suitable methods. + +```ini +# backend_app.service +[Unit] +Description=Backend Application + +[Service] +ExecStart=/home/ubuntu/backend_app + +[Install] +WantedBy=multi-user.target +``` + +```ini +# frontend_app.service +[Unit] +Description=Frontend Application + +[Service] +ExecStart=/home/ubuntu/frontend_app + +[Install] +WantedBy=multi-user.target +``` + +Start and enable services: + +```bash +sudo systemctl daemon-reload +sudo systemctl start backend_app +sudo systemctl enable backend_app + +sudo systemctl start frontend_app +sudo systemctl enable frontend_app +``` + +*Gotcha*: Ensure the service files are correctly placed in `/etc/systemd/system/`. + +
+ +
+ III. Terraform Deployment (Local State)
+ +1. **Create Terraform Code:** Write Terraform code to manage the deployment process, including: + * **Create Remote-Exec Provider:** Configure the remote-exec provider for running commands on the server. + * **Provision Ansible:** If not already installed, use the Terraform provider to install Ansible on the server (optional). + * **Run Ansible Playbook:** Use the remote-exec provider to execute the Ansible playbook on the server for initial setup. + * **Manage Application Files:** Transfer the Dockerfiles, docker-compose.yml, compiled binaries (for the native approach), or application configuration files to the server. + * **Run Docker-Compose (Containerized Approach):** Use the remote-exec provider to run `docker-compose up -d` command on the server to start the containers. + * **Start Applications (Native Approach):** Use the remote-exec provider to start the application processes on the server. + +```hcl +provider "local" {} + +provider "null" {} + +resource "null_resource" "setup" { + connection { + type = "ssh" + user = "ubuntu" + private_key = file("~/.ssh/id_rsa") + host = "192.168.1.10" + } + + provisioner "remote-exec" { + inline = [ + "sudo apt-get update", + "sudo apt-get install -y ansible", + "ansible-playbook -i inventory.ini setup.yml" + ] + } +} +``` + +*Context*: This config uses the `null` provider to run remote commands via SSH. + +1. **Initialize Terraform:** Initialize Terraform in the working directory. + +```bash +terraform init +``` + +*Gotcha*: Ensure you are in the correct directory containing `main.tf`. + +1. **Apply Terraform Changes:** Use `terraform apply` to apply the infrastructure and configuration changes. + +```bash +terraform apply +``` + +*Context*: Review the changes before applying by typing `yes` when prompted. + +
+ +
+ IV. Definition of Done
+ +* Both the frontend and backend applications are successfully built and started. +* The applications are accessible and function as expected. +* The Ansible playbook, Dockerfiles, and Terraform code are written and documented. +* All necessary steps are automated and can be deployed to any Ubuntu 22.04 instance running Ansible. + +
+ +
+ Additional Considerations
+ +* **Logging and Monitoring:** Implement logging and monitoring solutions like Prometheus, Grafana, or ELK stack for application performance and health checks. +* **Resilience and Elasticity:** Consider using load balancers and auto-scaling groups to handle traffic spikes and ensure high availability. + +
diff --git a/entity_api/Cargo.toml b/entity_api/Cargo.toml index 554a8fb..338bd9c 100644 --- a/entity_api/Cargo.toml +++ b/entity_api/Cargo.toml @@ -19,16 +19,11 @@ sqlx-sqlite = { version = "0.8.2" } utoipa = { version = "4.2.0", features = ["axum_extras", "uuid"] } [dependencies.sea-orm] -version = "1.1.0" # sea-orm version -features = [ - "debug-print", - "runtime-tokio-native-tls", - "sqlx-postgres" -] +version = "1.1.0" # sea-orm version +features = ["debug-print", "runtime-tokio-native-tls", "sqlx-postgres"] [dev-dependencies] tokio = { version = "1.40", features = ["full"] } [features] mock = ["sea-orm/mock"] - diff --git a/migration/src/setup.sql b/migration/src/setup.sql new file mode 100644 index 0000000..f4add38 --- /dev/null +++ b/migration/src/setup.sql @@ -0,0 +1,10 @@ +-- TODO: Is it possible to make the DB name, DB user, schema name, etc all environment variables from the container? + +CREATE SCHEMA refactor_platform AUTHORIZATION refactor; + +GRANT ALL PRIVILEGES ON DATABASE refactor TO refactor; +GRANT ALL ON SCHEMA refactor_platform TO refactor; + +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON TABLES TO refactor; +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON SEQUENCES TO refactor; +ALTER DEFAULT PRIVILEGES IN SCHEMA refactor_platform GRANT ALL ON FUNCTIONS TO refactor; \ No newline at end of file diff --git a/scripts/rebuild_db_container.sh b/scripts/rebuild_db_container.sh new file mode 100644 index 0000000..149b2c2 --- /dev/null +++ b/scripts/rebuild_db_container.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Define default variables +DB_NAME=${1:-"refactor_platform"} +DB_USER=${2:-"refactor"} +SCHEMA_NAME=${3:-"refactor_platform"} + +echo "Using the following configuration:" +echo "Database Name: $DB_NAME" +echo "Database User: $DB_USER" +echo "Schema Name: $SCHEMA_NAME" + +# Check if postgres is installed with its client CLI +[ -f $(which postgres) ] && +[ -f $(which pg_ctl ) ] && +[ -f $(which psql) ] > /dev/null 2>&1 || +{ echo "Postgres and psql are not completely installed. Please install postgres with your package manager or Postgres.app and try again"; exit 1; } + +# if [[ -z "${PGDATA}" ]]; then +# echo 'Environment variable PGDATA unset. See `pg_ctl --help for more information.' +# fi + +# Ensure postgres is running and start postgres with homebrew if it is not running +pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB +#pg_ctl status > /dev/null 2>&1 || { echo "Starting Postgres..."; pg_ctl -w -t 15 start; } + +# Check if the postgres database exists and create it if it doesn't +POSTGRES_DB_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_database WHERE datname='postgres'") +if [ "$POSTGRES_DB_EXISTS" != "1" ]; then + echo "Creating 'postgres' database..." + createdb -U postgres postgres || { echo "Failed to create 'postgres' database"; exit 1; } +fi + +# Check if the user exists and create it if it doesn't +USER_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'") +if [ "$USER_EXISTS" != "1" ]; then + echo "Creating user $DB_USER..." + psql -U postgres -c "CREATE USER $DB_USER;" || { echo "Failed to create user $DB_USER"; exit 1; } +fi + +# Check if the database exists +DB_EXISTS=$(psql $POSTGRES_URL -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'") + +# If the database exists, drop it +if [ "$DB_EXISTS" = "1" ]; then + echo "Database $DB_NAME exists. Dropping the database..." + psql $POSTGRES_URL -c "DROP DATABASE IF EXISTS $DB_NAME;" || { echo "Failed to drop database $DB_NAME"; exit 1; } +fi + +echo "Creating the database $DB_NAME..." +psql $POSTGRES_URL -c "CREATE DATABASE $DB_NAME;" || { echo "Failed to create database $DB_NAME"; exit 1; } + +echo "Granting privileges to $DB_USER on $DB_NAME..." +psql $POSTGRES_URL -c "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;" + +# Check if the schema exists +SCHEMA_EXISTS=$(psql $POSTGRES_URL -d $DB_NAME -tAc "SELECT 1 FROM information_schema.schemata WHERE schema_name = '$SCHEMA_NAME'") + +# If the schema does not exist, create it +if [ "$SCHEMA_EXISTS" != "1" ]; then + echo "Creating schema $SCHEMA_NAME..." + psql $POSTGRES_URL -d $DB_NAME -c "CREATE SCHEMA $SCHEMA_NAME;" || { echo "Failed to create schema $SCHEMA_NAME"; exit 1; } +fi + +# Generating SQL for the migrations using dbml2sql +echo "Generating SQL for the migrations..." +dbml2sql docs/db/refactor_platform_rs.dbml -o migration/src/refactor_platform_rs.sql || { echo "Error generating SQL file"; exit 1; } + +# Remove the line to create a schema from the generated SQL file +echo "Modifying the generated SQL file..." +sed -i '' '/CREATE SCHEMA/d' migration/src/refactor_platform_rs.sql + +echo "Running the migrations..." +DATABASE_URL=$POSTGRES_URL sea-orm-cli migrate up -s $SCHEMA_NAME || { echo "Failed to run migrations"; exit 1; } + +echo "Database setup and migrations completed successfully" \ No newline at end of file diff --git a/web/Cargo.toml b/web/Cargo.toml index b376c6d..814c7aa 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -33,7 +33,7 @@ features = [ "runtime-tokio-native-tls", "sea-orm-internal", "sqlx-postgres", - "with-uuid" + "with-uuid", ] [features]