From 4044f79e3646664655988a7abed30321a41d3e65 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 22:39:07 -0400 Subject: [PATCH 01/25] setup script --- config/.env.template | 5 ----- deployment/setup.sh | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 deployment/setup.sh diff --git a/config/.env.template b/config/.env.template index b5242821..79c692d7 100644 --- a/config/.env.template +++ b/config/.env.template @@ -28,11 +28,6 @@ SAC_AWS_REGION="SAC_AWS_REGION" SAC_SUDO_PASSWORD="Password#!1" -SAC_AWS_BUCKET_NAME="SAC_AWS_BUCKET_NAME" -SAC_AWS_ID="SAC_AWS_ID" -SAC_AWS_SECRET="SAC_AWS_SECRET" -SAC_AWS_REGION="SAC_AWS_REGION" - SAC_RESEND_API_KEY="SAC_RESEND_API_KEY" SAC_CALENDAR_MAX_TERMINATION_DATE="12-31-2024" diff --git a/deployment/setup.sh b/deployment/setup.sh new file mode 100644 index 00000000..7ea1eaf6 --- /dev/null +++ b/deployment/setup.sh @@ -0,0 +1,18 @@ +sudo yum update -y +sudo yum install -y docker git golang +sudo systemctl enable --now docker +sudo usermod -a -G docker ec2-user + +# Install the docker compose plugin for all users +sudo mkdir -p /usr/local/lib/docker/cli-plugins + +sudo curl -sL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-"$(uname -m)" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose + +# Set ownership to root and make executable +test -f /usr/local/lib/docker/cli-plugins/docker-compose \ + && sudo chown root:root /usr/local/lib/docker/cli-plugins/docker-compose +test -f /usr/local/lib/docker/cli-plugins/docker-compose \ + && sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + +git clone -b prod-business https://github.com/GenerateNU/sac From 4e333247b0eb1ac467acfeba70b70452f44422b0 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 22:55:43 -0400 Subject: [PATCH 02/25] caddy and more compose --- deployment/Caddyfile | 3 ++ deployment/compose.yml | 101 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 deployment/Caddyfile create mode 100644 deployment/compose.yml diff --git a/deployment/Caddyfile b/deployment/Caddyfile new file mode 100644 index 00000000..6c105104 --- /dev/null +++ b/deployment/Caddyfile @@ -0,0 +1,3 @@ +54.158.91.126 { + reverse_proxy sac_webserver:8080 +} \ No newline at end of file diff --git a/deployment/compose.yml b/deployment/compose.yml new file mode 100644 index 00000000..9f0d5bff --- /dev/null +++ b/deployment/compose.yml @@ -0,0 +1,101 @@ +services: + # WEBSERVER + caddy: + image: caddy:latest + restart: unless-stopped + cap_add: + - NET_ADMIN + ports: + - 80:80 + - 443:443 + - 443:443/udp + - 8443:8443 + - 8443:8443/udp + volumes: + - $PWD/Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + webserver: + container_name: sac_webserver + build: + context: ../backend + dockerfile: Dockerfile.server + env_file: + - .env + + # REDIS + redis-session: + build: + context: ../backend + dockerfile: Dockerfile.redis + container_name: redis_session + ports: + - 6380:6379 + environment: + - REDIS_USERNAME=${SAC_REDIS_SESSION_USERNAME} + - REDIS_PASSWORD=${SAC_REDIS_SESSION_PASSWORD} + - REDIS_DISABLE_DEFAULT_USER="true" + volumes: + - redis-session-data:/data + redis-limiter: + build: + context: . + dockerfile: Dockerfile.redis + container_name: redis_limiter + ports: + - 6381:6379 + environment: + - REDIS_USERNAME=${SAC_REDIS_LIMITER_USERNAME} + - REDIS_PASSWORD=${SAC_REDIS_LIMITER_PASSWORD} + - REDIS_DISABLE_DEFAULT_USER="true" + volumes: + - redis-limiter-data:/data + + # OPENSEARCH + opensearch-node1: + image: opensearchproject/opensearch:latest + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.type=single-node + - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - DISABLE_SECURITY_PLUGIN=true # + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data + ports: + - 9200:9200 + - 9600:9600 # required for Performance Analyzer + networks: + - opensearch-net + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest + container_name: opensearch-dashboards + ports: + - 5601:5601 + expose: + - "5601" + environment: + OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]' + DISABLE_SECURITY_DASHBOARDS_PLUGIN: true + networks: + - opensearch-net + +volumes: + redis-session-data: + redis-limiter-data: + opensearch-data1: + caddy_data: + external: true + caddy_config: + +networks: + opensearch-net: From 3825678304a0d767bd42db52713e2a468e6a3ceb Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 23:10:47 -0400 Subject: [PATCH 03/25] change to .env.prod --- deployment/compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/compose.yml b/deployment/compose.yml index 9f0d5bff..32ac89d6 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -21,7 +21,7 @@ services: context: ../backend dockerfile: Dockerfile.server env_file: - - .env + - .env.prod # REDIS redis-session: From 1bebe611e079ca91cdc23c7a82f9bd6948b6e235 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 23:15:27 -0400 Subject: [PATCH 04/25] compose debugging I --- deployment/compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/compose.yml b/deployment/compose.yml index 32ac89d6..4d47f47c 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -19,7 +19,7 @@ services: container_name: sac_webserver build: context: ../backend - dockerfile: Dockerfile.server + dockerfile: ../backend/Dockerfile.server env_file: - .env.prod @@ -27,7 +27,7 @@ services: redis-session: build: context: ../backend - dockerfile: Dockerfile.redis + dockerfile: ../backend/Dockerfile.redis container_name: redis_session ports: - 6380:6379 @@ -39,8 +39,8 @@ services: - redis-session-data:/data redis-limiter: build: - context: . - dockerfile: Dockerfile.redis + context: ../backend + dockerfile: ../backend/Dockerfile.redis container_name: redis_limiter ports: - 6381:6379 From 29f6c6f7d4281d474168d5c225202e5281f40660 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 23:23:07 -0400 Subject: [PATCH 05/25] fix dockerfile --- backend/Dockerfile.server | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 57044ccc..9e633966 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM golang:1.22-alpine as builder WORKDIR /app @@ -9,7 +10,10 @@ RUN templ generate RUN go get ./... RUN go mod tidy RUN go mod download -RUN go build -tags prod -o bin/sac main.go + +# Build the Go binary (optimized for production) +RUN mkdir -p /app/bin +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /app/bin/sac FROM scratch COPY --from=builder /app/bin/sac /sac From 0f05208cf9cb85667ba644db02890b08ede4003f Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 23:56:47 -0400 Subject: [PATCH 06/25] Fix dockerfile --- backend/Dockerfile.server | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 9e633966..20f1970f 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -1,22 +1,17 @@ # syntax=docker/dockerfile:1 -FROM golang:1.22-alpine as builder + +FROM golang:1.22.2 WORKDIR /app -RUN apk add --no-cache git + +COPY go.mod go.sum ./ COPY . ./ RUN go install github.com/a-h/templ/cmd/templ@latest RUN templ generate -RUN go get ./... -RUN go mod tidy -RUN go mod download -# Build the Go binary (optimized for production) -RUN mkdir -p /app/bin -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /app/bin/sac - -FROM scratch -COPY --from=builder /app/bin/sac /sac +RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac EXPOSE 8080 -ENTRYPOINT [ "./sac" ] \ No newline at end of file + +CMD ["/sac"] \ No newline at end of file From a5a05a67b71887f5861b00395959c6694be98d42 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 10 Jun 2024 23:59:49 -0400 Subject: [PATCH 07/25] docker compose --- backend/Dockerfile.server | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 20f1970f..4924d2ff 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -9,6 +9,7 @@ COPY go.mod go.sum ./ COPY . ./ RUN go install github.com/a-h/templ/cmd/templ@latest RUN templ generate +RUN go get github.com/GenerateNU/backend/docs RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac From 02075101e6e127023f7f000ab075fcd33ab8acab Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:01:25 -0400 Subject: [PATCH 08/25] Fix dockerifle II --- backend/Dockerfile.server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 4924d2ff..c9a324c8 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -9,7 +9,7 @@ COPY go.mod go.sum ./ COPY . ./ RUN go install github.com/a-h/templ/cmd/templ@latest RUN templ generate -RUN go get github.com/GenerateNU/backend/docs +RUN go get github.com/GenerateNU/sac/backend/docs RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac From b7306ac48f2d7f734d5f3d6d8766e18945e2cdd1 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:04:15 -0400 Subject: [PATCH 09/25] Fix dockerfile III --- backend/Dockerfile.server | 1 - backend/main.go | 1 - 2 files changed, 2 deletions(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index c9a324c8..20f1970f 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -9,7 +9,6 @@ COPY go.mod go.sum ./ COPY . ./ RUN go install github.com/a-h/templ/cmd/templ@latest RUN templ generate -RUN go get github.com/GenerateNU/sac/backend/docs RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac diff --git a/backend/main.go b/backend/main.go index aea4c560..2b4d6179 100644 --- a/backend/main.go +++ b/backend/main.go @@ -18,7 +18,6 @@ import ( "github.com/GenerateNU/sac/backend/constants" "github.com/GenerateNU/sac/backend/database" "github.com/GenerateNU/sac/backend/database/store" - _ "github.com/GenerateNU/sac/backend/docs" "github.com/GenerateNU/sac/backend/integrations" "github.com/GenerateNU/sac/backend/integrations/email" "github.com/GenerateNU/sac/backend/integrations/file" From d8e631a68dff606531a1fdeb4d03f774b93c24f0 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:09:54 -0400 Subject: [PATCH 10/25] init db script --- deployment/init-db.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 deployment/init-db.sh diff --git a/deployment/init-db.sh b/deployment/init-db.sh new file mode 100644 index 00000000..8f958fa4 --- /dev/null +++ b/deployment/init-db.sh @@ -0,0 +1,2 @@ +go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest +migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file From 2133e087df55ae70a1c394d9f60806fb73f16bdd Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:12:30 -0400 Subject: [PATCH 11/25] init db debug I --- deployment/init-db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/init-db.sh b/deployment/init-db.sh index 8f958fa4..cee722eb 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,2 +1,2 @@ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file +~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file From ef741110af12585008cd879f07c58a6b2a41bca9 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:14:52 -0400 Subject: [PATCH 12/25] init db debugging II --- deployment/init-db.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/init-db.sh b/deployment/init-db.sh index cee722eb..e153a66d 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,2 +1,3 @@ +export $(cat .env.prod) | xargs) go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file +~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:x@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file From 58e32c3a9ee22e46dc32364985ad6000372ccb9a Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:15:37 -0400 Subject: [PATCH 13/25] init db debugging III --- deployment/init-db.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment/init-db.sh b/deployment/init-db.sh index e153a66d..3d5868d0 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,3 +1,2 @@ -export $(cat .env.prod) | xargs) go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest ~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:x@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file From 2daeab379fb9cd994a234478254c6c65df87832a Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:19:08 -0400 Subject: [PATCH 14/25] iniit db debugging IV --- deployment/init-db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/init-db.sh b/deployment/init-db.sh index 3d5868d0..cee722eb 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,2 +1,2 @@ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:x@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file +~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file From bac596ea46f78fd3787d687aacd3aec0e7c0d2cf Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 00:26:09 -0400 Subject: [PATCH 15/25] init db debugging V --- deployment/init-db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/init-db.sh b/deployment/init-db.sh index cee722eb..4a5d4442 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,2 +1,2 @@ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=disable -verbose up \ No newline at end of file +~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=enable -verbose up \ No newline at end of file From c2cff26c3adc4e5a5b520d4aba5d35b5d8f517a7 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 11:52:38 -0400 Subject: [PATCH 16/25] Debugging part 267 --- backend/Dockerfile.server | 2 +- deployment/init-db.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 20f1970f..928cc22b 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -14,4 +14,4 @@ RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac EXPOSE 8080 -CMD ["/sac"] \ No newline at end of file +CMD ["/app/sac"] \ No newline at end of file diff --git a/deployment/init-db.sh b/deployment/init-db.sh index 4a5d4442..486b8605 100644 --- a/deployment/init-db.sh +++ b/deployment/init-db.sh @@ -1,2 +1,2 @@ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest -~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=enable -verbose up \ No newline at end of file +~/go/bin/migrate -path ../backend/migrations/ -database postgres://${SAC_DB_USERNAME}:${SAC_DB_PASSWORD}@${SAC_DB_HOST}:${SAC_DB_PORT}/${SAC_DB_NAME}?sslmode=require -verbose up \ No newline at end of file From b84b7f3d16dcfcca98e8f5008a5d947e3625f753 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 12:09:17 -0400 Subject: [PATCH 17/25] make config path optional --- backend/config/config.go | 8 +++++--- backend/main.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/config/config.go b/backend/config/config.go index 010b5a55..47d728a4 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -7,9 +7,11 @@ import ( "github.com/joho/godotenv" ) -func GetConfiguration(path string) (*Settings, error) { - if err := godotenv.Load(path); err != nil { - return nil, fmt.Errorf("failed to load environment variables: %s", err.Error()) +func GetConfiguration(path *string) (*Settings, error) { + if path != nil { + if err := godotenv.Load(*path); err != nil { + return nil, fmt.Errorf("failed to load environment variables: %s", err.Error()) + } } intSettings, err := env.ParseAs[intermediateSettings]() diff --git a/backend/main.go b/backend/main.go index 2b4d6179..192d6efb 100644 --- a/backend/main.go +++ b/backend/main.go @@ -33,7 +33,7 @@ import ( func main() { onlyMigrate, seedSearch, configPath := parseFlags() - config, err := config.GetConfiguration(*configPath) + config, err := config.GetConfiguration(configPath) if err != nil { utilities.Exit("Error getting configuration: %s", err.Error()) } From 66a8f74d20d58135099b4a65b3cbbe02925675fc Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 12:13:41 -0400 Subject: [PATCH 18/25] fix --- backend/config/config.go | 6 +++--- backend/main.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/config/config.go b/backend/config/config.go index 47d728a4..152c8cbb 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -7,9 +7,9 @@ import ( "github.com/joho/godotenv" ) -func GetConfiguration(path *string) (*Settings, error) { - if path != nil { - if err := godotenv.Load(*path); err != nil { +func GetConfiguration(path string) (*Settings, error) { + if path != "" { + if err := godotenv.Load(path); err != nil { return nil, fmt.Errorf("failed to load environment variables: %s", err.Error()) } } diff --git a/backend/main.go b/backend/main.go index 192d6efb..2b7a97a0 100644 --- a/backend/main.go +++ b/backend/main.go @@ -9,7 +9,6 @@ import ( "net" "os" "os/signal" - "path/filepath" "syscall" "github.com/GenerateNU/sac/backend/background" @@ -33,7 +32,7 @@ import ( func main() { onlyMigrate, seedSearch, configPath := parseFlags() - config, err := config.GetConfiguration(configPath) + config, err := config.GetConfiguration(*configPath) if err != nil { utilities.Exit("Error getting configuration: %s", err.Error()) } @@ -82,7 +81,7 @@ func main() { func parseFlags() (onlyMigrate, seedSearch *bool, configPath *string) { onlyMigrate = flag.Bool("only-migrate", false, "Specify if you want to only perform the database migration") seedSearch = flag.Bool("seed-search", true, "Specify if you want to seed the opensearch nodes.") - configPath = flag.String("config", filepath.Join("..", "config", ".env.dev"), "Specify the path to the config file (.env)") + configPath = flag.String("config", "", "Specify the path to the config file (.env)") flag.Parse() return } From 772681d76b1178a582b39ce4506f8ed9f65fb934 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Tue, 11 Jun 2024 12:21:38 -0400 Subject: [PATCH 19/25] fix caddyfile --- deployment/Caddyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/Caddyfile b/deployment/Caddyfile index 6c105104..38e2e44c 100644 --- a/deployment/Caddyfile +++ b/deployment/Caddyfile @@ -1,3 +1,3 @@ -54.158.91.126 { +sac.tech0tron.net { reverse_proxy sac_webserver:8080 } \ No newline at end of file From 9f8221354864fd38339d79646fea7b99b918107c Mon Sep 17 00:00:00 2001 From: Generate NEU Date: Mon, 17 Jun 2024 04:13:51 +0000 Subject: [PATCH 20/25] fixes --- backend/config/app.go | 7 ++++--- backend/main.go | 16 ++++++++-------- backend/redis_entrypoint.sh | 4 ++-- backend/search/base/controller.go | 6 ++++++ backend/search/base/transactions.go | 12 ++++++++++-- backend/server/server.go | 6 ++++-- config/.env.template | 1 + deployment/Caddyfile | 2 +- deployment/compose.yml | 16 ++++++---------- 9 files changed, 42 insertions(+), 28 deletions(-) diff --git a/backend/config/app.go b/backend/config/app.go index 7b6f45ac..b3e611ca 100644 --- a/backend/config/app.go +++ b/backend/config/app.go @@ -3,9 +3,10 @@ package config import "fmt" type ApplicationSettings struct { - Port uint16 `env:"PORT"` - Host string `env:"HOST"` - BaseUrl string `env:"BASE_URL"` + Port uint16 `env:"PORT"` + Host string `env:"HOST"` + BaseUrl string `env:"BASE_URL"` + PublicUrl string `env:"PUBLIC_URL"` } func (s *ApplicationSettings) ApplicationURL() string { diff --git a/backend/main.go b/backend/main.go index 2b7a97a0..84239695 100644 --- a/backend/main.go +++ b/backend/main.go @@ -21,6 +21,7 @@ import ( "github.com/GenerateNU/sac/backend/integrations/email" "github.com/GenerateNU/sac/backend/integrations/file" "github.com/GenerateNU/sac/backend/integrations/oauth/soth/sothic" + "github.com/GenerateNU/sac/backend/search" "github.com/GenerateNU/sac/backend/server" "github.com/GenerateNU/sac/backend/telemetry" "github.com/GenerateNU/sac/backend/utilities" @@ -97,14 +98,13 @@ func checkServerRunning(host string, port uint16) error { } func seedSearchData(db *gorm.DB) { - slog.Info("to appease linter", "seedSearch", true, "db", db) - // if err := search.SeedClubs(db); err != nil { - // return - // } - - // if err := search.SeedEvents(db); err != nil { - // return - // } + if err := search.SeedClubs(db); err != nil { + return + } + + if err := search.SeedEvents(db); err != nil { + return + } } func startBackgroundJobs(ctx context.Context, db *gorm.DB) { diff --git a/backend/redis_entrypoint.sh b/backend/redis_entrypoint.sh index ffa4913b..37ba71ef 100644 --- a/backend/redis_entrypoint.sh +++ b/backend/redis_entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # set up redis configuration directory mkdir -p /usr/local/etc/redis @@ -12,7 +12,7 @@ if [ -n ${REDIS_USERNAME} ] && [ -n ${REDIS_PASSWORD} ]; then fi # disable default user -if [ $(echo ${REDIS_DISABLE_DEFAULT_USER}) == "true" ]; then +if [ "$REDIS_DISABLE_DEFAULT_USER" == "true" ]; then echo "user default off nopass nocommands" >> /usr/local/etc/redis/custom_aclfile.acl fi diff --git a/backend/search/base/controller.go b/backend/search/base/controller.go index bd8a6b47..02909ace 100644 --- a/backend/search/base/controller.go +++ b/backend/search/base/controller.go @@ -1,7 +1,9 @@ package base import ( + "log" "net/http" + "os" search_types "github.com/GenerateNU/sac/backend/search/types" "github.com/GenerateNU/sac/backend/utilities" @@ -30,14 +32,18 @@ func NewSearchController(searchService SearchServiceInterface) *SearchController // @Failure 500 {object} error // @Router /search/clubs [get] func (s *SearchController) SearchClubs(c *fiber.Ctx) error { + log.SetOutput(os.Stdout) + var searchQuery search_types.ClubSearchRequest + log.Println("RAHHHHHH") if err := c.BodyParser(&searchQuery); err != nil { return utilities.InvalidJSON() } result, err := s.searchService.SearchClubs(searchQuery) if err != nil { + log.Println("RAHHHHHH XD") return err } diff --git a/backend/search/base/transactions.go b/backend/search/base/transactions.go index 8ddfdce3..c07a9e2d 100644 --- a/backend/search/base/transactions.go +++ b/backend/search/base/transactions.go @@ -3,6 +3,7 @@ package base import ( "fmt" "io" + "log" "log/slog" "net/http" @@ -18,16 +19,21 @@ import ( func doSearchGetRequest[T, V any](url string, requestBody T) (*V, error) { payload, err := json.Marshal(requestBody) if err != nil { + log.Println("json marshal failed") return nil, err } resp, err := utilities.Request(http.MethodGet, fmt.Sprintf("%s%s", constants.SEARCH_URI, url), payload, utilities.JSON()) if err != nil { + log.Println("response failed") return nil, err } defer resp.Body.Close() responseBody, err := io.ReadAll(resp.Body) + log.Println("response body VVV") + log.Println(string(responseBody)) + if err != nil { return nil, err } @@ -44,7 +50,8 @@ func doSearchGetRequest[T, V any](url string, requestBody T) (*V, error) { func Search[T types.Searchable](db *gorm.DB, query types.SearchRequest) (*types.SearchResult[T], error) { result, err := doSearchGetRequest[types.SearchEndpointRequest, types.SearchEndpointResponse](fmt.Sprintf("/%s/_search", query.Index()), query.ToSearchEndpointRequest()) if err != nil { - return nil, nil + log.Println("dosearchgetrequest failed") + return nil, err } ids := make([]string, len(result.Hits.Hits)) @@ -55,7 +62,8 @@ func Search[T types.Searchable](db *gorm.DB, query types.SearchRequest) (*types. var results []T if err = query.Preload(db).Where("id IN ?", ids).Find(&results).Error; err != nil { - return nil, nil + log.Println("query preload failed") + return nil, err } return &types.SearchResult[T]{ diff --git a/backend/server/server.go b/backend/server/server.go index a916d37c..8c10b6d3 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -57,8 +57,10 @@ func Init(db *gorm.DB, stores *store.Stores, integrations integrations.Integrati applicationURL := settings.Application.ApplicationURL() - msftProvider := msft.New(settings.Microsft.Key, settings.Microsft.Secret, fmt.Sprintf("%s/api/v1/auth/microsoftonline/callback", applicationURL), settings.Microsft.Tenant) - googProvider := goog.New(settings.Google.Key, settings.Google.Secret, fmt.Sprintf("%s/api/v1/auth/google/callback", applicationURL)) + publicURL := settings.Application.PublicUrl + + msftProvider := msft.New(settings.Microsft.Key, settings.Microsft.Secret, fmt.Sprintf("%s/api/v1/auth/microsoftonline/callback", publicURL), settings.Microsft.Tenant) + googProvider := goog.New(settings.Google.Key, settings.Google.Secret, fmt.Sprintf("%s/api/v1/auth/google/callback", publicURL)) authMiddleware := authMiddleware.New( db, diff --git a/config/.env.template b/config/.env.template index 79c692d7..bfce26fc 100644 --- a/config/.env.template +++ b/config/.env.template @@ -1,6 +1,7 @@ SAC_APPLICATION_PORT="8080" SAC_APPLICATION_HOST="127.0.0.1" SAC_APPLICATION_BASE_URL="http://127.0.0.1" +SAC_APPLICATION_PUBLIC_URL="http://127.0.0.1" SAC_DB_USERNAME="postgres" SAC_DB_PASSWORD="password" diff --git a/deployment/Caddyfile b/deployment/Caddyfile index 38e2e44c..062cdc7f 100644 --- a/deployment/Caddyfile +++ b/deployment/Caddyfile @@ -1,3 +1,3 @@ -sac.tech0tron.net { +studentactivitycalendar.xyz { reverse_proxy sac_webserver:8080 } \ No newline at end of file diff --git a/deployment/compose.yml b/deployment/compose.yml index 4d47f47c..ab9e9847 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -30,7 +30,7 @@ services: dockerfile: ../backend/Dockerfile.redis container_name: redis_session ports: - - 6380:6379 + - "6379" environment: - REDIS_USERNAME=${SAC_REDIS_SESSION_USERNAME} - REDIS_PASSWORD=${SAC_REDIS_SESSION_PASSWORD} @@ -42,8 +42,8 @@ services: context: ../backend dockerfile: ../backend/Dockerfile.redis container_name: redis_limiter - ports: - - 6381:6379 + expose: + - "6379" environment: - REDIS_USERNAME=${SAC_REDIS_LIMITER_USERNAME} - REDIS_PASSWORD=${SAC_REDIS_LIMITER_PASSWORD} @@ -74,8 +74,9 @@ services: ports: - 9200:9200 - 9600:9600 # required for Performance Analyzer - networks: - - opensearch-net + expose: + - "9200" + - "9600" opensearch-dashboards: image: opensearchproject/opensearch-dashboards:latest container_name: opensearch-dashboards @@ -86,8 +87,6 @@ services: environment: OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]' DISABLE_SECURITY_DASHBOARDS_PLUGIN: true - networks: - - opensearch-net volumes: redis-session-data: @@ -96,6 +95,3 @@ volumes: caddy_data: external: true caddy_config: - -networks: - opensearch-net: From 6a744b753371ca6e5e19440e20865193b3eda193 Mon Sep 17 00:00:00 2001 From: Michael Brennan Date: Mon, 17 Jun 2024 10:54:20 -0400 Subject: [PATCH 21/25] remove debug prints --- backend/search/base/controller.go | 2 -- backend/search/base/transactions.go | 8 -------- 2 files changed, 10 deletions(-) diff --git a/backend/search/base/controller.go b/backend/search/base/controller.go index 02909ace..02c63700 100644 --- a/backend/search/base/controller.go +++ b/backend/search/base/controller.go @@ -36,14 +36,12 @@ func (s *SearchController) SearchClubs(c *fiber.Ctx) error { var searchQuery search_types.ClubSearchRequest - log.Println("RAHHHHHH") if err := c.BodyParser(&searchQuery); err != nil { return utilities.InvalidJSON() } result, err := s.searchService.SearchClubs(searchQuery) if err != nil { - log.Println("RAHHHHHH XD") return err } diff --git a/backend/search/base/transactions.go b/backend/search/base/transactions.go index acd8118e..f9730157 100644 --- a/backend/search/base/transactions.go +++ b/backend/search/base/transactions.go @@ -3,7 +3,6 @@ package base import ( "fmt" "io" - "log" "log/slog" "net/http" @@ -20,21 +19,16 @@ import ( func doSearchGetRequest[T, V any](url string, requestBody T) (*V, error) { payload, err := json.Marshal(requestBody) if err != nil { - log.Println("json marshal failed") return nil, err } resp, err := utilities.Request(http.MethodGet, fmt.Sprintf("%s%s", constants.SEARCH_URI, url), payload, utilities.JSON()) if err != nil { - log.Println("response failed") return nil, err } defer resp.Body.Close() responseBody, err := io.ReadAll(resp.Body) - log.Println("response body VVV") - log.Println(string(responseBody)) - if err != nil { return nil, err } @@ -51,7 +45,6 @@ func doSearchGetRequest[T, V any](url string, requestBody T) (*V, error) { func Search[T types.Searchable](db *gorm.DB, query types.SearchRequest) (*types.SearchResult[T], error) { result, err := doSearchGetRequest[types.SearchEndpointRequest, types.SearchEndpointResponse](fmt.Sprintf("/%s/_search", query.Index()), query.ToSearchEndpointRequest()) if err != nil { - log.Println("dosearchgetrequest failed") return nil, err } @@ -64,7 +57,6 @@ func Search[T types.Searchable](db *gorm.DB, query types.SearchRequest) (*types. var results []T if err = query.Preload(db).Where("id IN ?", ids).Find(&results).Error; err != nil { - log.Println("query preload failed") return nil, err } From 871cd4f570e7c9a9b33057fc42ebf958f810f7a3 Mon Sep 17 00:00:00 2001 From: Generate NEU Date: Tue, 18 Jun 2024 21:09:42 +0000 Subject: [PATCH 22/25] some changes --- deployment/compose.yml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/deployment/compose.yml b/deployment/compose.yml index ab9e9847..494b0c6d 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -51,43 +51,6 @@ services: volumes: - redis-limiter-data:/data - # OPENSEARCH - opensearch-node1: - image: opensearchproject/opensearch:latest - container_name: opensearch-node1 - environment: - - cluster.name=opensearch-cluster - - node.name=opensearch-node1 - - discovery.type=single-node - - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - DISABLE_SECURITY_PLUGIN=true # - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems - hard: 65536 - volumes: - - opensearch-data1:/usr/share/opensearch/data - ports: - - 9200:9200 - - 9600:9600 # required for Performance Analyzer - expose: - - "9200" - - "9600" - opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:latest - container_name: opensearch-dashboards - ports: - - 5601:5601 - expose: - - "5601" - environment: - OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]' - DISABLE_SECURITY_DASHBOARDS_PLUGIN: true - volumes: redis-session-data: redis-limiter-data: From e56330a1220beb67153778de413fe604768e63b8 Mon Sep 17 00:00:00 2001 From: Generate NEU Date: Tue, 18 Jun 2024 23:28:05 +0000 Subject: [PATCH 23/25] fixes and changes --- backend/server/server.go | 2 +- deployment/compose.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/server/server.go b/backend/server/server.go index feaf548d..5ab61941 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -60,7 +60,7 @@ func Init(db *gorm.DB, stores *store.Stores, integrations integrations.Integrati publicURL := settings.Application.PublicUrl - msftProvider := msft.New(settings.Microsft.Key, settings.Microsft.Secret, fmt.Sprintf("%s/api/v1/auth/microsoftonline/callback", publicURL), settings.Microsft.Tenant) + msftProvider := msft.New(settings.Microsft.Key, "myapp://auth/callback", settings.Microsft.Tenant) googProvider := goog.New(settings.Google.Key, settings.Google.Secret, fmt.Sprintf("%s/api/v1/auth/google/callback", publicURL)) authMiddleware := authMiddleware.New( diff --git a/deployment/compose.yml b/deployment/compose.yml index 494b0c6d..fda9ec29 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -24,6 +24,19 @@ services: - .env.prod # REDIS + redis-db-cache: + build: + context: ../backend + dockerfile: ../backend/Dockerfile.redis + container_name: redis_db_cache + ports: + - "6379" + environment: + - REDIS_USERNAME=redis_db_cache + - REDIS_PASSWORD=redis_db_cache!#1 + - REDIS_DISABLE_DEFAULT_USER="true" + volumes: + - redis-db-cache-data:/data redis-session: build: context: ../backend @@ -54,6 +67,7 @@ services: volumes: redis-session-data: redis-limiter-data: + redis-db-cache-data: opensearch-data1: caddy_data: external: true From 2abc3efea8a78a7440fe9f9f8953b1e737bf20ac Mon Sep 17 00:00:00 2001 From: Generate NEU Date: Tue, 18 Jun 2024 23:44:25 +0000 Subject: [PATCH 24/25] dockerfile mods --- backend/Dockerfile.server | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index bf71fb2d..129823c6 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -FROM golang:1.22.2 +FROM golang:1.22.2 as builder WORKDIR /app @@ -13,6 +13,10 @@ RUN templ generate RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac +FROM alpine:latest as release + +COPY --from=builder /app/sac /sac + EXPOSE 8080 -CMD ["/app/sac"] +ENTRYPOINT [ "/sac" ] \ No newline at end of file From 862d4f409b5ae9a70ea99dcccaf2ebab9768dc3a Mon Sep 17 00:00:00 2001 From: Garrett Ladley <92384606+garrettladley@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:12:19 -0400 Subject: [PATCH 25/25] update Dockerfile.server --- backend/Dockerfile.server | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/Dockerfile.server b/backend/Dockerfile.server index 129823c6..2d8194e3 100644 --- a/backend/Dockerfile.server +++ b/backend/Dockerfile.server @@ -1,22 +1,21 @@ -# syntax=docker/dockerfile:1 - -FROM golang:1.22.2 as builder +FROM golang:1.22-alpine as builder WORKDIR /app -COPY go.mod go.sum ./ - COPY . . RUN go install github.com/a-h/templ/cmd/templ@latest + RUN templ generate -RUN CGO_ENABLED=0 GOOS=linux go build -v -o ./sac +RUN go mod download + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o bin/sac main.go FROM alpine:latest as release -COPY --from=builder /app/sac /sac +COPY --from=builder /app/bin/sac /sac EXPOSE 8080 -ENTRYPOINT [ "/sac" ] \ No newline at end of file +ENTRYPOINT [ "/sac" ]