From 55130e73c7be69c3c85d83b4fa20fc5ff9755fdd Mon Sep 17 00:00:00 2001 From: Kyle Somers <17753062+kylesomers@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:53:08 -0500 Subject: [PATCH] docs: consolidate Ansible playbooks under assets (#117) --- .github/workflows/docs.yml | 2 +- Makefile | 8 +- .../ansible-playbooks}/README.md | 0 .../perforce/helix-core/commit_playbook.yml | 0 .../perforce/helix-core/replica_playbook.yml | 0 assets/packer/.ci/Dockerfile | 25 ++++++- assets/packer/.ci/README.md | 74 +++++++++++++++++++ assets/packer/.config.yml | 2 +- assets/packer/build-agents/README.md | 2 +- docs/assets/dockerfiles.md | 1 + docs/{assets.md => assets/index.md} | 15 +--- docs/assets/jenkins-pipelines.md | 1 + docs/assets/packer.md | 15 ++++ docs/assets/playbooks.md | 1 + docs/ci/index.md | 14 ++++ docs/ci/packer.md | 1 + docs/contributing.md | 1 + docs/index.md | 25 ++----- docs/{modules.md => modules/index.md} | 0 docs/modules/jenkins/index.md | 1 + .../perforce/helix-authentication-service.md | 1 + docs/modules/perforce/helix-core.md | 1 + docs/modules/perforce/helix-swarm.md | 1 + docs/modules/perforce/index.md | 0 docs/playbooks.md | 1 - docs/{samples.md => samples/index.md} | 0 docs/samples/simple-build-pipeline.md | 1 + mkdocs.yml | 30 ++++++-- modules/README.md | 15 ++-- modules/jenkins/README.md | 2 +- .../helix-authentication-service/README.md | 11 ++- modules/perforce/helix-core/README.md | 4 +- samples/README.md | 2 +- 33 files changed, 199 insertions(+), 58 deletions(-) rename {playbooks => assets/ansible-playbooks}/README.md (100%) rename {playbooks => assets/ansible-playbooks}/perforce/helix-core/commit_playbook.yml (100%) rename {playbooks => assets/ansible-playbooks}/perforce/helix-core/replica_playbook.yml (100%) create mode 100644 assets/packer/.ci/README.md create mode 100644 docs/assets/dockerfiles.md rename docs/{assets.md => assets/index.md} (61%) create mode 100644 docs/assets/jenkins-pipelines.md create mode 100644 docs/assets/packer.md create mode 100644 docs/assets/playbooks.md create mode 100644 docs/ci/index.md create mode 100644 docs/ci/packer.md rename docs/{modules.md => modules/index.md} (100%) create mode 100644 docs/modules/jenkins/index.md create mode 100644 docs/modules/perforce/helix-authentication-service.md create mode 100644 docs/modules/perforce/helix-core.md create mode 100644 docs/modules/perforce/helix-swarm.md create mode 100644 docs/modules/perforce/index.md delete mode 100644 docs/playbooks.md rename docs/{samples.md => samples/index.md} (100%) create mode 100644 docs/samples/simple-build-pipeline.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f4e60f5..62b8cb7f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} diff --git a/Makefile b/Makefile index 924ab14d..64a5b50f 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ docs-build: ## Build the docs using docker. Example: `make docs-build VERSION=v1 docker build -f ./docs/Dockerfile -t docs:$(VERSION) . \ --build-arg GIT_USER_NAME="$(GIT_USER_NAME)" \ --build-arg GIT_USER_EMAIL="$(GIT_USER_EMAIL)" \ - --build-arg GITHUB_ACTIONS=$(GITHUB_ACTIONS) \ + --build-arg GITHUB_ACTIONS="$(GITHUB_ACTIONS)" \ --no-cache .PHONY: docs-deploy @@ -38,11 +38,11 @@ docs-deploy: ## Build and deploy the docs using 'mike'. Example: `make docs-depl docker build -f ./docs/Dockerfile -t docs:$(VERSION) . \ --build-arg GIT_USER_NAME="$(GIT_USER_NAME)" \ --build-arg GIT_USER_EMAIL="$(GIT_USER_EMAIL)" \ - --build-arg GITHUB_ACTIONS=$(GITHUB_ACTIONS) \ + --build-arg GITHUB_ACTIONS="$(GITHUB_ACTIONS)" \ --no-cache docker run -t docs:$(VERSION) mike deploy $(VERSION) ${ALIAS} --update-aliases docker run -t docs:$(VERSION) mike set-default $(ALIAS) --push - + .PHONY: docs-local-docker docs-local-docker: ## Build and run the docs locally using docker and 'serve'. Example: `make docs-local-docker VERSION=v1.0.0` @if [ -z "${VERSION}" ]; then echo -e "${RED}VERSION is not set. Example: 'make docs-local-docker VERSION=v1.0.0'. Run 'make help' for usage. ${RESET}"; exit 1; fi @@ -50,7 +50,7 @@ docs-local-docker: ## Build and run the docs locally using docker and 'serve'. E docker build -f ./docs/Dockerfile -t docs:$(VERSION) . \ --build-arg GIT_USER_NAME="$(GIT_USER_NAME)" \ --build-arg GIT_USER_EMAIL="$(GIT_USER_EMAIL)" \ - --build-arg GITHUB_ACTIONS=$(GITHUB_ACTIONS) \ + --build-arg GITHUB_ACTIONS="$(GITHUB_ACTIONS)" --no-cache docker run --rm -it -p 8000:8000 -v ${PWD}:/docs docs:$(VERSION) mkdocs serve --dev-addr=0.0.0.0:8000 diff --git a/playbooks/README.md b/assets/ansible-playbooks/README.md similarity index 100% rename from playbooks/README.md rename to assets/ansible-playbooks/README.md diff --git a/playbooks/perforce/helix-core/commit_playbook.yml b/assets/ansible-playbooks/perforce/helix-core/commit_playbook.yml similarity index 100% rename from playbooks/perforce/helix-core/commit_playbook.yml rename to assets/ansible-playbooks/perforce/helix-core/commit_playbook.yml diff --git a/playbooks/perforce/helix-core/replica_playbook.yml b/assets/ansible-playbooks/perforce/helix-core/replica_playbook.yml similarity index 100% rename from playbooks/perforce/helix-core/replica_playbook.yml rename to assets/ansible-playbooks/perforce/helix-core/replica_playbook.yml diff --git a/assets/packer/.ci/Dockerfile b/assets/packer/.ci/Dockerfile index fb0983cc..71fe3942 100644 --- a/assets/packer/.ci/Dockerfile +++ b/assets/packer/.ci/Dockerfile @@ -4,11 +4,16 @@ FROM ubuntu:latest ARG AWS_REGION ARG AWS_VPC_ID ARG AWS_SUBNET_ID +<<<<<<< HEAD +ARG AWS_PROFILE +ARG PUBLIC_KEY +======= ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY ARG AWS_SESSION_TOKEN ARG PUBLIC_KEY ARG TEMPLATE +>>>>>>> main # Install updates and tools RUN apt-get update -y && apt-get install -y \ @@ -19,12 +24,15 @@ RUN apt-get update -y && apt-get install -y \ curl \ jq +<<<<<<< HEAD +======= # Download and install AWS CLI v2 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ && unzip awscliv2.zip \ && ./aws/install \ && aws --version +>>>>>>> main # Download and install the latest version of Packer RUN PACKER_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/packer | jq -r -M '.current_version') \ && wget https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip \ @@ -32,6 +40,14 @@ RUN PACKER_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/packe && mv packer /usr/local/bin \ && rm packer_${PACKER_VERSION}_linux_amd64.zip +<<<<<<< HEAD +# Download and install yq +RUN YQ_LATEST_VERSION=$(curl --silent "https://api.github.com/repos/mikefarah/yq/releases/latest" | jq -r .tag_name) && \ + curl -L https://github.com/mikefarah/yq/releases/download/${YQ_LATEST_VERSION}/yq_linux_amd64 -o /usr/bin/yq && \ + chmod +x /usr/bin/yq + +======= +>>>>>>> main # Copy packer directory to the container WORKDIR / COPY . /cloud-game-development-toolkit @@ -39,6 +55,12 @@ COPY . /cloud-game-development-toolkit # Create Packer variables file WORKDIR /cloud-game-development-toolkit RUN chmod +x .ci/setup.sh && .ci/setup.sh +<<<<<<< HEAD + +# Validate the Packer template files +WORKDIR /cloud-game-development-toolkit +RUN chmod +x .ci/packer-validate.sh && .ci/packer-validate.sh +======= RUN cat << EOF > ci.pkrvars.hcl \ region = "${AWS_REGION}" \ vpc_id = "${AWS_VPC_ID}" \ @@ -51,4 +73,5 @@ RUN cat << EOF > ci.pkrvars.hcl \ WORKDIR /cloud-game-development-toolkit RUN packer init ${TEMPLATE} RUN packer validate ${TEMPLATE} -#RUN packer build ${TEMPLATE} \ No newline at end of file +#RUN packer build ${TEMPLATE} +>>>>>>> main diff --git a/assets/packer/.ci/README.md b/assets/packer/.ci/README.md new file mode 100644 index 00000000..b8a44e28 --- /dev/null +++ b/assets/packer/.ci/README.md @@ -0,0 +1,74 @@ +# Packer CI + +This README provides steps for running basic CI on Packer templates contained in the project. + +## Configuration + +In the root of the `./packer` directory is a configuration yaml file (`.config.yml`) that contains the location of each of the Packer templates in the project. When new Packer templates are added, the config file should be updated as well. + + +### `.config.yml` + +``` title=".confi.yml" +--8<-- "assets/packer/.config.yml" +``` + +```bash +docker build -t packer-ci -f .ci/Dockerfile . \ +--build-arg AWS_REGION=us-east-1 \ +--build-arg AWS_VPC_ID=vpc-086839c0e28ad1f29 \ +--build-arg AWS_SUBNET_ID=subnet-0e6bb0e5c155610c0 \ +--build-arg AWS_PROFILE=default \ +--build-arg PUBLIC_KEY=Key_Pair_Linux +``` + + +## Packer Directory + +``` +. +├── build-agents +│ ├── linux +│ │ ├── amazon-linux-2023-arm64.pkr.hcl +│ │ ├── amazon-linux-2023-x86_64.pkr.hcl +│ │ ├── create_swap.service +│ │ ├── create_swap.sh +│ │ ├── example.pkrvars.hcl +│ │ ├── fsx_automounter.py +│ │ ├── fsx_automounter.service +│ │ ├── install_common.al2023.sh +│ │ ├── install_common.ubuntu.sh +│ │ ├── install_mold.sh +│ │ ├── install_octobuild.al2023.x86_64.sh +│ │ ├── install_octobuild.ubuntu.x86_64.sh +│ │ ├── install_sccache.sh +│ │ ├── mount_ephemeral.service +│ │ ├── mount_ephemeral.sh +│ │ ├── octobuild.conf +│ │ ├── README.md +│ │ ├── sccache.service +│ │ ├── ubuntu-jammy-22.04-amd64-server.pkr.hcl +│ │ └── ubuntu-jammy-22.04-arm64-server.pkr.hcl +│ ├── README.md +│ └── windows +│ ├── base_setup.ps1 +│ ├── example.pkrvars.hcl +│ ├── install_vs_tools.ps1 +│ ├── setup_jenkins_agent.ps1 +│ ├── userdata.ps1 +│ └── windows.pkr.hcl +├── .ci +│ ├── Dockerfile +│ ├── packer-ci.md +│ ├── packer-validate.sh +│ ├── README.md +│ └── setup.sh +├── .config.yml +└── perforce + └── helix-core + ├── example.pkrvars.hcl + ├── p4_configure.sh + ├── p4_setup.sh + ├── perforce.pkr.hcl + └── README.md +``` diff --git a/assets/packer/.config.yml b/assets/packer/.config.yml index e337dcf3..e5690ef0 100644 --- a/assets/packer/.config.yml +++ b/assets/packer/.config.yml @@ -17,4 +17,4 @@ packer_templates: dir: 'build-agents/windows' - description: 'Perforce Helix Core Server (x86)' file_name: 'perforce.pkr.hcl' - dir: 'perforce/helix-core' \ No newline at end of file + dir: 'perforce/helix-core' diff --git a/assets/packer/build-agents/README.md b/assets/packer/build-agents/README.md index 474e5e2e..231b73b0 100644 --- a/assets/packer/build-agents/README.md +++ b/assets/packer/build-agents/README.md @@ -12,4 +12,4 @@ Packer templates to create machine images for Unreal Engine build agents. #### Unreal Engine Build Agent (Windows) -Documentation for UE Build Agents for Windows \ No newline at end of file +Documentation for UE Build Agents for Windows. diff --git a/docs/assets/dockerfiles.md b/docs/assets/dockerfiles.md new file mode 100644 index 00000000..09d6af25 --- /dev/null +++ b/docs/assets/dockerfiles.md @@ -0,0 +1 @@ +# Dockerfiles diff --git a/docs/assets.md b/docs/assets/index.md similarity index 61% rename from docs/assets.md rename to docs/assets/index.md index eb54d255..4942325f 100644 --- a/docs/assets.md +++ b/docs/assets/index.md @@ -1,6 +1,6 @@ # Assets -**Assets** are reusable scripts, pipeline definitions, Dockerfiles, Packer templates, and other resources that might prove useful or are dependencies of any of the modules. Each asset contains its own documentation as a `README.md` file. +**Assets** are reusable scripts, pipeline definitions, Dockerfiles, Packer templates, and other resources that might prove useful or are dependencies of any of the modules. Each asset contains its own documentation as a `README.md` file. !!! info **Don't see an asset listed?** Create a [feature request](https://github.com/aws-games/cloud-game-development-toolkit/issues/new?assignees=&labels=feature-request&projects=&template=feature_request.yml&title=Feature+request%3A+TITLE) for a new asset or learn [how to contribute new assets to the project below](#Contribute new Assets to the Cloud Game Development Toolkit) @@ -9,19 +9,12 @@ Packer templates provide an easy way to build machine images for commonly used game dev infrastructure. Currently the project includes Packer templates for UE5 build agents for Linux and Windows, as well as a Packer template for building a Perforce Helix Core version control AMI. -- **[Unreal Engine Build Agents](#unreal-engine-build-agents)**: Packer templates for building Unreal Engine on Windows and Linux, including support for x86 and ARM (i.e. Graviton). -- **[Perforce Helix Core AMI](#perforce-helix-core-packer-template)**: Packer templates for creating an Amazon Machine Image (AMI) of Perforce Helix Core on Linux and configuration with Perforce Server Deployment Package (SDP) - - ---8<-- "assets/packer/build-agents/README.md" - - ---8<-- "assets/packer/perforce/helix-core/README.md" +[Packer Template Documentation](./packer.md) ## Dockerfiles -Dockerfiles for creating Docker images of commonly used game dev infrastructure. +Dockerfiles for creating Docker images of commonly used game dev infrastructure. ## Contribute new Assets to the Cloud Game Development Toolkit -This section will contain documentation about how to create new assets, how they should be tested and documented, and the process for submitting them as PRs to the project. \ No newline at end of file +This section will contain documentation about how to create new assets, how they should be tested and documented, and the process for submitting them as PRs to the project. diff --git a/docs/assets/jenkins-pipelines.md b/docs/assets/jenkins-pipelines.md new file mode 100644 index 00000000..27a4f300 --- /dev/null +++ b/docs/assets/jenkins-pipelines.md @@ -0,0 +1 @@ +# Jenkins Pipelines \ No newline at end of file diff --git a/docs/assets/packer.md b/docs/assets/packer.md new file mode 100644 index 00000000..ddf3db27 --- /dev/null +++ b/docs/assets/packer.md @@ -0,0 +1,15 @@ +# Packer Templates + + + +- **[Unreal Engine Build Agents](#unreal-engine-build-agents)**: Packer templates for building Unreal Engine on Windows and Linux, including support for x86 and ARM (i.e. Graviton). +- **[Perforce Helix Core AMI](#perforce-helix-core-packer-template)**: Packer templates for creating an Amazon Machine Image (AMI) of Perforce Helix Core on Linux and configuration with Perforce Server Deployment Package (SDP) + + +--8<-- "assets/packer/build-agents/README.md" + + +--8<-- "assets/packer/perforce/helix-core/README.md" + + +--8<-- "assets/packer/.ci/README.md" \ No newline at end of file diff --git a/docs/assets/playbooks.md b/docs/assets/playbooks.md new file mode 100644 index 00000000..3fe66047 --- /dev/null +++ b/docs/assets/playbooks.md @@ -0,0 +1 @@ +--8<-- "assets/ansible-playbooks/README.md" \ No newline at end of file diff --git a/docs/ci/index.md b/docs/ci/index.md new file mode 100644 index 00000000..26f8c8d8 --- /dev/null +++ b/docs/ci/index.md @@ -0,0 +1,14 @@ +# CI and Testing + +This project uses Github Actions to automate continuous integration (CI) testing using utilities contained in the `.ci/` directories within the project's assets, modules, and samples. Dockerfiles are included to simplify running these CI workflows locally in your development environment or in a cloud CI environment. + +## Example CI directory: `packer/.ci/` + +```shell +.ci/ +├── Dockerfile <------------------ Dockerfile for running Packer CI +├── packer-validate.sh <---------- Script for linting with Packer Validate +├── README.md <------------------- Instructions for Packer CI +└── setup.sh <-------------------- Commands to setup environment (i.e install Packer) +``` + diff --git a/docs/ci/packer.md b/docs/ci/packer.md new file mode 100644 index 00000000..2550a789 --- /dev/null +++ b/docs/ci/packer.md @@ -0,0 +1 @@ +--8<-- "assets/packer/.ci/README.md" \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md index e69de29b..e079654f 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -0,0 +1 @@ +--8<-- "CONTRIBUTING.md" \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index f489eedb..02502610 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,16 +1,13 @@ ---- -title: Home -description: Cloud Game Development Toolkit ---- +# Welcome to the Cloud Game Development Toolkit The **Cloud Game Development Toolkit (a.k.a. CGD Toolkit)** is a collection of templates and configurations for deploying game development infrastructure and tools on AWS. !!! info - **This project is under active development** and we have yet to solve for many developer needs. If you would like to see something in this repository please create a feature request in the Issues tab, or raise a pull request. You'll find our contribution guidelines [here](./contributing.md). + **This project is under active development and community contributions are welcomed!**. If you would like to see something in this repository please create a feature request in the Issues tab. If you'd like to contribute, raise a pull request. You'll find our contribution guidelines [here](./contributing.md). ## Introduction -The **CGD Toolkit** consists of four key components: +The **CGD Toolkit** consists of three key components: