From a821473f513b5c19b201c0b978300a8b92fc0915 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 25 Feb 2024 20:56:30 +0100 Subject: [PATCH 1/3] add docs workflow --- .github/workflows/docs.yaml | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/docs.yaml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..5709fb83 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,49 @@ +name: Documentation + +on: + pull_request: + paths: + - 'docs/**' + - '.github/workflows/docs.yml' + push: + branches: + - develop + paths: + - 'docs/**' + - '.github/workflows/docs.yml' + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ammaraskar/sphinx-action@master + with: + docs-folder: "docs/" + build-command: make html dirhtml + - uses: actions/upload-artifact@v3 + with: + name: Documentation + path: docs/build/html/ + - uses: actions/upload-pages-artifact@v2 + with: + path: docs/build/dirhtml/ + + # Deployment job + deploy: + if: github.ref == 'refs/heads/develop' + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: docs + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file From 5fbe2358c6b421fceb5149c6bb878a8a292690da Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 25 Feb 2024 21:05:40 +0100 Subject: [PATCH 2/3] fix secfinding --- api/requirements.in | 3 +-- api/requirements.txt | 10 ++++------ cli/requirements.in | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/api/requirements.in b/api/requirements.in index dbb9574b..d61d2730 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -1,6 +1,5 @@ #run to update: pip-compile --output-file requirements.txt ../cli/requirements.in requirements.in -fastapi -starlette +fastapi>=0.110.0 pydantic uvicorn gunicorn \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt index 58dbeb08..61c97a0c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -35,13 +35,13 @@ docker==7.0.0 # via -r ../cli/requirements.in email-validator==2.1.0.post1 # via pydantic -fastapi==0.109.1 +fastapi==0.110.0 # via -r requirements.in genson==1.2.2 # via datamodel-code-generator gitdb==4.0.11 # via gitpython -gitpython==3.1.41 +gitpython==3.1.42 # via -r ../cli/requirements.in gunicorn==21.2.0 # via -r requirements.in @@ -124,10 +124,8 @@ smmap==5.0.1 # via gitdb sniffio==1.3.0 # via anyio -starlette==0.35.1 - # via - # -r requirements.in - # fastapi +starlette==0.36.3 + # via fastapi types-pyyaml==6.0.12.12 # via -r ../cli/requirements.in types-requests==2.31.0.20240125 diff --git a/cli/requirements.in b/cli/requirements.in index 73708096..2d2dbb75 100644 --- a/cli/requirements.in +++ b/cli/requirements.in @@ -11,7 +11,7 @@ docker pytest pydantic coverage -gitpython>=3.1.41 +gitpython>=3.1.42 types-requests urllib3 python-jenkins From 142bc42801a03a47904118104763fe8d99af0245 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 25 Feb 2024 21:47:12 +0100 Subject: [PATCH 3/3] add some words --- .github/workflows/docs.yaml | 10 +++--- api/main.py | 2 +- docs/requirements.txt | 8 +++-- docs/setup/index.rst | 63 ++++++++++++++++++++++++++++++++-- docs/targets/bamboo/index.rst | 18 ++++++---- docs/targets/bamboo/setup.rst | 16 --------- docs/targets/cli/index.rst | 21 +++++++++++- docs/targets/jenkins/index.rst | 6 +++- docs/user_guide/index.rst | 16 ++++++++- 9 files changed, 123 insertions(+), 37 deletions(-) delete mode 100644 docs/targets/bamboo/setup.rst diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 5709fb83..d920112a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -23,17 +23,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: ammaraskar/sphinx-action@master + - uses: ammaraskar/sphinx-action@0.4 with: docs-folder: "docs/" build-command: make html dirhtml - uses: actions/upload-artifact@v3 with: - name: Documentation - path: docs/build/html/ - - uses: actions/upload-pages-artifact@v2 + name: documentation + path: docs/_build/html/ + - uses: actions/upload-pages-artifact@v3 with: - path: docs/build/dirhtml/ + path: docs/_build/dirhtml/ # Deployment job deploy: diff --git a/api/main.py b/api/main.py index c877b960..a046c88e 100644 --- a/api/main.py +++ b/api/main.py @@ -37,7 +37,7 @@ app = FastAPI() -origins = ["http://localhost", "http://localhost:3000", "https://aeolus.resch.io", "http://localhost:9000"] +origins = ["http://localhost", "http://localhost:3000", "http://localhost:9000"] app.add_middleware( CORSMiddleware, diff --git a/docs/requirements.txt b/docs/requirements.txt index 4a0d0bae..85389720 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -36,12 +36,14 @@ packaging==23.2 # via # build # sphinx -pip-tools==7.3.0 +pip-tools==7.4.0 # via -r requirements.in pygments==2.17.2 # via sphinx pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools requests==2.31.0 # via sphinx six==1.16.0 @@ -74,7 +76,7 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx tornado==6.4 # via livereload -urllib3==2.2.0 +urllib3==2.2.1 # via requests wheel==0.42.0 # via pip-tools diff --git a/docs/setup/index.rst b/docs/setup/index.rst index e9da9523..69cd8c46 100644 --- a/docs/setup/index.rst +++ b/docs/setup/index.rst @@ -1,6 +1,6 @@ -***** -Setup -***** +********* +Languages +********* Aeolus is implemented using different components, each of which is responsible for a specific part of the system. The main components, structured by their language, are: @@ -17,3 +17,60 @@ Aeolus is implemented using different components, each of which is responsible f * **Bamboo Generator**: A tool that generates the Bamboo configuration files from the CI configuration files. +*************** +Local Dev Setup +*************** + +To set up a local development environment, you will need to use a virtual environment, we use `venv` with `pip-tools` for this purpose. + +To create a virtual environment, follow the official Python documentation: https://docs.python.org/3/library/venv.html + +After creating the virtual environment, you can install the dependencies using pip: + +.. code-block:: bash + + cd api + pip install -r requirements.txt + +.. code-block:: bash + + cd cli + pip install -r requirements.txt + +After installing the dependencies, you can use the CLI using the following commands: + +.. code-block:: bash + + cd cli + python main.py --help + +The cli will guide you through the available commands. +How updating the requirements.txt works, is described in the two files respectively. + +To run the API, you can use the following command: + +.. code-block:: bash + + cd api + uvicorn --reload --host 127.0.0.1 --port 8090 main:app --reload + +This will start the API on http://127.0.0.1:8090. + +To run the Aeolus Playground, you can use the following command: + +.. code-block:: bash + + cd playground + npm install + npm start + +To run the Bamboo Generator, you can use the following command: + +.. code-block:: bash + + cd bamboo-generator + ./gradlew shadowJar --no-daemon -x :generateJsonSchema2Pojo + cp ./build/libs/bamboo-generator*-all.jar bamboo-generator.jar + java -jar bamboo-generator.jar + +You can ofcourse also use the IDE of your choice to run, develop, and debug the different components, the above commands are just examples. \ No newline at end of file diff --git a/docs/targets/bamboo/index.rst b/docs/targets/bamboo/index.rst index 79be6380..24d2ed32 100644 --- a/docs/targets/bamboo/index.rst +++ b/docs/targets/bamboo/index.rst @@ -2,12 +2,6 @@ Bamboo ****** -.. toctree:: - :maxdepth: 3 - :caption: Bamboo Setup - - setup - Bamboo is an Atlassian product for CI/CD. Within Bamboo, you use so called Build Plans to define the steps that are executed to build and deploy your software. @@ -15,5 +9,17 @@ Aeolus uses the Bamboo YAML Specs plugin to generate build plans from the input, usable with Java, the subsystem is implemented in said language and can be used as a standalone REST API or as a container that is started if the target system is Bamboo. ++++++++++++++++++++ +Bamboo Subcomponent ++++++++++++++++++++ + +The Bamboo subcomponent is a small Java Spring Boot application that generates the Bamboo build plan configuration for +Bamboo. It is a simple RESTful service that takes a JSON payload and returns the build plan in a JSON format or directly +generates the build plan in Bamboo. + ## What does Aeolus generate? +Aeolus always tries to generate a simple and clean build plan, if the windfile contains a single docker configuration, +the generated build plan will be a single stage that uses the defined docker image and runs the defined commands inside it. +Whenever the windfile contains multiple docker configurations, Aeolus will generate a multi-stage build plan, where each +stage is a separate docker build and run step, the same applies if some actions need to be skipped or executed conditionally. diff --git a/docs/targets/bamboo/setup.rst b/docs/targets/bamboo/setup.rst deleted file mode 100644 index 33107da5..00000000 --- a/docs/targets/bamboo/setup.rst +++ /dev/null @@ -1,16 +0,0 @@ -*********************** -Bamboo Specs Generation -*********************** - -Bamboo is an Atlassian product for CI/CD. -Within Bamboo, you use so called Build Plans to define the steps that are executed to build and deploy your software. - -Aeolus uses the Bamboo YAML Specs plugin to generate build plans from the input, as the plugin is only -usable with Java, the subsystem is implemented in said language and can be used as a standalone REST API or as a -container that is started if the target system is Bamboo. - -************************** -What does Aeolus generate? -************************** - -Aeolus generates \ No newline at end of file diff --git a/docs/targets/cli/index.rst b/docs/targets/cli/index.rst index 097d3d0a..234eefeb 100644 --- a/docs/targets/cli/index.rst +++ b/docs/targets/cli/index.rst @@ -2,5 +2,24 @@ CLI *** -## What does Aeolus generate? +Aeolus can generate scripts that can be used to automate the deployment of a project. The script is generated based on +the input, the windfile. +++++++++++++++++++++++++++ +What does Aeolus generate? +++++++++++++++++++++++++++ + +Aeolus generates a bash script using best practices to provide a similar workflow to Jenkins and Bamboo. To ensure that +each action is executed in the correct order, the script is divided into functions. Each function contains the commands +of a single defined action and is called in the `main` function. + ++++++++++++++++++++++++ +Structure of the script ++++++++++++++++++++++++ + +To ensure that the script is executed correctly, Aeolus uses the `set -e` command to stop the execution of the script if +commands fail. This is useful to avoid the execution of the next commands if the previous ones failed. Aeolus also offers +the possibility to define actions that always need to be executed, regardless of the result of the previous commands. +For this we use the `trap` command that captures the exit signal and executes the defined action. To also ensure that +an `exit` command within an action does not break the script, we source the script instead of executing it and then execute +the `main` function which spawns a subshell for each action. \ No newline at end of file diff --git a/docs/targets/jenkins/index.rst b/docs/targets/jenkins/index.rst index aa6b7582..5fcb4cad 100644 --- a/docs/targets/jenkins/index.rst +++ b/docs/targets/jenkins/index.rst @@ -2,5 +2,9 @@ Jenkins ******* -## What does Aeolus generate? +++++++++++++++++++++++++++ +What does Aeolus generate? +++++++++++++++++++++++++++ +Aeolus generates a simple Jenkinsfile that is then used in a simple pipeline. Each action from the input file is +translated into a stage in the Jenkinsfile. \ No newline at end of file diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 768083f2..0090f1d9 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -2,5 +2,19 @@ Using Aeolus ************ -## How do I run this? +++++++++++++++++++ +How do I run this? +++++++++++++++++++ +It's easy! Just run the following command: + +.. code-block:: bash + + cd cli/ + python3 main.py + +or if you prefer docker: + +.. code-block:: bash + + docker run --rm -v "./:/tmp" ghcr.io/ls1intum/aeolus/cli:nightly --help \ No newline at end of file