fix: Use S3 to download nox and ccp snapshots (#32)

* Update snapshot logic * Update versions * Download snapshot from s3 * Use older versions * Update roles/nox/README.md Co-authored-by: folex <[email protected]> --------- Co-authored-by: folex <[email protected]>
fluencelabs · Jul 24, 2024 · d513e20 · d513e20
1 parent b97b4ed
commit d513e20
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 126 deletions.
diff --git a/molecule/converge.yml b/molecule/converge.yml
@@ -27,6 +27,10 @@
       include_role:
         name: nox
 
+    - name: Run ccp role
+      include_role:
+        name: ccp
+
     - name: Run promtail role
       include_role:
         name: promtail

diff --git a/roles/ccp/README.md b/roles/ccp/README.md
@@ -15,15 +15,9 @@ ansible-playbook ccp.yml -e "ccp_cleanup_state=true"
 
 ### Install ccp snapshot from PR
 
-Only for Fluence Labs members.
-
-- Go to GitHub to e2e run from your PR, for example
-  https://github.com/fluencelabs/capacity-commitment-peer/actions/runs/7409293504 - `7409293504` is
-  run id
-- Rerun role providing your `GITHUB_TOKEN` as env variable:
-    ```bash
-    GITHUB_TOKEN=<your_token> ansible-playbook ccp.yml -e "ccp_run_id=7409293504"
-    ```
+```bash
+ansible-playbook ccp.yml -e "ccp_branch=FLU-668"
+```
 
 ## Role Variables
 
@@ -92,12 +86,8 @@ It will contain everything this role creates: ccp binaries, configs and state.
     ccp_cleanup_state: false
     ```
 
-#### `ccp_run_id`
+#### `ccp_branch`
 
-- GitHub actions run id of workflow in
-  [ccp e2e run](https://github.com/fluencelabs/capacity-commitment-prover/actions/workflows/e2e.yml).
-  Used by Fluence Labs internally to install a snapshot version of ccp for
-  testing. `GITHUB_TOKEN` is required.
 - type: string
 
 ## Author

diff --git a/roles/ccp/defaults/main.yml b/roles/ccp/defaults/main.yml
@@ -3,7 +3,7 @@ ccp_dir: "/opt/fluence/ccp"
 ccp_project_dir: ""
 ccp_group: "ccp"
 ccp_user: "ccp"
-ccp_run_id: ""
+ccp_branch: ""
 ccp_cleanup_state: false
 ccp_unit_file: |
   [Unit]

diff --git a/roles/ccp/tasks/00-preflight.yml b/roles/ccp/tasks/00-preflight.yml
@@ -1,14 +1,3 @@
-- name: check that GITHUB_TOKEN is defined
-  tags: always
-  vars:
-    github_token: "{{ lookup('env', 'GITHUB_TOKEN') }}"
-  ansible.builtin.assert:
-    that:
-      - github_token is string
-      - github_token | length
-    quiet: true
-  when: ccp_run_id | string | length
-
 - name: check that tar is GNU type # noqa
   tags: always
   fluencelabs.provider.check_tar_type:

diff --git a/roles/ccp/tasks/01-install.yml b/roles/ccp/tasks/01-install.yml
@@ -31,7 +31,7 @@
   become: false
   run_once: true
   delegate_to: localhost
-  when: ccp_run_id | string | length == 0
+  when: ccp_branch | string | length == 0
   block:
     - name: create files directory
       become: false
@@ -71,53 +71,31 @@
   delegate_to: localhost
   vars:
     github_token: "{{ lookup('env', 'GITHUB_TOKEN') }}"
-  when: ccp_run_id | string | length
+  when: ccp_branch | string | length
   block:
-    - name: get list of artifacts from GitHub API
-      ansible.builtin.uri:
-        url: "https://api.github.com/repos/fluencelabs/capacity-commitment-prover/actions/runs/{{ ccp_run_id }}/artifacts"
-        method: GET
-        headers:
-          Accept: "application/vnd.github.v3+json"
-        return_content: true
-      register: _artifacts_response
-
-    - name: parse the artifact download URL
-      ansible.builtin.set_fact:
-        _artifact_download_url: "{{ item.archive_download_url }}"
-      loop: "{{ _artifacts_response.json.artifacts }}"
-      when: "item.name == _ccp_bin"
-      loop_control:
-        label: "{{ item.url }}"
-
     - name: create a temporary directory
       ansible.builtin.tempfile:
         state: directory
-      register: _artifact_temp_dir
+      register: _snapshot_temp_dir
 
-    - name: download the artifact
+    - name: download snapshot
+      vars:
+        _snapshot_download_url: "https://fluence-artifacts.s3.eu-west-1.amazonaws.com/ccp/snapshots/{{ ccp_branch }}/{{ _ccp_bin }}"
       ansible.builtin.get_url:
-        url: "{{ _artifact_download_url }}"
-        dest: "{{ _artifact_temp_dir.path }}/{{ _ccp_bin }}.zip"
-        headers:
-          Authorization: "token {{ github_token }}"
-      register: _download_artifact
-      until: _download_artifact is succeeded
+        url: "{{ _snapshot_download_url }}"
+        dest: "{{ _snapshot_temp_dir.path }}/{{ _ccp_bin }}"
+      register: _download_snapshot
+      until: _download_snapshot is succeeded
       retries: 5
       delay: 2
 
-    - name: unarchive the downloaded artifact
-      ansible.builtin.unarchive:
-        src: "{{ _artifact_temp_dir.path }}/{{ _ccp_bin }}.zip"
-        dest: "{{ _artifact_temp_dir.path }}/"
-
-    - name: propagate ccp artifact binary
+    - name: propagate ccp snapshot binary
       become: true
       become_user: root
       run_once: false
       delegate_to: "{{ inventory_hostname }}"
       ansible.builtin.copy:
-        src: "{{ _artifact_temp_dir.path }}/ccp"
+        src: "{{ _snapshot_temp_dir.path }}/{{ _ccp_bin }}"
         dest: "{{ ccp_dir }}/ccp"
         owner: "{{ ccp_user }}"
         group: "{{ ccp_group }}"
@@ -128,9 +106,9 @@
   always:
     - name: remove temporary directory
       ansible.builtin.file:
-        path: "{{ _artifact_temp_dir.path }}"
+        path: "{{ _snapshot_temp_dir.path }}"
         state: absent
-      when: _artifact_temp_dir.path is defined
+      when: _snapshot_temp_dir.path is defined
 
 - name: copy ccp config
   ansible.builtin.template:

diff --git a/roles/nox/README.md b/roles/nox/README.md
@@ -16,15 +16,9 @@ ansible-playbook nox.yml -e "nox_cleanup_state=true"
 
 ### Install nox snapshot from PR
 
-Only for Fluence Labs members.
-
-- Go to GitHub to e2e run from your PR, for example
-  https://github.com/fluencelabs/nox/actions/runs/7409293504 - `7409293504` is
-  run id
-- Rerun role providing your `GITHUB_TOKEN` as env variable:
-    ```bash
-    GITHUB_TOKEN=<your_token> ansible-playbook nox.yml -e "nox_run_id=7409293504"
-    ```
+```bash
+ansible-playbook nox.yml -e "nox_branch=FLU-688"
+```
 
 ## Role Variables
 
@@ -93,12 +87,9 @@ It will contain everything this role creates: nox binaries, configs, secrets.
     nox_cleanup_state: false
     ```
 
-#### `nox_run_id`
+#### `nox_branch`
+- whole catalogue of branches can be viewed here https://files.fluence.dev/buckets/fluence-artifacts
 
-- GitHub actions run id of workflow in
-  [nox e2e run](https://github.com/fluencelabs/nox/actions/workflows/e2e.yml).
-  Used by Fluence Labs internally to install a snapshot version of nox for
-  testing. `GITHUB_TOKEN` is required.
 - type: string
 
 ## Author

diff --git a/roles/nox/defaults/main.yml b/roles/nox/defaults/main.yml
@@ -3,7 +3,7 @@ nox_dir: "/opt/fluence/nox"
 nox_project_dir: ""
 nox_group: "nox"
 nox_user: "nox"
-nox_run_id: ""
+nox_branch: ""
 nox_cleanup_state: false
 nox_unit_file: |
   [Unit]

diff --git a/roles/nox/tasks/00-preflight.yml b/roles/nox/tasks/00-preflight.yml
@@ -1,14 +1,3 @@
-- name: check that GITHUB_TOKEN is defined
-  tags: always
-  vars:
-    github_token: "{{ lookup('env', 'GITHUB_TOKEN') }}"
-  ansible.builtin.assert:
-    that:
-      - github_token is string
-      - github_token | length
-    quiet: true
-  when: nox_run_id | string | length
-
 - name: check that tar is GNU type # noqa
   tags: always
   fluencelabs.provider.check_tar_type:

diff --git a/roles/nox/tasks/01-install.yml b/roles/nox/tasks/01-install.yml
@@ -31,7 +31,7 @@
   become: false
   run_once: true
   delegate_to: localhost
-  when: nox_run_id | string | length == 0
+  when: nox_branch | string | length == 0
   block:
     - name: create files directory
       become: false
@@ -69,55 +69,31 @@
   become: false
   run_once: true
   delegate_to: localhost
-  vars:
-    github_token: "{{ lookup('env', 'GITHUB_TOKEN') }}"
-  when: nox_run_id | string | length
+  when: nox_branch | string | length
   block:
-    - name: get list of artifacts from GitHub API
-      ansible.builtin.uri:
-        url: "https://api.github.com/repos/fluencelabs/nox/actions/runs/{{ nox_run_id }}/artifacts"
-        method: GET
-        headers:
-          Accept: "application/vnd.github.v3+json"
-        return_content: true
-      register: _artifacts_response
-
-    - name: parse the artifact download URL
-      ansible.builtin.set_fact:
-        _artifact_download_url: "{{ item.archive_download_url }}"
-      loop: "{{ _artifacts_response.json.artifacts }}"
-      when: "item.name == _nox_bin"
-      loop_control:
-        label: "{{ item.url }}"
-
     - name: create a temporary directory
       ansible.builtin.tempfile:
         state: directory
-      register: _artifact_temp_dir
+      register: _snapshot_temp_dir
 
-    - name: download the artifact
+    - name: download the snapshot
+      vars:
+        _snapshot_download_url: "https://fluence-artifacts.s3.eu-west-1.amazonaws.com/nox/snapshots/{{ nox_branch }}/{{ _nox_bin }}"
       ansible.builtin.get_url:
-        url: "{{ _artifact_download_url }}"
-        dest: "{{ _artifact_temp_dir.path }}/{{ _nox_bin }}.zip"
-        headers:
-          Authorization: "token {{ github_token }}"
+        url: "{{ _snapshot_download_url }}"
+        dest: "{{ _snapshot_temp_dir.path }}/{{ _nox_bin }}"
       register: _download_artifact
       until: _download_artifact is succeeded
       retries: 5
       delay: 2
 
-    - name: unarchive the downloaded artifact
-      ansible.builtin.unarchive:
-        src: "{{ _artifact_temp_dir.path }}/{{ _nox_bin }}.zip"
-        dest: "{{ _artifact_temp_dir.path }}/"
-
-    - name: propagate nox artifact binary
+    - name: propagate nox snapshot binary
       become: true
       become_user: root
       run_once: false
       delegate_to: "{{ inventory_hostname }}"
       ansible.builtin.copy:
-        src: "{{ _artifact_temp_dir.path }}/nox"
+        src: "{{ _snapshot_temp_dir.path }}/{{ _nox_bin }}"
         dest: "{{ nox_dir }}/nox"
         owner: "{{ nox_user }}"
         group: "{{ nox_group }}"
@@ -127,9 +103,9 @@
   always:
     - name: remove temporary directory
       ansible.builtin.file:
-        path: "{{ _artifact_temp_dir.path }}"
+        path: "{{ _snapshot_temp_dir.path }}"
         state: absent
-      when: _artifact_temp_dir.path is defined
+      when: _snapshot_temp_dir.path is defined
 
 - name: copy nox config
   ansible.builtin.template:

diff --git a/roles/nox/vars/main.yml b/roles/nox/vars/main.yml
@@ -6,7 +6,7 @@ _arch_map:
 _arch: "{{ _arch_map[ansible_architecture] }}"
 _semver_regex: "^v?\\d+\\.\\d+\\.\\d+$"
 
-_nox_version: "{{ nox_version | regex_replace('^(?!v)', 'v') }}"
-_nox_download_url: "https://github.com/fluencelabs/nox/releases/download/nox-{{ _nox_version }}"
+_nox_version: "{{ nox_version | regex_replace('^(?!v)', '') }}"
+_nox_download_url: "https://fluence-artifacts.s3.eu-west-1.amazonaws.com/nox/{{ _nox_version }}"
 _nox_checksums: "sha256:{{ _nox_download_url + '/nox_SHA256_SUMS' }}"
 _nox_bin: "nox-{{ _arch }}"