Merge branch 'hyperledger:iroha2-dev' into iroha2-dev

.dockerignore

@@ -1,2 +1,2 @@
 **/target
-Dockerfile
+Dockerfile*

.github/scripts/ci_test/ci_image_scan.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+"""
+CI script for locating the improperly configured images
+in Docker's Compose files.
+
+Scans a list of file masks/names and checks for allowed branches.
+"""
+
+from typing import List
+import sys
+from logging import getLogger, warning, error, info, INFO
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+from yaml import safe_load
+from yaml.error import YAMLError
+from git_root import git_root
+
+def parse_arguments() -> Namespace:
+    """
+    Returns:
+        Namespace: An object containing two attributes:
+            - masks: A list of file masks and names provided as positional arguments.
+            - allow: A list of Docker images provided
+                     to the 'allow' option, or [] if not provided.
+    """
+    parser = ArgumentParser(description='Process some file names.')
+    parser.add_argument(
+        '--allow',
+        nargs='+',
+        help='one or more allowed image names',
+        default=[]
+    )
+    parser.add_argument(
+        'masks',
+        nargs='*',
+        help='list of file masks and exact names to be checked',
+        default=[]
+    )
+    return parser.parse_args()
+
+def get_paths(file_masks: List[str], root: Path):
+    """
+    Generate a list of pathlib.Path instances for given file masks
+    and filenames within a root directory.
+
+    This function searches for files in a specified root directory
+    matching the patterns and filenames provided in `file_masks`.
+    It returns a list of pathlib.Path instances for files that exist.
+    Patterns can include wildcards (e.g., "*.yml").
+    Only files that actually exist in the filesystem are included in the result.
+
+    Args:
+        file_masks (list of str): A list of strings representing file masks
+                                  and filenames.
+                                  File masks can include wildcard characters
+                                  (e.g., "topic.*.yml").
+        root (pathlib.Path):
+                                  A pathlib.Path instance representing
+                                  the root directory in which to search for files.
+
+    Returns:
+        list: A list containing pathlib.Path instances for each existing 
+              file matching the file masks and filenames
+              in the specified root directory.
+
+    Raises:
+        TypeError: If `root` is not an instance of pathlib.Path.
+
+    Note:
+        The function does not return paths for files that do not exist.
+    """
+    if not isinstance(root, Path):
+        raise TypeError("The root argument must be a pathlib.Path instance")
+    paths = []
+    for mask in file_masks:
+        if '*' in mask:
+            matching_files = root.glob(mask)
+            paths.extend([file for file in matching_files if file.exists()])
+        else:
+            path = root / mask
+            if path.exists():
+                paths.append(path)
+            else:
+                warning(f'File not found: {path.name}')
+    return paths
+
+def validate_docker_config(compose_file: Path, allow: List[str]):
+    """
+    Validates a single Path for a Compose config.
+
+    Returns:
+        (int) 1 if the image is using a config that isn't allowed,
+              0 otherwise
+    """
+    status = 0
+    services = {}
+    try:
+        with open(compose_file, 'r', encoding='utf8') as compose_file_contents:
+            config_inst = safe_load(compose_file_contents.read())
+            if isinstance(config_inst, dict):
+                services = config_inst.get('services', {})
+            else:
+                error(f'Improper configuration at "{compose_file}"')
+                status = 1
+    except YAMLError:
+        error(f'Improper formatting at "{compose_file}"')
+        status = 1
+    for _, service in services.items():
+        if service.get('image', '') not in allow:
+            status = 1
+            break
+    return status
+
+def main():
+    """
+    Validate the supplied Docker configurations
+    and exit with an error if one of them is using
+    an incorrect image.
+    """
+    getLogger().setLevel(INFO)
+    args = parse_arguments()
+    for current_file in get_paths(args.masks, git_root()):
+        if validate_docker_config(current_file, args.allow):
+            warning(f'Wrong image in "{current_file.name}"')
+            sys.exit(1)
+    info('No incorrect Compose configurations found')
+
+if __name__ == '__main__':
+    main()

.github/scripts/ci_test/git_root.py

@@ -0,0 +1,19 @@
+"""
+This module contains a modified copy of git_root by Jan Tilly.
+It allows to find a repo root on GitHub for the CI purposes.
+
+https://github.com/jtilly/git_root/blob/master/git_root/git_root.py
+"""
+
+from subprocess import Popen, PIPE, DEVNULL
+from os.path import abspath
+from pathlib import Path
+
+def git_root():
+    root = '.'
+    with Popen(
+        ['git', 'rev-parse', '--show-toplevel'],
+        stdout=PIPE, stderr=DEVNULL
+    ) as git_proc:
+        root = git_proc.communicate()[0].rstrip().decode('utf-8')
+    return Path(abspath(root))

.github/scripts/ci_test/requirements.txt

@@ -0,0 +1 @@
+PyYAML==6.0.1

.github/workflows/iroha2-ci-image.yml

@@ -1,6 +1,11 @@
 name: I2::CI::Publish
 
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+    inputs:
+      IROHA2_CI_DOCKERFILE:
+        required: true
+        default: Dockerfile.build
 
 jobs:
   dockerhub:
@@ -17,6 +22,6 @@ jobs:
           push: true
           tags: hyperledger/iroha2-ci:nightly-2024-01-12
           labels: commit=${{ github.sha }}
-          file: Dockerfile.build
+          file: ${{ github.event.inputs.IROHA2_CI_DOCKERFILE }}
           # This context specification is required
           context: .

.github/workflows/iroha2-dev-pr-wasm.yaml

@@ -33,4 +33,4 @@ jobs:
         run: cargo install --path tools/wasm_test_runner
       - name: Run smart contract tests on WebAssembly VM
         working-directory: smart_contract
-        run: mold --run cargo test --tests --target wasm32-unknown-unknown --no-fail-fast --quiet
+        run: mold --run cargo test --release --tests --target wasm32-unknown-unknown --no-fail-fast --quiet

.github/workflows/iroha2-dev-pr.yml

@@ -69,6 +69,13 @@ jobs:
           compare-ref: ${{ github.base_ref }}
           compare-sha: ${{ github.event.pull_request.base.sha}}
           github-token: ${{ secrets.GITHUB_TOKEN }}
+      # (Temporally) Add the parallel coverage upload to Codecov to compare the results with Coveralls
+      # - name: Upload coverage to codecov.io
+      #   uses: codecov/[email protected]
+      #   with:
+      #     files: lcov.info
+      #     commit_parent: ${{ github.event.pull_request.base.sha }}
+      #     fail_ci_if_error: false
 
   integration:
     runs-on: [self-hosted, Linux, iroha2ci]

.github/workflows/iroha2-dev.yml

@@ -10,11 +10,40 @@ env:
 jobs:
   registry:
     runs-on: [self-hosted, Linux, iroha2-dev-push]
-    container:
-      image: hyperledger/iroha2-ci:nightly-2024-01-12
     steps:
       - uses: actions/checkout@v4
-      - uses: docker/login-action@v3
+      - name: Set up Docker Buildx
+        id: buildx
+        if: always()
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Build and export to Docker iroha2:dev image
+        uses: docker/build-push-action@v5
+        if: always()
+        with:
+          context: .
+          load: true
+          file: Dockerfile
+          tags: |
+            hyperledger/iroha2:dev
+            docker.soramitsu.co.jp/iroha2/iroha2:dev
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Test docker-compose.single.yml before pushing
+        run: |
+          docker compose -f docker-compose.single.yml up --wait || exit 1
+          docker compose -f docker-compose.single.yml down
+      - name: Test docker-compose.local.yml before pushing
+        run: |
+          docker compose -f docker-compose.local.yml up --wait || exit 1
+          docker compose -f docker-compose.local.yml down
+      - name: Test docker-compose.yml before pushing
+        run: |
+          docker compose -f docker-compose.yml up --wait || exit 1
+          docker compose -f docker-compose.yml down
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -24,27 +53,15 @@ jobs:
           registry: docker.soramitsu.co.jp
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_TOKEN }}
-      - name: Set up Docker Buildx
-        id: buildx
-        if: always()
-        uses: docker/setup-buildx-action@v3
-        with:
-          install: true
-      - name: Build and push iroha2:dev image
+      - name: Push iroha2:dev image
         uses: docker/build-push-action@v5
-        if: always()
         with:
+          context: .
           push: true
           tags: |
             hyperledger/iroha2:dev
             docker.soramitsu.co.jp/iroha2/iroha2:dev
           labels: commit=${{ github.sha }}
-          build-args: TAG=dev
-          file: Dockerfile
-          # This context specification is required
-          context: .
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
 
   archive_binaries_and_schema:
     runs-on: ubuntu-latest

.github/workflows/iroha2-no-incorrect-image.yml

@@ -0,0 +1,25 @@
+name: I2::CI::check_for_incorrect_images
+
+on:
+  push:
+    branches:
+      - iroha2-dev
+      - iroha2-stable
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.11"
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: pip install -r .github/scripts/ci_test/requirements.txt --no-input
+      - name: Check containers on iroha2-stable branch
+        if: github.base_ref == 'iroha2-stable'
+        run: python .github/scripts/ci_test/ci_image_scan.py --allow iroha2:stable -- docker-compose*.yml
+      - name: Check containers on iroha2-dev branch
+        if: github.base_ref == 'iroha2-dev'
+        run: python .github/scripts/ci_test/ci_image_scan.py --allow iroha2:dev -- docker-compose*.yml

.github/workflows/iroha2-profiling-image.yml

@@ -0,0 +1,64 @@
+name: I2::Profiling::Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      IROHA2_IMAGE_TAG:
+        required: true
+        default: stable
+      IROHA2_IMAGE_RELEASE:
+        required: true
+      IROHA2_DOCKERFILE:
+        required: true
+        default: Dockerfile.glibc
+      IROHA2_PROFILE:
+        required: true
+        default: profiling
+      IROHA2_RUSTFLAGS:
+        required: false
+        default: -C force-frame-pointers=on
+      IROHA2_FEATURES:
+        required: false
+        default: profiling
+      IROHA2_CARGOFLAGS:
+        required: false
+        default: -Z build-std
+
+jobs:
+  registry:
+    runs-on: [self-hosted, Linux, iroha2-dev-push]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Soramitsu Harbor
+        uses: docker/login-action@v3
+        with:
+          registry: docker.soramitsu.co.jp
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_TOKEN }}
+      - name: Set up Docker Buildx
+        id: buildx
+        if: always()
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Build and push iroha2:profiling-image
+        uses: docker/build-push-action@v5
+        if: always()
+        with:
+          push: true
+          tags: |
+            hyperledger/iroha2:${{ github.event.inputs.IROHA2_IMAGE_TAG }}-${{ github.event.inputs.IROHA2_IMAGE_RELEASE }}-profiling
+            docker.soramitsu.co.jp/iroha2/iroha2:${{ github.event.inputs.IROHA2_IMAGE_TAG }}-${{ github.event.inputs.IROHA2_IMAGE_RELEASE }}-profiling
+          labels: commit=${{ github.sha }}
+          build-args: |
+            "PROFILE=${{ github.event.inputs.IROHA2_PROFILE }}"
+            "RUSTFLAGS=${{ github.event.inputs.IROHA2_RUSTFLAGS }}"
+            "FEATURES=${{ github.event.inputs.IROHA2_FEATURES }}"
+            "CARGOFLAGS=${{ github.event.inputs.IROHA2_CARGOFLAGS }}"
+          file: ${{ github.event.inputs.IROHA2_DOCKERFILE }}
+          # This context specification is required
+          context: .