diff --git a/.bazelignore b/.bazelignore index 19dbe95779..26c872b968 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,4 +1,5 @@ bin +doc/_build docker/_build -rules_openapi/tools/node_modules +private/mgmtapi/tools/node_modules tools/lint/logctxcheck/testdata/src diff --git a/.bazelversion b/.bazelversion index 8a30e8f94a..19b860c187 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.4.0 +6.4.0 diff --git a/.buildkite/hooks/bazel-remote.yml b/.buildkite/hooks/bazel-remote.yml index 07523b1c2a..b4fdae7b29 100644 --- a/.buildkite/hooks/bazel-remote.yml +++ b/.buildkite/hooks/bazel-remote.yml @@ -1,4 +1,5 @@ version: "2.4" +name: bazel_remote services: bazel-remote: container_name: bazel-remote-cache diff --git a/.buildkite/hooks/go-module-proxy.yml b/.buildkite/hooks/go-module-proxy.yml index 4c852825f1..566068cca6 100644 --- a/.buildkite/hooks/go-module-proxy.yml +++ b/.buildkite/hooks/go-module-proxy.yml @@ -1,5 +1,6 @@ --- version: "2.4" +name: athens services: go-module-proxy: container_name: go-module-proxy diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ada7f07ae8..13ccb7e58a 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -30,9 +30,10 @@ rm -f $HOME/.bazelrc # --nostamp is required for better caching (only on non-release jobs). if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then echo "build --nostamp" > $HOME/.bazelrc - # Also set a fixed GIT_VERSION so that the workspace_status_command always - # returns the same value on CI to improve cache reuse. - export GIT_VERSION="ci-fixed" + # Shorten the git version to omit commit information, improving cache reuse. + # The format of git-version is "--" + # This will be shortened to "-modified-ci" + export GIT_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/') else echo "build --stamp" > $HOME/.bazelrc fi @@ -43,7 +44,7 @@ echo "~~~ Starting bazel remote cache proxy" # Start bazel remote cache proxy for S3 # Note that S3 keys are injected by buildkite, see # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws -docker compose --compatibility -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d +docker compose -f .buildkite/hooks/bazel-remote.yml up -d echo "~~~ Starting go module proxy" -docker compose --compatibility -f .buildkite/hooks/go-module-proxy.yml -p athens up -d +docker compose -f .buildkite/hooks/go-module-proxy.yml up -d diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 65ed0a2038..233efe828a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,7 +12,7 @@ steps: plugins: - scionproto/metahook#v0.3.0: post-artifact: | - cat << EOF | buildkite-agent annotate --style "info" + cat << EOF | buildkite-agent annotate --style "info" --context "binaries" #### Build outputs - SCION binaries - SCION test tools and utilities @@ -23,6 +23,28 @@ steps: - exit_status: 255 # Forced agent shutdown timeout_in_minutes: 10 - wait + - label: "Package :debian:" + command: + - make dist-deb + - cd deb; + - tar -chaf scion-deb-amd64.tar.gz *_amd64.deb + - tar -chaf scion-deb-arm64.tar.gz *_arm64.deb + - tar -chaf scion-deb-i386.tar.gz *_i386.deb + - tar -chaf scion-deb-armel.tar.gz *_armel.deb + artifact_paths: + - "deb/*.tar.gz" + plugins: + - scionproto/metahook#v0.3.0: + post-artifact: | + cat << EOF | buildkite-agent annotate --style "info" --context "packages" + #### Packages :debian: + - amd64 + - arm64 + - i386 + - armel + EOF + key: dist-deb + retry: *automatic-retry - label: "Unit Tests :bazel:" command: - bazel test --config=race --config=unit_all @@ -70,6 +92,7 @@ steps: timeout_in_minutes: 20 key: check_generated retry: *automatic-retry + - wait - group: "End to End" key: e2e steps: @@ -118,7 +141,7 @@ steps: - label: "E2E: default :docker: (ping)" command: - echo "--- build" - - make build docker-images + - make build-dev docker-images - echo "--- start topology" - ./scion.sh topology -d - ./scion.sh run diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index ced606e47a..bf61b3cdea 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -53,8 +53,12 @@ gen_bazel_test_steps() { echo " - \"bazel-testlogs.tar.gz\"" echo " timeout_in_minutes: 20" echo " retry:" + echo " manual:" + echo " permit_on_passed: true" echo " automatic:" echo " - exit_status: -1 # Agent was lost" echo " - exit_status: 255 # Forced agent shutdown" + echo " - exit_status: 3 # Test may be flaky or it just didn't pass" + echo " limit: 2" done } diff --git a/.buildkite/provision-agent.sh b/.buildkite/provision-agent.sh index 69dc76da8c..d8f2a5fcca 100755 --- a/.buildkite/provision-agent.sh +++ b/.buildkite/provision-agent.sh @@ -13,4 +13,4 @@ echo "~~~ Install build tools" tools/install_bazel tools/install_deps -sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps > /tmp/buildkite-scionproto-runner-provision.sum +sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps tools/env/rhel/pkgs.txt tools/env/debian/deps tools/env/debian/pkgs.txt > /tmp/buildkite-scionproto-runner-provision.sum diff --git a/.gitignore b/.gitignore index fb86551894..d3df276f72 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,10 @@ doc/venv/ /bin/* !/bin/.keepme +# Generated package files +########################## +/deb/ + # CTags ########################## tags @@ -91,6 +95,11 @@ _build node_modules # gobra +####### *.vpr tmp/ logger.log + +# emacs backup files +#################### +*~ diff --git a/BUILD.bazel b/BUILD.bazel index 267914fd4b..c43159ca4e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,7 +7,7 @@ load("//tools/lint:write_source_files.bzl", "write_source_files") load("//tools/lint/python:flake8_config.bzl", "flake8_lint_config") load("//:nogo.bzl", "nogo_deps") load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier") -load("@cgrindel_bazel_starlib//updatesrc:defs.bzl", "updatesrc_update_all") +load("@npm//private/mgmtapi/tools:@stoplight/spectral-cli/package_json.bzl", spectral_bin = "bin") # gazelle:prefix github.com/scionproto/scion # gazelle:map_kind go_library go_library //tools/lint:go.bzl @@ -198,6 +198,7 @@ pkg_tar( "//tools/buildkite/cmd/buildkite_artifacts", "//tools/end2end", "//tools/end2end_integration", + "//tools/end2endblast", "//tools/pktgen/cmd/pktgen", "//tools/scion_integration", "//tools/udpproxy", @@ -233,11 +234,8 @@ buildifier( mode = "check", ) -# Runs all update_src targets in this Workspace. Currently, generating the -# OpenAPI specs is the last target that depends on update_src. Eventually, -# this should be transitioned to write_all_source_files below. -updatesrc_update_all( - name = "update_all", +spectral_bin.spectral_binary( + name = "spectral", ) # Runs all write_source_files targets in this Workspace. To update the list run @@ -245,7 +243,16 @@ updatesrc_update_all( write_source_files( name = "write_all_source_files", additional_update_targets = [ - "//doc/_build/_static/command:write_files", + "//control/mgmtapi:write_files", + "//daemon/mgmtapi:write_files", + "//dispatcher/mgmtapi:write_files", "//doc/command:write_files", + "//gateway/mgmtapi:write_files", + "//private/ca/api:write_files", + "//private/mgmtapi/cppki/api:write_files", + "//private/mgmtapi/health/api:write_files", + "//private/mgmtapi/segments/api:write_files", + "//router/mgmtapi:write_files", + "//spec:write_files", ], ) diff --git a/Makefile b/Makefile index 2cb2419407..48e9aa6e07 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,30 @@ -.PHONY: all antlr bazel clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files +.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files -build: bazel +build-dev: + rm -f bin/* + bazel build //:scion //:scion-ci + tar -kxf bazel-bin/scion.tar -C bin + tar -kxf bazel-bin/scion-ci.tar -C bin + +build: + rm -f bin/* + bazel build //:scion + tar -kxf bazel-bin/scion.tar -C bin + +dist-deb: + bazel build //dist:deb_all + mkdir -p deb; rm -f deb/*; + @ # Bazel cannot include the version in the filename, if we want to set it automatically from the git tag. + @ # Extract the version from the .deb "control" manifest and expand the "__" in the filename to "__". + @ # See e.g. https://en.wikipedia.org/wiki/Deb_(file_format)#Control_archive + @for f in `bazel cquery //dist:deb_all --output=files 2>/dev/null`; do \ + if [ -f "$$f" ]; then \ + bf=`basename $$f`; \ + v="$$(ar p $$f control.tar.gz | tar -xz --to-stdout ./control | sed -n 's/Version: //p')"; \ + bfv=$${bf%%__*}_$${v}_$${bf#*__}; \ + cp -v "$$f" deb/$$bfv; \ + fi \ + done # all: performs the code-generation steps and then builds; the generated code # is git controlled, and therefore this is only necessary when changing the @@ -8,7 +32,7 @@ build: bazel # Use NOTPARALLEL to force correct order. # Note: From GNU make 4.4, this still allows building any other targets (e.g. lint) in parallel. .NOTPARALLEL: all -all: go_deps.bzl protobuf mocks gazelle build antlr write_all_source_files licenses +all: go_deps.bzl protobuf mocks gazelle build-dev antlr write_all_source_files licenses clean: bazel clean @@ -18,12 +42,6 @@ scrub: bazel clean --expunge rm -f bin/* -bazel: - rm -f bin/* - bazel build //:scion //:scion-ci - tar -kxf bazel-bin/scion.tar -C bin - tar -kxf bazel-bin/scion-ci.tar -C bin - test: bazel test --config=unit_all @@ -70,7 +88,6 @@ antlr: write_all_source_files: bazel run //:write_all_source_files - bazel run //:update_all .PHONY: lint lint-bazel lint-bazel-buildifier lint-doc lint-doc-mdlint lint-go lint-go-bazel lint-go-gazelle lint-go-golangci lint-go-semgrep lint-openapi lint-openapi-spectral lint-protobuf lint-protobuf-buf @@ -123,11 +140,11 @@ lint-openapi: lint-openapi-spectral lint-openapi-spectral: $(info ==> $@) - @tools/quiet bazel run --config=quiet @rules_openapi_npm//@stoplight/spectral-cli/bin:spectral -- lint --ruleset ${PWD}/spec/.spectral.yml ${PWD}/spec/*.gen.yml + @tools/quiet bazel run --config=quiet //:spectral -- lint --ruleset ${PWD}/spec/.spectral.yml ${PWD}/spec/*.gen.yml lint-doc: lint-doc-mdlint lint-doc-mdlint: $(info ==> $@) - @FILES=$$(find -type f -iname '*.md' -not -path "./rules_openapi/tools/node_modules/*" -not -path "./.github/**/*" | grep -vf tools/md/skipped); \ + @FILES=$$(find -type f -iname '*.md' -not -path "./private/mgmtapi/tools/node_modules/*" -not -path "./.github/**/*" | grep -vf tools/md/skipped); \ docker run --rm -v ${PWD}:/data -v ${PWD}/tools/md/mdlintstyle.rb:/style.rb $$(docker build -q tools/md) $${FILES} -s /style.rb diff --git a/README.md b/README.md index 0c3016ad61..7842cf3a22 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # SCION [![Slack chat](https://img.shields.io/badge/chat%20on-slack-blue?logo=slack)](https://scionproto.slack.com) +[![Matrix chat](https://img.shields.io/badge/chat%20on-matrix-blue?logo=matrix)](https://matrix.to/#/#dev:matrix.scion.org) +[![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/scionproto/awesome-scion) [![ReadTheDocs](https://img.shields.io/badge/doc-reference-blue?version=latest&style=flat&label=docs&logo=read-the-docs&logoColor=white)](https://docs.scion.org/en/latest) -[![Documentation](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/scionproto/scion) +[![Go Docs](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/scionproto/scion) [![Nightly Build](https://badge.buildkite.com/b70b65b38a75eb8724f41a6f1203c9327cfb767f07a0c1934e.svg)](https://buildkite.com/scionproto/scion-nightly/builds/latest) [![Go Report Card](https://goreportcard.com/badge/github.com/scionproto/scion)](https://goreportcard.com/report/github.com/scionproto/scion) [![GitHub issues](https://img.shields.io/github/issues/scionproto/scion/help%20wanted.svg?label=help%20wanted&color=purple)](https://github.com/scionproto/scion/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) @@ -10,28 +12,42 @@ [![Release](https://img.shields.io/github/release-pre/scionproto/scion.svg)](https://github.com/scionproto/scion/releases) [![License](https://img.shields.io/github/license/scionproto/scion.svg?maxAge=2592000)](https://github.com/scionproto/scion/blob/master/LICENSE) -Welcome to the open-source implementation of -[SCION](http://www.scion-architecture.net) (Scalability, Control and Isolation -On next-generation Networks), a future Internet architecture. SCION is the first -clean-slate Internet architecture designed to provide route control, failure -isolation, and explicit trust information for end-to-end communication. To find -out more about the project, please visit our [documentation -site](https://docs.scion.org/en/latest/). +Welcome to the open-source implementation of [SCION](http://www.scion-architecture.net) +(Scalability, Control and Isolation On next-generation Networks), a future Internet architecture. +SCION provides route control, failure isolation, and explicit trust information for end-to-end communication. +To find out more about the project, please visit our [documentation site](https://docs.scion.org/en/latest/). -## Connecting to the SCION Test Network +## Installation -Join [SCIONLab](https://www.scionlab.org) if you're interested in playing with -SCION in an operational global test deployment of SCION. As part of the SCIONLab -project, we support [pre-built binaries as Debian -packages](https://docs.scionlab.org/content/install/). +Installation packages for Debian and derivatives are available for x86-64, arm64, x86-32 and arm. +These packages can be found in the [latest release](https://github.com/scionproto/scion/releases/latest). +Packages for in-development versions can be found from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). -## Building +Alternatively, "naked" pre-built binaries are available for Linux x86-64 and +can be downloaded from the [latest release](https://github.com/scionproto/scion/releases/latest) or the +[latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). -To find out how to work with SCION, please visit our [documentation -site](https://docs.scion.org/en/latest/dev/setup.html) -for instructions on how to install build dependencies, build and run SCION. +### Build from sources -Pre-built binaries for x86-64 Linux are available from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). +SCION can be built with `go build`. To build all binaries used in a SCION deployment (i.e. +excluding the testing and development tools), run + +```sh +CGO_ENABLED=0 go build -o bin ./router/... ./control/... ./dispatcher/... ./daemon/... ./scion/... ./scion-pki/... ./gateway/... +``` + +The default way to build SCION, however, uses Bazel. +In particular, this allows to run all the tests, linters etc. +Please visit our [documentation site](https://docs.scion.org/en/latest/dev/setup.html) for +instructions on how to set up Bazel and the full development environment. + +### Connecting to the SCION Network + +Join [SCIONLab](https://www.scionlab.org) if you're interested in playing with SCION in an +operational global test deployment of SCION. + +The [awesome-scion](https://github.com/scionproto/awesome-scion#deployments) list contains +pointers to production deployments of SCION. ## Contributing diff --git a/WORKSPACE b/WORKSPACE index 491241f5f5..e0c3552cf0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,5 @@ workspace( name = "com_github_scionproto_scion", - managed_directories = { - "@rules_openapi_npm": ["rules_openapi/tools/node_modules"], - }, ) load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") @@ -31,22 +28,28 @@ lint_setup({ http_archive( name = "aspect_bazel_lib", - sha256 = "e3151d87910f69cf1fc88755392d7c878034a69d6499b287bcfc00b1cf9bb415", - strip_prefix = "bazel-lib-1.32.1", - url = "https://github.com/aspect-build/bazel-lib/archive/refs/tags/v1.32.1.tar.gz", + sha256 = "a185ccff9c1b8589c63f66d7eb908de15c5d6bb05562be5f46336c53e7a7326a", + strip_prefix = "bazel-lib-2.0.0-rc1", + url = "https://github.com/aspect-build/bazel-lib/releases/download/v2.0.0-rc1/bazel-lib-v2.0.0-rc1.tar.gz", ) -load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies") +load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains") + +# Required bazel-lib dependencies aspect_bazel_lib_dependencies() +# Register bazel-lib toolchains + +aspect_bazel_lib_register_toolchains() + # Bazel rules for Golang http_archive( name = "io_bazel_rules_go", - sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", + sha256 = "91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip", ], ) @@ -54,34 +57,20 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_register_toolchains( nogo = "@//:nogo", - version = "1.21.1", + version = "1.21.3", ) # Gazelle http_archive( name = "bazel_gazelle", - sha256 = "29218f8e0cebe583643cbf93cae6f971be8a2484cdcfa1e45057658df8d54002", + sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.32.0/bazel-gazelle-v0.32.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.32.0/bazel-gazelle-v0.32.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", ], ) -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") - -# Explictly override golang.org/x/sys. Our github.com/quic-go/quic-go cannot -# compile without at least golang.org/x/sys@v0.2.0. The rules_go version that -# we use (v0.34.0) imports an older version. A recent enough version was only introduced -# in v0.36.0. See: https://github.com/bazelbuild/rules_go/commit/64b9226a3bca997866b8831889ffb9de87405a0d -# -# This version should be kept in sync with the go_deps.bzl file. We can remove it -# once we update the rules_go version. -go_repository( - name = "org_golang_x_sys", - importpath = "golang.org/x/sys", - sum = "h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=", - version = "v0.8.0", -) +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") go_rules_dependencies() @@ -94,26 +83,20 @@ load("//:go_deps.bzl", "go_deps") go_deps() -## Explictly override xerrors: https://github.com/bazelbuild/bazel-gazelle/issues/1217 -# go_repository( -# name = "org_golang_x_xerrors", -# importpath = "golang.org/x/xerrors", -# sum = "h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=", -# version = "v0.0.0-20220907171357-04be3eba64a2", -# ) - gazelle_dependencies() # XXX Needs to be before rules_docker # Python rules http_archive( name = "rules_python", - sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4", - strip_prefix = "rules_python-0.13.0", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() python_register_toolchains( name = "python3_10", @@ -131,10 +114,10 @@ install_python_deps() http_archive( name = "rules_pkg", - sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66", + sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", ], ) @@ -158,9 +141,8 @@ rules_antlr_dependencies("4.9.3") http_archive( name = "io_bazel_rules_docker", - sha256 = "85ffff62a4c22a74dbd98d05da6cf40f497344b3dbf1e1ab0a37ab2a1a6ca014", - strip_prefix = "rules_docker-0.23.0", - urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.23.0/rules_docker-v0.23.0.tar.gz"], + sha256 = "b1e80761a8a8243d03ebca8845e9cc1ba6c82ce7c5179ce2b295cd36f7e394bf", + urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.25.0/rules_docker-v0.25.0.tar.gz"], ) load("@io_bazel_rules_docker//repositories:repositories.bzl", container_repositories = "repositories") @@ -241,9 +223,9 @@ container_pull( # protobuf/gRPC http_archive( name = "rules_proto_grpc", - sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419", - strip_prefix = "rules_proto_grpc-3.1.1", - urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"], + sha256 = "9ba7299c5eb6ec45b6b9a0ceb9916d0ab96789ac8218269322f0124c0c0d24e2", + strip_prefix = "rules_proto_grpc-4.5.0", + urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.5.0/rules_proto_grpc-4.5.0.tar.gz"], ) load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") @@ -258,18 +240,16 @@ rules_proto_dependencies() rules_proto_toolchains() -load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos") - -rules_proto_grpc_python_repos() - -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") +load("@rules_proto_grpc//buf:repositories.bzl", rules_proto_grpc_buf_repos = "buf_repos") -grpc_deps() +rules_proto_grpc_buf_repos() http_archive( name = "com_github_bazelbuild_buildtools", - strip_prefix = "buildtools-master", - url = "https://github.com/bazelbuild/buildtools/archive/2.2.1.zip", + strip_prefix = "buildtools-6.3.3", + urls = [ + "https://github.com/bazelbuild/buildtools/archive/refs/tags/6.3.3.tar.gz", + ], ) http_file( @@ -290,16 +270,32 @@ load("@com_github_scionproto_scion_python_lint_deps//:requirements.bzl", install install_python_lint_deps() -load("//rules_openapi:dependencies.bzl", "rules_openapi_dependencies") +http_archive( + name = "aspect_rules_js", + sha256 = "a949d56fed8fa0a8dd82a0a660acc949253a05b2b0c52a07e4034e27f11218f6", + strip_prefix = "rules_js-1.33.1", + url = "https://github.com/aspect-build/rules_js/releases/download/v1.33.1/rules_js-v1.33.1.tar.gz", +) + +load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies") -rules_openapi_dependencies() +rules_js_dependencies() -load("//rules_openapi:install.bzl", "rules_openapi_install_yarn_dependencies") +load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains") -rules_openapi_install_yarn_dependencies() +nodejs_register_toolchains( + name = "nodejs", + node_version = DEFAULT_NODE_VERSION, +) + +load("@aspect_rules_js//npm:npm_import.bzl", "npm_translate_lock") + +npm_translate_lock( + name = "npm", + pnpm_lock = "@com_github_scionproto_scion//private/mgmtapi/tools:pnpm-lock.yaml", + verify_node_modules_ignored = "@com_github_scionproto_scion//:.bazelignore", +) -# TODO(lukedirtwalker): can that be integrated in the rules_openapi_dependencies -# call above somehow? -load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies") +load("@npm//:repositories.bzl", "npm_repositories") -bazel_starlib_dependencies() +npm_repositories() diff --git a/acceptance/cert_renewal/test.py b/acceptance/cert_renewal/test.py index 799fa42258..2a3a9ef30d 100755 --- a/acceptance/cert_renewal/test.py +++ b/acceptance/cert_renewal/test.py @@ -72,9 +72,9 @@ def _run(self): end2end.run_fg() logger.info("==> Shutting down control servers and purging caches") - for container in self.dc.list_containers("scion_sd.*"): + for container in self.dc.list_containers("sd.*"): self.dc("rm", container) - for container in self.dc.list_containers("scion_cs.*"): + for container in self.dc.list_containers("cs.*"): self.dc.stop_container(container) for cs_config in cs_configs: files = list((pathlib.Path(self.artifacts) / diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index e4a1b7014b..c6f56fb2c8 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -35,15 +35,12 @@ from plumbum import cmd SCION_DC_FILE = "gen/scion-dc.yml" -DC_PROJECT = "scion" SCION_TESTING_DOCKER_ASSERTIONS_OFF = 'SCION_TESTING_DOCKER_ASSERTIONS_OFF' class Compose(object): def __init__(self, - project: str = DC_PROJECT, compose_file: str = SCION_DC_FILE): - self.project = project self.compose_file = compose_file def __call__(self, *args, **kwargs) -> str: @@ -51,8 +48,7 @@ def __call__(self, *args, **kwargs) -> str: # Note: not using plumbum here due to complications with encodings in the captured output try: res = subprocess.run( - ["docker", "compose", "--compatibility", - "-f", self.compose_file, "-p", self.project, *args], + ["docker", "compose", "-f", self.compose_file, *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") except subprocess.CalledProcessError as e: raise _CalledProcessErrorWithOutput(e) from None @@ -201,7 +197,7 @@ def assert_no_networks(writer=None): writer.write("Docker networking assertions are OFF\n") return - allowed_nets = ['bridge', 'host', 'none'] + allowed_nets = ['bridge', 'host', 'none', 'benchmark'] unexpected_nets = [] for net in _get_networks(): if net.name not in allowed_nets: diff --git a/acceptance/hidden_paths/test.py b/acceptance/hidden_paths/test.py index cf0d3ae438..90bc3101cb 100755 --- a/acceptance/hidden_paths/test.py +++ b/acceptance/hidden_paths/test.py @@ -5,10 +5,9 @@ import http.server import threading -from plumbum import cmd - from acceptance.common import base from acceptance.common import scion +from tools.topology.scion_addr import ISD_AS class Test(base.TestTopogen): @@ -108,12 +107,6 @@ def setup_start(self): super().setup_start() - self._testers = { - "2": "tester_1-ff00_0_2", - "3": "tester_1-ff00_0_3", - "4": "tester_1-ff00_0_4", - "5": "tester_1-ff00_0_5", - } self._ases = { "2": "1-ff00:0:2", "3": "1-ff00:0:3", @@ -126,27 +119,30 @@ def _run(self): self._server.shutdown() # by now configuration must have been downloaded everywhere # Group 3 - self._showpaths_bidirectional("2", "3", 0) - self._showpaths_bidirectional("2", "5", 0) - self._showpaths_bidirectional("3", "5", 0) + self._showpaths_bidirectional("2", "3") + self._showpaths_bidirectional("2", "5") + self._showpaths_bidirectional("3", "5") # Group 4 - self._showpaths_bidirectional("2", "4", 0) - self._showpaths_bidirectional("2", "5", 0) - self._showpaths_bidirectional("4", "5", 0) + self._showpaths_bidirectional("2", "4") + self._showpaths_bidirectional("2", "5") + self._showpaths_bidirectional("4", "5") # Group 3 X 4 - self._showpaths_bidirectional("3", "4", 1) - - def _showpaths_bidirectional(self, source: str, destination: str, retcode: int): - self._showpaths_run(source, destination, retcode) - self._showpaths_run(destination, source, retcode) - - def _showpaths_run(self, source_as: str, destination_as: str, retcode: int): - print(cmd.docker("exec", "-t", self._testers[source_as], "scion", - "sp", self._ases[destination_as], - "--timeout", "2s", - retcode=retcode)) + try: + self._showpaths_bidirectional("3", "4") + except Exception as e: + print(e) + else: + raise AssertionError("Unexpected success; should not have paths 3 -> 4") + + def _showpaths_bidirectional(self, source: str, destination: str): + self._showpaths_run(source, destination) + self._showpaths_run(destination, source) + + def _showpaths_run(self, source_as: str, destination_as: str): + print(self.execute_tester(ISD_AS(self._ases[source_as]), + "scion", "sp", self._ases[destination_as], "--timeout", "2s")) def configuration_server(server): diff --git a/acceptance/router_benchmark/BUILD.bazel b/acceptance/router_benchmark/BUILD.bazel new file mode 100644 index 0000000000..67e95e56a0 --- /dev/null +++ b/acceptance/router_benchmark/BUILD.bazel @@ -0,0 +1,28 @@ +load("//acceptance/common:raw.bzl", "raw_test") + +exports_files([ + "conf", + "test.py", +]) + +args = [ + "--executable", + "brload:$(location //acceptance/router_benchmark/brload:brload)", + "--container-loader=posix-router:latest#$(location //docker:posix_router)", +] + +data = [ + ":conf", + "//docker:posix_router", + "//acceptance/router_benchmark/brload:brload", +] + +raw_test( + name = "test", + src = "test.py", + args = args, + data = data, + homedir = "$(rootpath //docker:posix_router)", + # This test uses sudo and accesses /var/run/netns. + local = True, +) diff --git a/acceptance/router_benchmark/brload/BUILD.bazel b/acceptance/router_benchmark/brload/BUILD.bazel new file mode 100644 index 0000000000..c408b5e0d5 --- /dev/null +++ b/acceptance/router_benchmark/brload/BUILD.bazel @@ -0,0 +1,27 @@ +load("//tools/lint:go.bzl", "go_library") +load("//:scion.bzl", "scion_go_binary") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/scionproto/scion/acceptance/router_benchmark/brload", + visibility = ["//visibility:private"], + deps = [ + "//acceptance/router_benchmark/cases:go_default_library", + "//pkg/log:go_default_library", + "//pkg/private/serrors:go_default_library", + "//pkg/scrypto:go_default_library", + "//pkg/slayers:go_default_library", + "//private/keyconf:go_default_library", + "@com_github_google_gopacket//:go_default_library", + "@com_github_google_gopacket//afpacket:go_default_library", + "@com_github_google_gopacket//layers:go_default_library", + "@com_github_spf13_cobra//:go_default_library", + ], +) + +scion_go_binary( + name = "brload", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/acceptance/router_benchmark/brload/main.go b/acceptance/router_benchmark/brload/main.go new file mode 100644 index 0000000000..3d764255f3 --- /dev/null +++ b/acceptance/router_benchmark/brload/main.go @@ -0,0 +1,326 @@ +// Copyright 2023 SCION Association +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/binary" + "errors" + "fmt" + "hash" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/afpacket" + "github.com/google/gopacket/layers" + "github.com/spf13/cobra" + + "github.com/scionproto/scion/acceptance/router_benchmark/cases" + "github.com/scionproto/scion/pkg/log" + "github.com/scionproto/scion/pkg/private/serrors" + "github.com/scionproto/scion/pkg/scrypto" + "github.com/scionproto/scion/pkg/slayers" + "github.com/scionproto/scion/private/keyconf" +) + +type Case func(payload string, mac hash.Hash) (string, string, []byte) + +type caseChoice string + +func (c *caseChoice) String() string { + return string(*c) +} + +func (c *caseChoice) Set(v string) error { + _, ok := allCases[v] + if !ok { + return errors.New("No such case") + } + *c = caseChoice(v) + return nil +} + +func (c *caseChoice) Type() string { + return "string enum" +} + +func (c *caseChoice) Allowed() string { + return fmt.Sprintf("One of: %v", reflect.ValueOf(allCases).MapKeys()) +} + +var ( + allCases = map[string]Case{ + "in": cases.In, + "out": cases.Out, + "in_transit": cases.InTransit, + "out_transit": cases.OutTransit, + "br_transit": cases.BrTransit, + } + logConsole string + dir string + numPackets int + numStreams uint16 + caseToRun caseChoice + interfaces []string +) + +func main() { + rootCmd := &cobra.Command{ + Use: "brload", + Short: "Generates traffic into a specific router of a specific topology", + } + intfCmd := &cobra.Command{ + Use: "show-interfaces", + Short: "Provides a terse list of the interfaces that this test requires", + Run: func(cmd *cobra.Command, args []string) { + os.Exit(showInterfaces(cmd)) + }, + } + runCmd := &cobra.Command{ + Use: "run", + Short: "Executes the test", + Run: func(cmd *cobra.Command, args []string) { + os.Exit(run(cmd)) + }, + } + runCmd.Flags().IntVar(&numPackets, "num-packets", 10, "Number of packets to send") + runCmd.Flags().Uint16Var(&numStreams, "num-streams", 4, + "Number of independent streams (flowID) to use") + runCmd.Flags().StringVar(&logConsole, "log.console", "error", + "Console logging level: debug|info|error|etc.") + runCmd.Flags().StringVar(&dir, "artifacts", "", "Artifacts directory") + runCmd.Flags().Var(&caseToRun, "case", "Case to run. "+caseToRun.Allowed()) + runCmd.Flags().StringArrayVar(&interfaces, "interface", []string{}, + `label=host_interface,mac,peer_mac where: + host_interface: use this to exchange traffic with interface