diff --git a/.bazelrc b/.bazelrc index ae43f2c0dd..5ac68e7012 100644 --- a/.bazelrc +++ b/.bazelrc @@ -18,6 +18,7 @@ build --incompatible_default_to_explicit_init_py # Enable resolution of cc toolchain by go toolchain build --incompatible_enable_cc_toolchain_resolution build --flag_alias=file_name_version=//:file_name_version +build --build_python_zip # include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc" # cannot be in common, because query chokes on it. @@ -25,7 +26,7 @@ build --define gotags=sqlite_modernc,netgo build:osx --define gotags=sqlite_modernc ### options for test -test --build_tests_only --print_relative_test_log_paths --test_output=errors +test --build_tests_only --print_relative_test_log_paths --test_output=errors --nobuild_python_zip ### predefined configurations (use with --config, eg. `bazel test --config=unit_all`) test:race --@io_bazel_rules_go//go/config:race diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d3dda80ee9..ca8d28325e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -96,7 +96,7 @@ steps: - ./tools/licenses.sh /tmp/test-artifacts/licenses - diff -rNu3 /tmp/test-artifacts/licenses ./licenses/data - echo "--- gomocks" - - ./tools/gomocks.py diff + - make mocksdiff - echo "--- antlr" - rm -rf /tmp/test-artifacts/antlr - cp -R antlr/ /tmp/test-artifacts/antlr diff --git a/BUILD.bazel b/BUILD.bazel index 0f04ad4d1e..ac12e7d60a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -209,14 +209,18 @@ pkg_tar( package_dir = "", ) -# This contains all of the binaries needed to run the topology generator. +# This contains all of the binaries needed to run the topology generator +# and start a local stack. pkg_tar( name = "scion-topo", srcs = [ - "//scion-pki/cmd/scion-pki", + "//tools:set_ipv6_addr", + "//tools:supervisorctl", + "//tools:supervisord", + "//tools:topodot", + "//tools:topogen", ], mode = "0755", - package_dir = "", ) # Nogo - Go code analysis tool diff --git a/Makefile b/Makefile index 8a43295c70..58044869c4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ -.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files git-version +.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks mocksdiff protobuf scion-topo test test-integration write_all_source_files git-version build-dev: rm -f bin/* - bazel build //:scion //:scion-ci + bazel build //:scion //:scion-ci //:scion-topo tar -kxf bazel-bin/scion.tar -C bin tar -kxf bazel-bin/scion-ci.tar -C bin + tar -kxf bazel-bin/scion-topo.tar -C bin build: rm -f bin/* @@ -75,10 +76,6 @@ docker-images: @echo "Load images" @bazel cquery '//docker:prod union //docker:test' --output=files 2>/dev/null | xargs -I{} docker load --input {} -scion-topo: - bazel build //:scion-topo - tar --overwrite -xf bazel-bin/scion-topo.tar -C bin - protobuf: rm -rf bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* bazel build --output_groups=go_generated_srcs //pkg/proto/... @@ -88,7 +85,10 @@ protobuf: chmod 0644 pkg/proto/*/*.pb.go pkg/proto/*/*/*.pb.go mocks: - tools/gomocks.py + bazel run //tools:gomocks + +mocksdiff: + bazel run //tools:gomocks -- diff gazelle: go_deps.bzl bazel run //:gazelle --verbose_failures --config=quiet diff --git a/acceptance/router_benchmark/BUILD.bazel b/acceptance/router_benchmark/BUILD.bazel index 420219394b..ce3b4f4468 100644 --- a/acceptance/router_benchmark/BUILD.bazel +++ b/acceptance/router_benchmark/BUILD.bazel @@ -53,6 +53,13 @@ py_library( py_binary( name = "benchmark", srcs = ["benchmark.py"], + args = [ + "--brload", + "$(location //acceptance/router_benchmark/brload:brload)", + ], + data = [ + "//acceptance/router_benchmark/brload", + ], visibility = ["//visibility:public"], deps = [ "benchmarklib", diff --git a/acceptance/router_benchmark/benchmark.py b/acceptance/router_benchmark/benchmark.py index 9f942b1e92..ccb73e83d3 100755 --- a/acceptance/router_benchmark/benchmark.py +++ b/acceptance/router_benchmark/benchmark.py @@ -86,19 +86,22 @@ class RouterBMTool(cli.Application, RouterBM): log_level = cli.SwitchAttr(["l", "loglevel"], str, default='warning', help="Logging level") doit = cli.Flag(["r", "run"], - help="Run the benchmark, as opposed to seeing the instructions.") + help="Run the benchmark, as opposed to seeing the instructions") json = cli.Flag(["j", "json"], - help="Output the report in json format.") + help="Output the report in json format") # Used by the RouterBM mixin: coremark = cli.SwitchAttr(["c", "coremark"], int, default=0, - help="The coremark score of the subject machine.") + help="The coremark score of the subject machine") mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0, - help="The mmbm score of the subject machine.") + help="The mmbm score of the subject machine") packet_size = cli.SwitchAttr(["s", "size"], int, default=172, - help="Test packet size (includes all headers - floored at 154).") + help="Test packet size (includes all headers - floored at 154)") + brload_path = cli.SwitchAttr(["b", "brload"], str, default="bin/brload", + help="Relative path to the brload tool") + intf_map: dict[str, Intf] = {} - brload: LocalCommand = local["./bin/brload"] + brload: LocalCommand = None brload_cpus: list[int] = [] artifacts = f"{os.getcwd()}/acceptance/router_benchmark" prom_address: str = "localhost:9090" @@ -332,6 +335,9 @@ def instructions(self): """) def main(self, *interfaces: str): + # brload cannot be set statically. It need the cli arguments to be + # processed. + self.brload = local[self.brload_path] status = 1 try: logging.basicConfig(level=self.log_level.upper()) diff --git a/scion.sh b/scion.sh index 2d92c5d14f..a21b45b278 100755 --- a/scion.sh +++ b/scion.sh @@ -23,11 +23,11 @@ cmd_topology() { cmd_topo-clean echo "Create topology, configuration, and execution files." - tools/topogen.py "$@" + ./bin/topogen "$@" } cmd_topodot() { - ./tools/topodot.py "$@" + ./bin/topodot "$@" } start_scion() { diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 778182bbcf..1184f3669c 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_python//python:defs.bzl", "py_binary") load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") exports_files([ "gzip_exec_interp", @@ -26,6 +27,24 @@ py_binary( ], ) +py_console_script_binary( + name = "supervisord", + pkg = requirement("supervisor"), + visibility = ["//visibility:public"], + deps = [ + requirement("supervisor-wildcards"), + ], +) + +py_console_script_binary( + name = "supervisorctl", + pkg = requirement("supervisor"), + visibility = ["//visibility:public"], + deps = [ + requirement("supervisor-wildcards"), + ], +) + py_binary( name = "topogen", srcs = ["topogen.py"], @@ -45,3 +64,34 @@ py_binary( requirement("pyyaml"), ], ) + +py_binary( + name = "topodot", + srcs = ["topodot.py"], + data = [ + ], + main = "topodot.py", + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//tools/topology:py_default_library", + "@bazel_tools//tools/python/runfiles", + requirement("plumbum"), + ], +) + +py_binary( + name = "set_ipv6_addr", + srcs = ["set_ipv6_addr.py"], + data = [ + ], + main = "set_ipv6_addr.py", + python_version = "PY3", + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//tools/topology:py_default_library", + "@bazel_tools//tools/python/runfiles", + ], +) diff --git a/tools/gomocks.py b/tools/gomocks.py index d1ae63b163..abf1ace79a 100755 --- a/tools/gomocks.py +++ b/tools/gomocks.py @@ -27,6 +27,7 @@ def rule_to_file(rule: str) -> Tuple[str, str]: def mock_rules() -> List[str]: bazel = plumbum.local['bazel'] + os.chdir(os.environ.get("BUILD_WORKING_DIRECTORY","/nonexistium")) raw_rules = bazel("query", "filter(\"go_default_mock$\", kind(gomock, //...))") return raw_rules.splitlines() diff --git a/tools/install_deps b/tools/install_deps index b43f43d756..65728d2506 100755 --- a/tools/install_deps +++ b/tools/install_deps @@ -8,4 +8,4 @@ if which apt-get >& /dev/null; then elif which yum >& /dev/null; then "$BASE/env/rhel/deps" fi -"$BASE/env/pip3/deps" + diff --git a/tools/supervisor.sh b/tools/supervisor.sh index 8a1a7bd62d..d19677700c 100755 --- a/tools/supervisor.sh +++ b/tools/supervisor.sh @@ -6,7 +6,7 @@ mkdir -p logs OPTIONS="$@" CONF_FILE="tools/supervisord.conf" if [ ! -e /tmp/supervisor.sock ]; then - supervisord -c $CONF_FILE + bin/supervisord -c $CONF_FILE fi -supervisorctl -c $CONF_FILE $OPTIONS +bin/supervisorctl -c $CONF_FILE $OPTIONS