diff --git a/.mailmap b/.mailmap index 9dacb3260de6..eab07801c9d4 100644 --- a/.mailmap +++ b/.mailmap @@ -16,5 +16,8 @@ Andrei Sandu <54316454+sandreim@users.noreply.github.com> Diana Popa Alexandru Cihodaru Liviu Berciu -Jonathan Woollett-Light +Jonathan Woollett-Light +Jonathan Woollett-Light karthik nedunchezhiyan +Babis Chalios +Pablo Barbáchano diff --git a/CHANGELOG.md b/CHANGELOG.md index 8588f8514410..afa5bdb2e068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,6 @@ and mitigations, the Guest vCPU will apear to look like a Skylake CPU, making it safe to snapshot uVMs running on a newer host CPU (Cascade Lake) and restore on a host that has a Skylake CPU. - - Added a new CLI option `--metrics-path PATH`. It accepts a file parameter where metrics will be sent to. - A MAC address is generated if one is not explicitly specified while adding diff --git a/docs/kernel-policy.md b/docs/kernel-policy.md index cb208dd7565c..c4080e6449b3 100644 --- a/docs/kernel-policy.md +++ b/docs/kernel-policy.md @@ -133,7 +133,7 @@ registers incompatibility. Load snapshot on host 4.15 successful - successful + unsuccessful due to mismatch in MSRs Load snapshot on host 5.10 diff --git a/src/api_server/swagger/firecracker.yaml b/src/api_server/swagger/firecracker.yaml index 15e0ac3a5bc1..83ea6217999d 100644 --- a/src/api_server/swagger/firecracker.yaml +++ b/src/api_server/swagger/firecracker.yaml @@ -1227,4 +1227,4 @@ definitions: description: Path to UNIX domain socket, used to proxy vsock connections. vsock_id: type: string - description: This parameter has been deprecated since v1.1.0. + description: This parameter has been deprecated since v1.0.0. diff --git a/tests/integration_tests/functional/test_snapshot_basic.py b/tests/integration_tests/functional/test_snapshot_basic.py index f14e182dbdf7..0d0b31ccd11b 100644 --- a/tests/integration_tests/functional/test_snapshot_basic.py +++ b/tests/integration_tests/functional/test_snapshot_basic.py @@ -309,9 +309,9 @@ def test_5_inc_snapshots( # Testing matrix: # - Guest kernel: All supported ones # - Rootfs: Ubuntu 18.04 - # - Microvm: 2vCPU with 4096 MB RAM + # - Microvm: 2vCPU with 512MB RAM # TODO: Multiple microvm sizes must be tested in the async pipeline. - microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_4096mb")) + microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_512mb")) kernel_artifacts = ArtifactSet(artifacts.kernels()) disk_artifacts = ArtifactSet(artifacts.disks(keyword="ubuntu")) diff --git a/tests/integration_tests/performance/test_versioned_serialization_benchmark.py b/tests/integration_tests/performance/test_versioned_serialization_benchmark.py index 95469ee360af..ee06bae51ea0 100644 --- a/tests/integration_tests/performance/test_versioned_serialization_benchmark.py +++ b/tests/integration_tests/performance/test_versioned_serialization_benchmark.py @@ -22,12 +22,12 @@ BASELINES = { "Intel": { "serialize": { - "no-crc": {"target": 0.150, "delta": 0.036}, # milliseconds # milliseconds + "no-crc": {"target": 0.205, "delta": 0.050}, # milliseconds # milliseconds "crc": {"target": 0.244, "delta": 0.44}, # milliseconds # milliseconds }, "deserialize": { "no-crc": {"target": 0.056, "delta": 0.02}, # milliseconds # milliseconds - "crc": {"target": 0.046, "delta": 0.035}, # milliseconds # milliseconds + "crc": {"target": 0.075, "delta": 0.030}, # milliseconds # milliseconds }, }, "AMD": { diff --git a/tests/integration_tests/style/test_licenses.py b/tests/integration_tests/style/test_licenses.py index fd104edc0f73..61810809c2ed 100644 --- a/tests/integration_tests/style/test_licenses.py +++ b/tests/integration_tests/style/test_licenses.py @@ -3,6 +3,7 @@ """Tests checking against the existence of licenses in each file.""" import datetime + from framework import utils AMAZON_COPYRIGHT_YEARS = range(2018, datetime.datetime.now().year + 1) @@ -105,3 +106,7 @@ def test_for_valid_licenses(): if _validate_license(file) is False: error_msg.append("{}".format(str(file))) assert not error_msg, "Files {} have invalid licenses".format((error_msg)) + + +if __name__ == "__main__": + test_for_valid_licenses() diff --git a/tools/devtool b/tools/devtool index 3d2c8ba790bf..5ea7e79f73f2 100755 --- a/tools/devtool +++ b/tools/devtool @@ -79,11 +79,9 @@ DEVCTR_IMAGE_TAG="v44" # (Yet another step on our way to reproducible builds.) DEVCTR_IMAGE="${DEVCTR_IMAGE_NO_TAG}:${DEVCTR_IMAGE_TAG}" -# Naming things is hard -MY_NAME="Firecracker $(basename "$0")" - # Full path to the Firecracker tools dir on the host. FC_TOOLS_DIR=$(cd "$(dirname "$0")" && pwd) +source "$FC_TOOLS_DIR/functions" # Full path to the Firecracker sources dir on the host. FC_ROOT_DIR=$(cd "${FC_TOOLS_DIR}/.." && pwd) @@ -159,57 +157,6 @@ DEFAULT_TEST_SESSION_ROOT_PATH=/srv DEFAULT_RAMDISK_PATH=/mnt/devtool-ramdisk -# Send a decorated message to stdout, followed by a new line -# -say() { - [ -t 1 ] && [ -n "$TERM" ] \ - && echo "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \ - || echo "[$MY_NAME] $*" -} - -# Send a decorated message to stdout, without a trailing new line -# -say_noln() { - [ -t 1 ] && [ -n "$TERM" ] \ - && echo -n "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \ - || echo "[$MY_NAME] $*" -} - -# Send a text message to stderr -# -say_err() { - [ -t 2 ] && [ -n "$TERM" ] \ - && echo -e "$(tput setaf 1)[$MY_NAME] $*$(tput sgr0)" 1>&2 \ - || echo -e "[$MY_NAME] $*" 1>&2 -} - -# Send a warning-highlighted text to stdout -say_warn() { - [ -t 1 ] && [ -n "$TERM" ] \ - && echo "$(tput setaf 3)[$MY_NAME] $*$(tput sgr0)" \ - || echo "[$MY_NAME] $*" -} - -# Exit with an error message and (optional) code -# Usage: die [-c ] -# -die() { - code=1 - [[ "$1" = "-c" ]] && { - code="$2" - shift 2 - } - say_err "$@" - exit $code -} - -# Exit with an error message if the last exit code is not 0 -# -ok_or_die() { - code=$? - [[ $code -eq 0 ]] || die -c $code "$@" -} - # Check if Docker is available and exit if it's not. # Upon returning from this call, the caller can be certain Docker is available. # @@ -278,14 +225,6 @@ ensure_devctr() { } } -# Check if /dev/kvm exists. Exit if it doesn't. -# Upon returning from this call, the caller can be certain /dev/kvm is -# available. -# -ensure_kvm() { - [[ -c /dev/kvm ]] || die "/dev/kvm not found. Aborting." -} - # Make sure the build/ dirs are available. Exit if we can't create them. # Upon returning from this call, the caller can be certain the build/ dirs exist. # @@ -380,50 +319,6 @@ cmd_build_devctr() { copy_poetry_lockfile } -# Prompt the user for confirmation before proceeding. -# Args: -# $1 prompt text. -# Default: Continue? (y/n) -# $2 confirmation input. -# Default: y -# Return: -# exit code 0 for successful confirmation -# exit code != 0 if the user declined -# -get_user_confirmation() { - - # Pass if running unattended - [[ "$OPT_UNATTENDED" = true ]] && return 0 - - # Fail if STDIN is not a terminal (there's no user to confirm anything) - [[ -t 0 ]] || return 1 - - # Otherwise, ask the user - # - msg=$([ -n "$1" ] && echo -n "$1" || echo -n "Continue? (y/n) ") - yes=$([ -n "$2" ] && echo -n "$2" || echo -n "y") - say_noln "$msg" - read c && [ "$c" = "$yes" ] && return 0 - return 1 -} - -# Validate the user supplied version number. -# It must start with 3 groups of integers separated by dot and -# must not contain `wip` or `dirty`. -validate_version() { - declare version_regex="^([0-9]+\.){2}[0-9]+" - version="$1" - - if [ -z "$version" ]; then - die "Version cannot be empty." - elif [[ ! "$version" =~ $version_regex ]]; then - die "Invalid version number: $version. Version should start with \$Major.\$Minor.\$Build, see - https://github.com/firecracker-microvm/firecracker/blob/main/docs/RELEASE_POLICY.md for more information." - elif [[ "$version" == *"wip"* ]] || [[ "$version" == *"dirty"* ]]; then - die "Invalid version number: $version. Version should not contain \`wip\` or \`dirty\`." - fi -} - # Validate that the repo targetted for a release exists. # validate_repo() { @@ -972,6 +867,16 @@ cmd_build() { return $ret } +function cmd_make_release { + cmd_test || die "Tests failed!" + + run_devctr \ + --user "$(id -u):$(id -g)" \ + --workdir "$CTR_FC_ROOT_DIR" \ + -- \ + ./tools/release.sh --libc musl --profile release --make-release +} + cmd_distclean() { # List of folders to remove. dirs=("build" "test_results") @@ -1180,11 +1085,6 @@ copy_release_artifact() { fi } -get_swagger_version() { - local file="$1" - grep -oP 'version: \K.*' "$file" -} - mount_ramdisk() { local ramdisk_size="$1" umount_ramdisk @@ -1278,6 +1178,8 @@ cmd_test() { ret=$? + say "$(date -u +'%F %H:%M:%S %Z')" + say "Finished test run ..." if [[ $ramdisk = true ]]; then umount_ramdisk diff --git a/tools/functions b/tools/functions new file mode 100644 index 000000000000..193b59f2d52e --- /dev/null +++ b/tools/functions @@ -0,0 +1,153 @@ +# -*- shell-script[bash] -*- + +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +################### +# Logging helpers # +################### + +# Naming things is hard +ARG0=${BASH_SOURCE[-1]} +MY_NAME="Firecracker $(basename "$ARG0")" + +# Send a decorated message to stdout, followed by a new line +# +say() { + [ -t 1 ] && [ -n "$TERM" ] \ + && echo "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \ + || echo "[$MY_NAME] $*" +} + +# Send a decorated message to stdout, without a trailing new line +# +say_noln() { + [ -t 1 ] && [ -n "$TERM" ] \ + && echo -n "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \ + || echo "[$MY_NAME] $*" +} + +# Send a text message to stderr +# +say_err() { + [ -t 2 ] && [ -n "$TERM" ] \ + && echo -e "$(tput setaf 1)[$MY_NAME] $*$(tput sgr0)" 1>&2 \ + || echo -e "[$MY_NAME] $*" 1>&2 +} + +# Send a warning-highlighted text to stdout +say_warn() { + [ -t 1 ] && [ -n "$TERM" ] \ + && echo "$(tput setaf 3)[$MY_NAME] $*$(tput sgr0)" \ + || echo "[$MY_NAME] $*" +} + +# Exit with an error message and (optional) code +# Usage: die [-c ] +# +die() { + code=1 + [[ "$1" = "-c" ]] && { + code="$2" + shift 2 + } + say_err "$@" + exit $code +} + +# Exit with an error message if the last exit code is not 0 +# +ok_or_die() { + code=$? + [[ $code -eq 0 ]] || die -c $code "$@" +} + +# ANSI output helper +# https://en.wikipedia.org/wiki/ANSI_escape_code +function SGR { + local codes=$* + printf "\e[%sm" "${codes// /;}" +} + +# Prompt the user for confirmation before proceeding. +# Args: +# $1 prompt text. +# Default: Continue? (y/n) +# $2 confirmation input. +# Default: y +# Return: +# exit code 0 for successful confirmation +# exit code != 0 if the user declined +# +OPT_UNATTENDED=false +get_user_confirmation() { + + # Pass if running unattended + [[ "$OPT_UNATTENDED" = true ]] && return 0 + + # Fail if STDIN is not a terminal (there's no user to confirm anything) + [[ -t 0 ]] || return 1 + + # Otherwise, ask the user + # + msg=$([ -n "${1:-}" ] && echo -n "$1" || echo -n "Continue? (y/n) ") + yes=$([ -n "${2:-}" ] && echo -n "$2" || echo -n "y") + say_noln "$msg" + read c && [ "$c" = "$yes" ] && return 0 + return 1 +} + +####################################### +# Release automation common functions # +####################################### + +# Get version from the swagger file +function get_swagger_version { + local file=${1:-"$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"} + grep -oP 'version: \K.*' "$file" +} + +function check_local_branch_is_release_branch { + local LOCAL_BRANCH=$(git rev-parse --abbrev-ref HEAD) + local RELEASE_BRANCH=firecracker-v$(echo "$version" |cut -d. -f-2) + + if [ "$LOCAL_BRANCH" != "$RELEASE_BRANCH" ]; then + cat < + + Example: $0 1.1.1 1.1.2 +EOF + exit 1; +fi + +prev_ver="$1" +curr_ver="$2" +changelog="$FC_ROOT_DIR/CHANGELOG.md" + +if [[ ! $prev_ver < $curr_ver ]]; then + echo "$prev_ver >= $curr_ver. Did you switch the argument order?" + exit 1 +fi + +# Patterns for the sections in the changelog corresponding to the versions. +pat_curr="^##\s\[$curr_ver\]" +pat_prev="^##\s\[$prev_ver\]" +# Extract the section enclosed between the 2 headers and strip off the first +# 2 and last 2 lines (one is blank and one contains the header `## [A.B.C]`). +# Then, replace `-` with `*` and remove section headers. +sed "/$pat_curr/,/$pat_prev/!d" "$changelog" \ + | sed '1,2d;$d' \ + | sed "s/^-/*/g" \ + | sed "s/^###\s//g" diff --git a/tools/release-prepare.sh b/tools/release-prepare.sh new file mode 100755 index 000000000000..105b4e57965e --- /dev/null +++ b/tools/release-prepare.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu -o pipefail + +FC_TOOLS_DIR=$(dirname $(realpath $0)) +source "$FC_TOOLS_DIR/functions" +FC_ROOT_DIR=$FC_TOOLS_DIR/.. + +if [ $# -ne 1 ]; then + cat < + + Example: $0 0.42.0 + + Prepare a new Firecracker release: + 1. Update the version number + 2. Update Crate dependencies + 3. Generate CREDITS.md and CHANGELOG.md + 4. Commit the result + 5. Create a link to PR the changes +EOF + exit 1 +fi +version=$1 +validate_version "$version" + +check_local_branch_is_release_branch + +# Create GitHub PR link +ORIGIN_URL=$(git config --get remote.origin.url) +GH_USER=$(git config --get github.user) +REPO=$(basename "$ORIGIN_URL" .git) +LOCAL_BRANCH=$(git rev-parse --abbrev-ref HEAD) +RELEASE_BRANCH=firecracker-v$(echo "$version" |cut -d. -f-2) +UPSTREAM=upstream +# In which branch should the change go, in the main repo? +TARGET_BRANCH=main +PATCH=$(echo "$version" |cut -d. -f3) +# If this is a patch release, the target branch should be the release branch +if [ "$PATCH" -gt 0 ]; then + TARGET_BRANCH=$RELEASE_BRANCH +fi +PR_URL="https://github.com/firecracker-microvm/$REPO/compare/$TARGET_BRANCH...$GH_USER:$REPO:$LOCAL_BRANCH?expand=1" + +# Get current version from the swagger spec. +prev_ver=$(get_swagger_version) + +say "Updating from $prev_ver to $version ..." +# Update version in files. +files_to_change=( + "$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml" + "$FC_ROOT_DIR/src/firecracker/Cargo.toml" + "$FC_ROOT_DIR/src/jailer/Cargo.toml" + "$FC_ROOT_DIR/src/rebase-snap/Cargo.toml" + "$FC_ROOT_DIR/src/seccompiler/Cargo.toml" +) +say "Updating source files:" +for file in "${files_to_change[@]}"; do + say "- $file" + # Dirty hack to make this work on both macOS/BSD and Linux. + # FIXME This is very hacky and can unintentionally bump other versions, so + # only do the replacement *once*. + sed -i "s/$prev_ver/$version/" "$file" +done + +# Run `cargo check` to update firecracker and jailer versions in +# `Cargo.lock`. +say "Updating lockfile..." +cargo check + +# Update credits. +say "Updating credits..." +$FC_TOOLS_DIR/update-credits.sh + +# Update changelog. +say "Updating changelog..." +sed -i "s/\[Unreleased\]/\[$version\]/g" "$FC_ROOT_DIR/CHANGELOG.md" + +git add "${files_to_change[@]}" Cargo.lock CREDITS.md CHANGELOG.md +git commit -s -m "Releasing v$version" + + +# pretty print code +function pp-code { + # grey background + echo "$(SGR 0 48 5 242)$*$(SGR 0)" +} + +# pretty print a list item +function pp-li { + bullet=$1; shift + # reset bg-color-5 bold + echo "$(SGR 0 48 5 101)$bullet$(SGR 0 1) $*$(SGR 0)" +} + +cat < + + Example: $0 1.1.1 1.1.2 + + It will create a local git tag and push it to the upstream +EOF + exit 1 +fi +prev_version=$1 +version=$2 +validate_version "$prev_version" +validate_version "$version" + +LOCAL_BRANCH=$(git rev-parse --abbrev-ref HEAD) +RELEASE_BRANCH=firecracker-v$(echo "$version" |cut -d. -f-2) +UPSTREAM=upstream +UPSTREAM_URL=$(git remote get-url $UPSTREAM) +check_local_branch_is_release_branch + +# Start by creating a local tag and associate to it a description. +say "Creating local tag..." +create_local_tag "$version" "$prev_version" "$LOCAL_BRANCH" + +# pretty print a warning +function warn { + # reset reverse yellow + echo "$(SGR 0 7 33)$*$(SGR 0)" +} + +warn "!WARNING! The next step will modify upstream: $UPSTREAM_URL by running:" +echo " git push $UPSTREAM v$version" +echo " git push $UPSTREAM $RELEASE_BRANCH" +get_user_confirmation || die "Cancelling tag push" +git push $UPSTREAM "v$version" +git push $UPSTREAM "$RELEASE_BRANCH" diff --git a/tools/release.sh b/tools/release.sh new file mode 100755 index 000000000000..9c282d69d0f8 --- /dev/null +++ b/tools/release.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# fail if we encounter an error, uninitialized variable or a pipe breaks +set -eu -o pipefail + +FC_TOOLS_DIR=$(dirname $(realpath $0)) +source "$FC_TOOLS_DIR/functions" +FC_ROOT_DIR=$FC_TOOLS_DIR/.. + +function get-profile-dir { + case $1 in + dev) + echo debug + ;; + *) + echo "$1" + ;; + esac +} + +function check_swagger_artifact { + # Validate swagger version against target version. + local swagger_path version swagger_ver + swagger_path=$1 + version=$2 + swagger_ver=$(get_swagger_version "$swagger_path") + if [[ $version =~ v$swagger_ver.* ]]; then + die "Artifact $swagger_path's version: $swagger_ver does not match release version $version." + fi +} + +function check_bin_artifact { + # Validate binary version against target version. + local bin_path version bin_version + bin_path=$1 + version=$2 + bin_version=$($bin_path --version | head -1 | grep -oP ' \Kv.*') + if [[ "$bin_version" != "$version" ]]; then + die "Artifact $bin_path's version: $bin_version does not match release version $version." + fi +} + +function strip-and-split-debuginfo { + local bin=$1 + echo "STRIP $bin" + objcopy --only-keep-debug $bin $bin.debug + chmod a-x $bin.debug + objcopy --strip-debug --add-gnu-debuglink=$bin.debug $bin +} + +#### MAIN #### + +# defaults +LIBC=musl +PROFILE=dev +MAKE_RELEASE= + +#### Option parsing + +while [[ $# -gt 0 ]]; do + case $1 in + --help) + cat <SHA256SUMS +) diff --git a/tools/update-credits.sh b/tools/update-credits.sh index 3bc858e98b69..9086439b01b0 100755 --- a/tools/update-credits.sh +++ b/tools/update-credits.sh @@ -28,7 +28,6 @@ written in Rust with a focus on safety and security. Thanks go to: * [Jason D. Clinton](https://github.com/jclinton) * Sonny Rao - Contributors to the Firecracker repository: EOH echo