From 466b2ea8605031310462b340932290f37cf3eab4 Mon Sep 17 00:00:00 2001 From: Yaron Kaikov Date: Wed, 15 Mar 2023 22:32:26 +0200 Subject: [PATCH] wip1 --- packer/build_image.sh | 88 ++++++++++++++++---------------- packer/scylla.pkr.hcl | 114 +++++++++++++++++++++++++----------------- 2 files changed, 115 insertions(+), 87 deletions(-) diff --git a/packer/build_image.sh b/packer/build_image.sh index a9c69335..9ed2e8fb 100755 --- a/packer/build_image.sh +++ b/packer/build_image.sh @@ -15,6 +15,7 @@ BUILD_MODE='release' TARGET= APT_KEYS_DIR='/etc/apt/keyrings' APT_KEY='d0a112e067426ab2' +AMI_REGIONS="us-east-1" print_usage() { echo "$0 --localdeb --repo [URL] --target [distribution]" @@ -242,11 +243,11 @@ if [ "$TARGET" = "aws" ]; then arch="$ARCH" case "$arch" in "x86_64") - SOURCE_AMI_FILTER="ubuntu-minimal/images/hvm-ssd/ubuntu-jammy-22.04-amd64*" + SOURCE_OS_FILTER="ubuntu-minimal/images/hvm-ssd/ubuntu-jammy-22.04-amd64*" INSTANCE_TYPE="c4.xlarge" ;; "aarch64") - SOURCE_AMI_FILTER="ubuntu-minimal/images/hvm-ssd/ubuntu-jammy-22.04-arm64*" + SOURCE_OS_FILTER="ubuntu-minimal/images/hvm-ssd/ubuntu-jammy-22.04-arm64*" INSTANCE_TYPE="a1.xlarge" ;; *) @@ -256,25 +257,27 @@ if [ "$TARGET" = "aws" ]; then PACKER_ARGS+=(-var=region="$REGION") PACKER_ARGS+=(-var=instance_type="$INSTANCE_TYPE") - PACKER_ARGS+=(-var=source_ami_filter="$SOURCE_AMI_FILTER") PACKER_ARGS+=(-var=source_ami_owner="$SOURCE_AMI_OWNER") - if [ -n "$AMI_REGIONS" ]; then - PACKER_ARGS+=(-var=ami_regions="$AMI_REGIONS") - fi + PACKER_ARGS+=(-var=ami_regions="$AMI_REGIONS") elif [ "$TARGET" = "gce" ]; then SSH_USERNAME=ubuntu - SOURCE_IMAGE_FAMILY="ubuntu-minimal-2204-lts" + SOURCE_OS_FILTER="ubuntu-minimal-2204-lts" + INSTANCE_TYPE="n2-standard-2" + PACKER_ARGS+=(-var=zone="$REGION") - PACKER_ARGS+=(-var=source_image_family="$SOURCE_IMAGE_FAMILY") elif [ "$TARGET" = "azure" ]; then REGION="EAST US" SSH_USERNAME=azureuser + SOURCE_OS_FILTER="minimal-22_04-lts-gen2" + INSTANCE_TYPE="Standard_D4_v4" PACKER_ARGS+=(-var=client_id="$AZURE_CLIENT_ID") PACKER_ARGS+=(-var=client_secret="$AZURE_CLIENT_SECRET") PACKER_ARGS+=(-var=tenant_id="$AZURE_TENANT_ID") PACKER_ARGS+=(-var=subscription_id="$AZURE_SUBSCRIPTION_ID") + PACKER_ARGS+=(-var=image_publisher="Canonical") + PACKER_ARGS+=(-var=image_offer="0001-com-ubuntu-minimal-jammy") fi IMAGE_NAME="$PRODUCT-$VERSION-$ARCH-$(date '+%FT%T')" @@ -290,11 +293,11 @@ mkdir -p build export PACKER_LOG=1 export PACKER_LOG_PATH -set -x /usr/bin/packer ${PACKER_SUB_CMD} \ -only "*.$TARGET" \ -var=target="$TARGET" \ -var=repo="$REPO_FOR_INSTALL" \ + -var=source_os_filter="$SOURCE_OS_FILTER" \ -var=ssh_username="$SSH_USERNAME" \ -var=scylla_version="$VERSION" \ -var=scylla_build_id="$BUILD_ID" \ @@ -304,43 +307,44 @@ set -x -var=branch="$BRANCH" \ -var=arch="$ARCH" \ -var=product="$PRODUCT" \ + -var=instance_type="$INSTANCE_TYPE" \ -var=build_mode="$BUILD_MODE" \ -var=image_name="$IMAGE_NAME" \ -var=scylla_full_version="$SCYLLA_FULL_VERSION" \ "${PACKER_ARGS[@]}" \ -var=scylla_machine_image_version="$SCYLLA_MACHINE_IMAGE_VERSION" \ "$DIR"/scylla.pkr.hcl -set +x -# For some errors packer gives a success status even if fails. -# Search log for errors -if $DRY_RUN ; then - echo "DryRun: No need to grep errors on log" -else - GREP_STATUS=0 - case "$TARGET" in - "aws") - grep "us-east-1:" $PACKER_LOG_PATH - GREP_STATUS=$? - ;; - "gce") - grep "A disk image was created" $PACKER_LOG_PATH - GREP_STATUS=$? - ;; - "azure") - grep "Builds finished. The artifacts of successful builds are:" $PACKER_LOG_PATH - GREP_STATUS=$? - ;; - *) - echo "No Target is defined" - exit 1 - esac - - if [ $GREP_STATUS -ne 0 ] ; then - echo "Error: No image line found on log." - exit 1 - else - echo "Success: image line found on log" - fi -fi -exit $EXIT_STATUS +## For some errors packer gives a success status even if fails. +## Search log for errors +#if $DRY_RUN ; then +# echo "DryRun: No need to grep errors on log" +#else +# GREP_STATUS=0 +# case "$TARGET" in +# "aws") +# grep "us-east-1:" $PACKER_LOG_PATH > /dev/null +# GREP_STATUS=$? +# ;; +# "gce") +# grep "A disk image was created" $PACKER_LOG_PATH > /dev/null +# GREP_STATUS=$? +# ;; +# "azure") +# grep "Builds finished. The artifacts of successful builds are:" $PACKER_LOG_PATH > /dev/null +# GREP_STATUS=$? +# ;; +# *) +# echo "No Target is defined" +# exit 1 +# esac +# +# if [ $GREP_STATUS -ne 0 ] ; then +# echo "Error: No image line found on log." +# exit 1 +# else +# echo "Success: image line found on log" +# fi +#fi +# +#exit $EXIT_STATUS diff --git a/packer/scylla.pkr.hcl b/packer/scylla.pkr.hcl index eb8e7d76..5b50e5b3 100644 --- a/packer/scylla.pkr.hcl +++ b/packer/scylla.pkr.hcl @@ -1,32 +1,56 @@ -variable "ssh_username" { type = string default = "" } -variable "client_id" { type = string default = "" } -variable "client_secret" { type = string default = "" } -variable "tenant_id" { type = string default = "" } -variable "subscription_id" { type = string default = "" } -variable "image_name" { type = string default = "" } -variable "arch" { type = string default = "" } -variable "product" { type = string default = "" } -variable "repo" { type = string default = "" } -variable "scylla_build_id" { type = string default = "" } -variable "build_mode" { type = string default = "" } -variable "scylla_version" { type = string default = "" } -variable "scylla_build_sha_id" { type = string default = "" } -variable "operating_system" { type = string default = "" } -variable "build_tag" { type = string default = "" } -variable "branch" { type = string default = "" } -variable "scylla_full_version" { type = string default = "" } -variable "scylla_machine_image_version" { type = string default = "" } -variable "target" { type = string default = "" } -variable "install_args" { type = string default = "" } -variable "instance_type" { type = string default = "" } -variable "region" { type = string default = "" } -variable "source_ami_owner" { type = string default = "" } -variable "source_ami_filter" { type = string default = "" } -variable "scylla_ami_description" { type = string default = "" } -variable "source_image_family" { type = string default = "" } -variable "project_id" { type = string default = "scylla-images" } -variable "image_storage_location" { type = string default = "" } -variable "ami_regions" { type = string default = "us-east-1" } +variable "ssh_username" {} +variable "image_name" {} +variable "arch" {} +variable "product" {} +variable "repo" {} +variable "scylla_build_id" {} +variable "build_mode" {} +variable "scylla_version" {} +variable "scylla_build_sha_id" {} +variable "operating_system" {} +variable "build_tag" {} +variable "branch" {} +variable "scylla_full_version" {} +variable "scylla_machine_image_version" {} +variable "target" {} +variable "instance_type" {} +variable "source_os_filter" {} +variable "ami_regions" { + type = string + default = "" +} +variable "source_ami_owner" { + type = string + default = "" +} +variable "region" { + type = string + default = "" +} +variable "client_id" { + type = string + default = "" +} +variable "client_secret" { + type = string + default = "" +} +variable "tenant_id" { + type = string + default = "" +} +variable "subscription_id" { + type = string + default = "" +} +variable "image_offer" { + type = string + default = "" +} +variable "image_publisher" { + type = string + default = "" +} source "azure-arm" "azure" { azure_tags = { @@ -52,10 +76,10 @@ source "azure-arm" "azure" { managed_image_resource_group_name = "scylla-images" managed_image_name = regex_replace("${var.image_name}", ":", "-") os_type = "Linux" - image_publisher = "Canonical" - image_offer = "0001-com-ubuntu-minimal-jammy" - image_sku = "minimal-22_04-lts-gen2" - vm_size = "Standard_D4_v4" + image_publisher = "${var.image_publisher}" + image_offer = "${var.image_offer}" + image_sku = "${var.source_os_filter}" + vm_size = "${var.instance_type}" keep_os_disk = true private_virtual_network_with_public_ip = true build_resource_group_name = "scylla-images" @@ -100,12 +124,11 @@ source "amazon-ebs" "aws" { } source_ami_filter { filters = { - name = "${var.source_ami_filter}" + name = "${var.source_os_filter}" } owners = ["${var.source_ami_owner}"] most_recent = true } - ami_description = "${var.scylla_ami_description}" ami_name = regex_replace("${var.image_name}", ":", "-") ena_support = true instance_type = "${var.instance_type}" @@ -147,6 +170,7 @@ source "googlecompute" "gce" { disk_size = 30 image_description = "Official ScyllaDB image ${var.scylla_version}" image_family = "scylla" + project_id = "scylla-images" image_labels = { arch = "${var.arch}" branch = "${var.branch}" @@ -163,7 +187,6 @@ source "googlecompute" "gce" { user_data_format_version = "3" } image_name = "${lower(regex_replace("${var.image_name}", "[: . _]", "-"))}" - image_storage_locations = ["${var.image_storage_location}"] labels = { keep = 1 keep_action = "terminate" @@ -174,8 +197,7 @@ source "googlecompute" "gce" { } omit_external_ip = false preemptible = true - project_id = "${var.project_id}" - source_image_family = "${var.source_image_family}" + source_image_family = "${var.source_os_filter}" ssh_timeout = "6m" ssh_username = "${var.ssh_username}" use_internal_ip = false @@ -187,13 +209,7 @@ build { "source.amazon-ebs.aws", "source.googlecompute.gce" ] - post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - image_name = regex_replace("${var.image_name}", ":", "-") - } - } + provisioner "file" { destination = "/home/${var.ssh_username}/" source = "files/" @@ -223,4 +239,12 @@ build { provisioner "shell" { inline = ["if [ ${var.target} = gce -o ${var.target} = azure ]; then sudo userdel -r -f ${var.ssh_username}; fi"] } + + post-processor "manifest" { + output = "manifest.json" + strip_path = false + custom_data = { + image_name = regex_replace("${var.image_name}", ":", "-") + } + } }