diff --git a/packer/ami_variables.json b/packer/ami_variables.json deleted file mode 100644 index 800ed39a..00000000 --- a/packer/ami_variables.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "subnet_id": "subnet-ec4a72c4", - "security_group_id": "sg-c5e1f7a0", - "region": "us-east-1", - "associate_public_ip_address": "true", - "instance_type": "c4.xlarge" -} diff --git a/packer/azure_variables.json b/packer/azure_variables.json deleted file mode 100644 index 28417ce3..00000000 --- a/packer/azure_variables.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "client_id": "", - "client_secret": "", - "tenant_id": "", - "subscription_id": "", - "security_group_id": "", - "region": "EAST US", - "vm_size": "Standard_D4_v4" -} diff --git a/packer/build_image.sh b/packer/build_image.sh index a4b0e069..a9c69335 100755 --- a/packer/build_image.sh +++ b/packer/build_image.sh @@ -9,7 +9,6 @@ source "$DIR"/../SCYLLA-VERSION-GEN BUILD_ID=$(date -u '+%FT%H-%M-%S') OPERATING_SYSTEM="ubuntu22.04" -EXIT_STATUS=0 DRY_RUN=false DEBUG=false BUILD_MODE='release' @@ -127,20 +126,6 @@ while [ $# -gt 0 ]; do TARGET="$2" shift 2 echo "--target parameter TARGET: |$TARGET|" - case "$TARGET" in - "aws") - JSON_FILE="ami_variables.json" - ;; - "gce") - JSON_FILE="gce_variables.json" - ;; - "azure") - JSON_FILE="azure_variables.json" - ;; - *) - print_usage - ;; - esac ;; "--arch") ARCH="$2" @@ -215,9 +200,6 @@ if [ $LOCALDEB -eq 1 ]; then SCYLLA_FULL_VERSION=$(get_version_from_local_deb "$DIR"/files/"$PRODUCT"-server*_$(deb_arch).deb) SCYLLA_MACHINE_IMAGE_VERSION=$(get_version_from_local_deb "$DIR"/files/"$PRODUCT"-machine-image*_all.deb) - SCYLLA_JMX_VERSION=$(get_version_from_local_deb "$DIR"/files/"$PRODUCT"-jmx*_all.deb) - SCYLLA_TOOLS_VERSION=$(get_version_from_local_deb "$DIR"/files/"$PRODUCT"-tools-*_all.deb) - SCYLLA_PYTHON3_VERSION=$(get_version_from_local_deb "$DIR"/files/"$PRODUCT"-python3*_$(deb_arch).deb) cd "$DIR"/files dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz @@ -246,9 +228,6 @@ else SCYLLA_FULL_VERSION=$(get_version_from_remote_deb $PRODUCT-server) SCYLLA_MACHINE_IMAGE_VERSION=$(get_version_from_remote_deb $PRODUCT-machine-image) - SCYLLA_JMX_VERSION=$(get_version_from_remote_deb $PRODUCT-jmx) - SCYLLA_TOOLS_VERSION=$(get_version_from_remote_deb $PRODUCT-tools) - SCYLLA_PYTHON3_VERSION=$(get_version_from_remote_deb $PRODUCT-python3) sudo rm -f $TMPREPO @@ -275,29 +254,27 @@ if [ "$TARGET" = "aws" ]; then exit 1 esac - SCYLLA_AMI_DESCRIPTION="scylla-$SCYLLA_FULL_VERSION scylla-machine-image-$SCYLLA_MACHINE_IMAGE_VERSION scylla-jmx-$SCYLLA_JMX_VERSION scylla-tools-$SCYLLA_TOOLS_VERSION scylla-python3-$SCYLLA_PYTHON3_VERSION" + PACKER_ARGS+=(-var=region="$REGION") + PACKER_ARGS+=(-var=instance_type="$INSTANCE_TYPE") + PACKER_ARGS+=(-var=source_ami_filter="$SOURCE_AMI_FILTER") + PACKER_ARGS+=(-var=source_ami_owner="$SOURCE_AMI_OWNER") + if [ -n "$AMI_REGIONS" ]; then + PACKER_ARGS+=(-var=ami_regions="$AMI_REGIONS") + fi - PACKER_ARGS+=(-var region="$REGION") - PACKER_ARGS+=(-var buildMode="$BUILD_MODE") - PACKER_ARGS+=(-var instance_type="$INSTANCE_TYPE") - PACKER_ARGS+=(-var source_ami_filter="$SOURCE_AMI_FILTER") - PACKER_ARGS+=(-var source_ami_owner="$SOURCE_AMI_OWNER") - PACKER_ARGS+=(-var scylla_ami_description="${SCYLLA_AMI_DESCRIPTION:0:255}") elif [ "$TARGET" = "gce" ]; then SSH_USERNAME=ubuntu SOURCE_IMAGE_FAMILY="ubuntu-minimal-2204-lts" - PACKER_ARGS+=(-var source_image_family="$SOURCE_IMAGE_FAMILY") + PACKER_ARGS+=(-var=source_image_family="$SOURCE_IMAGE_FAMILY") elif [ "$TARGET" = "azure" ]; then REGION="EAST US" SSH_USERNAME=azureuser - SCYLLA_IMAGE_DESCRIPTION="scylla-$SCYLLA_FULL_VERSION scylla-machine-image-$SCYLLA_MACHINE_IMAGE_VERSION scylla-jmx-$SCYLLA_JMX_VERSION scylla-tools-$SCYLLA_TOOLS_VERSION scylla-python3-$SCYLLA_PYTHON3_VERSION" - PACKER_ARGS+=(-var scylla_image_description="${SCYLLA_IMAGE_DESCRIPTION:0:255}") - PACKER_ARGS+=(-var client_id="$AZURE_CLIENT_ID") - PACKER_ARGS+=(-var client_secret="$AZURE_CLIENT_SECRET") - PACKER_ARGS+=(-var tenant_id="$AZURE_TENANT_ID") - PACKER_ARGS+=(-var subscription_id="$AZURE_SUBSCRIPTION_ID") + PACKER_ARGS+=(-var=client_id="$AZURE_CLIENT_ID") + PACKER_ARGS+=(-var=client_secret="$AZURE_CLIENT_SECRET") + PACKER_ARGS+=(-var=tenant_id="$AZURE_TENANT_ID") + PACKER_ARGS+=(-var=subscription_id="$AZURE_SUBSCRIPTION_ID") fi IMAGE_NAME="$PRODUCT-$VERSION-$ARCH-$(date '+%FT%T')" @@ -308,12 +285,6 @@ if $DEBUG ; then IMAGE_NAME="debug-$IMAGE_NAME" fi -if [ ! -f $JSON_FILE ]; then - echo "'$JSON_FILE not found. Please create it before start building Image." - echo "See variables.json.example" - exit 1 -fi - mkdir -p build export PACKER_LOG=1 @@ -321,28 +292,24 @@ export PACKER_LOG_PATH set -x /usr/bin/packer ${PACKER_SUB_CMD} \ - -only="$TARGET" \ - -var-file="$JSON_FILE" \ - -var install_args="$INSTALL_ARGS" \ - -var ssh_username="$SSH_USERNAME" \ - -var scylla_full_version="$SCYLLA_FULL_VERSION" \ - -var scylla_version="$VERSION" \ - -var scylla_machine_image_version="$SCYLLA_MACHINE_IMAGE_VERSION" \ - -var scylla_jmx_version="$SCYLLA_JMX_VERSION" \ - -var scylla_tools_version="$SCYLLA_TOOLS_VERSION" \ - -var scylla_python3_version="$SCYLLA_PYTHON3_VERSION" \ - -var scylla_build_id="$BUILD_ID" \ - -var scylla_build_sha_id="$SCYLLA_BUILD_SHA_ID" \ - -var build_tag="$BUILD_TAG" \ - -var operating_system="$OPERATING_SYSTEM" \ - -var branch="$BRANCH" \ - -var ami_regions="$AMI_REGIONS" \ - -var arch="$ARCH" \ - -var product="$PRODUCT" \ - -var build_mode="$BUILD_MODE" \ - -var image_name="$IMAGE_NAME" \ + -only "*.$TARGET" \ + -var=target="$TARGET" \ + -var=repo="$REPO_FOR_INSTALL" \ + -var=ssh_username="$SSH_USERNAME" \ + -var=scylla_version="$VERSION" \ + -var=scylla_build_id="$BUILD_ID" \ + -var=scylla_build_sha_id="$SCYLLA_BUILD_SHA_ID" \ + -var=build_tag="$BUILD_TAG" \ + -var=operating_system="$OPERATING_SYSTEM" \ + -var=branch="$BRANCH" \ + -var=arch="$ARCH" \ + -var=product="$PRODUCT" \ + -var=build_mode="$BUILD_MODE" \ + -var=image_name="$IMAGE_NAME" \ + -var=scylla_full_version="$SCYLLA_FULL_VERSION" \ "${PACKER_ARGS[@]}" \ - "$DIR"/scylla.json + -var=scylla_machine_image_version="$SCYLLA_MACHINE_IMAGE_VERSION" \ + "$DIR"/scylla.pkr.hcl set +x # For some errors packer gives a success status even if fails. # Search log for errors diff --git a/packer/gce_variables.json b/packer/gce_variables.json deleted file mode 100644 index fa326651..00000000 --- a/packer/gce_variables.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "project_id": "scylla-images", - "region": "europe-west1", - "zone": "europe-west1-b", - "image_storage_location": "europe-west1", - "instance_type": "n2-standard-2" -} diff --git a/packer/scylla.pkr.hcl b/packer/scylla.pkr.hcl new file mode 100644 index 00000000..c81f7f08 --- /dev/null +++ b/packer/scylla.pkr.hcl @@ -0,0 +1,313 @@ +variable "ssh_username" { + type = string + default = "" +} +variable "client_id" { + type = string + default = "" +} +variable "client_secret" { + type = string + default = "" +} +variable "tenant_id" { + type = string + default = "" +} +variable "subscription_id" { + type = string + default = "" +} +variable "image_name" { + type = string + default = "" +} +variable "arch" { + type = string + default = "" +} +variable "product" { + type = string + default = "" +} +variable "repo" { + type = string + default = "" +} +variable "scylla_build_id" { + type = string + default = "" +} +variable "build_mode" { + type = string + default = "" +} +variable "scylla_version" { + type = string + default = "" +} +variable "scylla_build_sha_id" { + type = string + default = "" +} +variable "operating_system" { + type = string + default = "" +} +variable "build_tag" { + type = string + default = "" +} +variable "branch" { + type = string + default = "" +} +variable "scylla_full_version" { + type = string + default = "" +} +variable "scylla_machine_image_version" { + type = string + default = "" +} +variable "target" { + type = string + default = "" +} +variable "install_args" { + type = string + default = "" +} +variable "instance_type" { + type = string + default = "" +} +variable "region" { + type = string + default = "" +} +variable "source_ami_owner" { + type = string + default = "" +} +variable "source_ami_filter" { + type = string + default = "" +} +variable "scylla_ami_description" { + type = string + default = "" +} +variable "source_image_family" { + type = string + default = "" +} +variable "project_id" { + type = string + default = "scylla-images" +} +variable "image_storage_location" { + type = string + default = "" +} +variable "ami_regions" { + type = string + default = "us-east-1" +} + +source "azure-arm" "azure" { + azure_tags = { + arch = "${var.arch}" + branch = "${var.branch}" + build_id = "${var.scylla_build_id}" + build_mode = "${var.build_mode}" + build_tag = "${var.build_tag}" + operating_system = "${var.operating_system}" + scylla_build_sha_id = "${var.scylla_build_sha_id}" + scylla_jmx_version = "${var.scylla_full_version}" + scylla_machine_image_version = "${var.scylla_machine_image_version}" + scylla_python3_version = "${var.scylla_full_version}" + scylla_tools_version = "${var.scylla_full_version}" + scylla_version = "${var.scylla_full_version}" + user_data_format_version = "3" + } + ssh_username = "${var.ssh_username}" + client_id = "${var.client_id}" + client_secret = "${var.client_secret}" + tenant_id = "${var.tenant_id}" + subscription_id = "${var.subscription_id}" + managed_image_resource_group_name = "scylla-images" + managed_image_name = regex_replace("${var.image_name}", ":", "-") + os_type = "Linux" + image_publisher = "Canonical" + image_offer = "0001-com-ubuntu-minimal-jammy" + image_sku = "minimal-22_04-lts-gen2" + vm_size = "Standard_D4_v4" + keep_os_disk = true + private_virtual_network_with_public_ip = true + build_resource_group_name = "scylla-images" + virtual_network_name = "scylla-images" +} +source "amazon-ebs" "aws" { + ami_block_device_mappings { + device_name = "/dev/sdb" + virtual_name = "ephemeral0" + } + ami_block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral1" + } + ami_block_device_mappings { + device_name = "/dev/sdd" + virtual_name = "ephemeral2" + } + ami_block_device_mappings { + device_name = "/dev/sde" + virtual_name = "ephemeral3" + } + ami_block_device_mappings { + device_name = "/dev/sdf" + virtual_name = "ephemeral4" + } + ami_block_device_mappings { + device_name = "/dev/sdg" + virtual_name = "ephemeral5" + } + ami_block_device_mappings { + device_name = "/dev/sdh" + virtual_name = "ephemeral6" + } + ami_block_device_mappings { + device_name = "/dev/sdi" + virtual_name = "ephemeral7" + } + aws_polling { + delay_seconds = 30 + max_attempts = 100 + } + source_ami_filter { + filters = { + name = "${var.source_ami_filter}" + } + owners = ["${var.source_ami_owner}"] + most_recent = true + } + ami_description = "${var.scylla_ami_description}" + ami_name = regex_replace("${var.image_name}", ":", "-") + ena_support = true + instance_type = "${var.instance_type}" + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/sda1" + volume_size = 30 + } + subnet_id = "subnet-ec4a72c4" + security_group_id = "sg-c5e1f7a0" + region = "${var.region}" + ami_regions = [var.ami_regions] + + associate_public_ip_address = "true" + shutdown_behavior = "terminate" + sriov_support = true + ssh_clear_authorized_keys = true + ssh_timeout = "5m" + ssh_username = "${var.ssh_username}" + tags = { + Name = "${var.image_name}" + arch = "${var.arch}" + branch = "${var.branch}" + build_id = "${var.scylla_build_id}" + build_mode = "${var.build_mode}" + build_tag = "${var.build_tag}" + operating_system = "${var.operating_system}" + scylla_build_sha_id = "${var.scylla_build_sha_id}" + scylla_jmx_version = "${var.scylla_full_version}" + scylla_machine_image_version = "${var.scylla_machine_image_version}" + scylla_python3_version = "${var.scylla_full_version}" + scylla_tools_version = "${var.scylla_full_version}" + scylla_version = "${var.scylla_full_version}" + user_data_format_version = "3" + } + user_data_file = "user_data.txt" +} +source "googlecompute" "gce" { + disk_size = 30 + image_description = "Official ScyllaDB image ${var.scylla_version}" + image_family = "scylla" + image_labels = { + arch = "${var.arch}" + branch = "${var.branch}" + build_id = "${var.scylla_build_id}" + build_mode = "${var.build_mode}" + build_tag = "${var.build_tag}" + operating_system = regex_replace("${var.operating_system}", "[. ~]", "-") + scylla_build_sha_id = "${var.scylla_build_sha_id}" + scylla_jmx_version = regex_replace("${var.scylla_full_version}", "[. ~]", "-") + scylla_machine_image_version = regex_replace("${var.scylla_machine_image_version}", "[. ~]", "-") + scylla_python3_version = regex_replace("${var.scylla_full_version}", "[. ~]", "-") + scylla_tools_version = regex_replace("${var.scylla_full_version}", "[. ~]", "-") + scylla_version = regex_replace("${var.scylla_full_version}", "[. ~]", "-") + user_data_format_version = "3" + } + image_name = "${lower(regex_replace("${var.image_name}", "[: . _]", "-"))}" + image_storage_locations = ["${var.image_storage_location}"] + labels = { + keep = 1 + keep_action = "terminate" + } + machine_type = "${var.instance_type}" + metadata = { + block-project-ssh-keys = "TRUE" + } + omit_external_ip = false + preemptible = true + project_id = "${var.project_id}" + source_image_family = "${var.source_image_family}" + ssh_timeout = "6m" + ssh_username = "${var.ssh_username}" + use_internal_ip = false + zone = "${var.region}" +} +build { + sources = [ + "source.azure-arm.azure", + "source.amazon-ebs.aws", + "source.googlecompute.gce" + ] + post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + image_name = regex_replace("${var.image_name}", ":", "-") + } + } + provisioner "file" { + destination = "/home/${var.ssh_username}/" + source = "files/" + } + + provisioner "file" { + destination = "/home/${var.ssh_username}/" + source = "scylla_install_image" + } + + provisioner "shell" { + inline = ["sudo /usr/bin/cloud-init status --wait", "sudo /home/${var.ssh_username}/scylla_install_image --target-cloud ${var.target} --scylla-version ${var.scylla_full_version} --repo ${var.repo} --product ${var.product}"] + } + + provisioner "file" { + destination = "build/" + direction = "download" + source = "/home/${var.ssh_username}/${var.product}-packages-${var.scylla_full_version}-${var.arch}.txt" + } + + provisioner "file" { + destination = "build/" + direction = "download" + source = "/home/${var.ssh_username}/${var.product}-kernel-${var.scylla_full_version}-${var.arch}.txt" + } + + provisioner "shell" { + inline = ["if [ ${var.target} = gce -o ${var.target} = azure ]; then sudo userdel -r -f ${var.ssh_username}; fi"] + } +}