Jenkinsfile.release

#!groovy

/**
 * This program and the accompanying materials are made available under the terms of the
 * Eclipse Public License v2.0 which accompanies this distribution, and is available at
 * https://www.eclipse.org/legal/epl-v20.html
 *
 * SPDX-License-Identifier: EPL-2.0
 *
 * Copyright IBM Corporation 2018, 2019
 */

node('zowe-jenkins-agent-dind-wdc') {
  def lib = library("jenkins-library").org.zowe.jenkins_shared_library

  def ZOWE_RELEASE_REPOSITORY = 'libs-release-local'
  def ZOWE_RELEASE_PATH = '/org/zowe'
  def ZOWE_RELEASE_FILEPATTERN = "zowe-*.pax"
  def ZOWE_RELEASE_SMPE_PAX_FILEPATTERN = "zowe-smpe-*.zip"
  def ZOWE_RELEASE_SMPE_PTF_PROMOTE_FILEPATTERN = "smpe-promote-*.tar"
  def ZOWE_RELEASE_DOCKER_AMD64_FILEPATTERN = "server-bundle.amd64-*.tar"
  def ZOWE_RELEASE_DOCKER_AMD64_SOURCE_FILEPATTERN = "server-bundle.sources.amd64-*.tar"
  def ZOWE_RELEASE_DOCKER_S390X_FILEPATTERN = "server-bundle.s390x-*.tar"
  def ZOWE_RELEASE_DOCKER_S390X_SOURCE_FILEPATTERN = "server-bundle.sources.s390x-*.tar"
  def ZOWE_RELEASE_CLI_CORE_FILEPATTERN = "zowe-cli-package-1*.zip"
  def ZOWE_RELEASE_CLI_PLUGINS_FILEPATTERN = "zowe-cli-plugins-1*.zip"
  def ZOWE_INSTALL_PACKAGING_REPO = 'zowe/zowe-install-packaging'

  def pipeline = lib.pipelines.generic.GenericPipeline.new(this)
  def github = lib.scm.GitHub.new(this)

  def isFormalRelease = false
  def releaseFilePath
  def releaseArtifacts = readJSON(text:'{}')
  def publishFilePath
  def smpePtfPromoteTarPath

  pipeline.admins.add("jackjia", "markackert")

  // we have extra parameters for the pipeline
  pipeline.addBuildParameters(
    string(
      name: 'ZOWE_BUILD_REPOSITORY',
      description: 'Zowe successful build repository',
      defaultValue: 'libs-snapshot-local',
      trim: true,
      required: true
    ),
    // the purpose of this variable is Artifactory sometimes messed up build/artifact association.
    // when search by build number, some artifacts from other builds may show up.
    // this can narrow down the path to find the artifacts
    string(
      name: 'ZOWE_BUILD_PATH',
      description: 'Zowe successful build path. For example, "org/zowe/1.12.0-SNAPSHOT".',
      defaultValue: '*',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_BUILD_NAME',
      description: 'Zowe successful build name',
      defaultValue: 'zowe-install-packaging :: master',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_BUILD_NUMBER',
      description: 'Zowe successful build number',
      defaultValue: '',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_CLI_BUILD_REPOSITORY',
      description: 'Zowe CLI successful build repository',
      defaultValue: 'libs-snapshot-local',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_CLI_BUILD_PATH',
      description: 'Zowe CLI successful build path. For example, "org/zowe/cli/*".',
      defaultValue: '*',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_CLI_BUILD_NAME',
      description: 'Zowe CLI successful build name',
      defaultValue: 'Zowe CLI Bundle :: master',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_CLI_BUILD_NUMBER',
      description: 'REQUIRED if ZOWE_CLI_BUILD_RC_PATH is empty. Zowe CLI successful build number',
      defaultValue: '',
      trim: true,
      required: true
    ),
    string(
      name: 'ZOWE_RELEASE_VERSION',
      description: 'REQUIRED. Zowe release version without prefix v. For example, 0.9.0',
      defaultValue: '',
      trim: true,
      required: true
    )
  )

  pipeline.setup(
    packageName: 'org.zowe',
    github: [
      email                      : lib.Constants.DEFAULT_GITHUB_ROBOT_EMAIL,
      usernamePasswordCredential : lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
    ],
    artifactory: [
      url                        : lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL,
      usernamePasswordCredential : lib.Constants.DEFAULT_LFJ_ARTIFACTORY_ROBOT_CREDENTIAL,
    ]
  )

  pipeline.build(
    name          : "Validate",
    timeout       : [time: 5, unit: 'MINUTES'],
    isSkippable   : true,
    operation     : {
      if (!params.ZOWE_BUILD_NAME) {
        error "ZOWE_BUILD_NAME is required to promote build."
      }
      if (!params.ZOWE_BUILD_NUMBER) {
        error "ZOWE_BUILD_NUMBER is required to promote build."
      }
      if (!params.ZOWE_CLI_BUILD_NAME) {
        error "ZOWE_CLI_BUILD_NAME is required to promote build."
      }
      if (!params.ZOWE_CLI_BUILD_NUMBER) {
        error "ZOWE_CLI_BUILD_NUMBER is required to promote build."
      }
      if (!params.ZOWE_RELEASE_VERSION) {
        error "ZOWE_RELEASE_VERSION is required to promote build."
      }

      echo "Checking if ${params.ZOWE_RELEASE_VERSION} is a valid semantic version ..."
      // validate release version scheme
      // thanks semver/semver, this regular expression comes from
      // https://github.com/semver/semver/issues/232#issuecomment-405596809
      if (!(params.ZOWE_RELEASE_VERSION ==~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/)) {
        error "${params.ZOWE_RELEASE_VERSION} is not a valid semantic version."
      }
      if (params.ZOWE_RELEASE_VERSION ==~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/) {
        isFormalRelease = true
        echo ">>>> Version ${params.ZOWE_RELEASE_VERSION} is considered as a FORMAL RELEASE."
      } else {
        echo ">>>> Version ${params.ZOWE_RELEASE_VERSION} is NOT considered as a FORMAL RELEASE."
      }

      // this is the target Artifactory path will be released to
      releaseFilePath = "${ZOWE_RELEASE_REPOSITORY}${ZOWE_RELEASE_PATH}/${params.ZOWE_RELEASE_VERSION}".toString()

      // check artifactory release path
      echo "Checking if ${params.ZOWE_RELEASE_VERSION} already exists in Artifactory ..."
      def versionOnArtifactory
      try {
        versionOnArtifactory = pipeline.artifactory.getArtifact(releaseFilePath + "/*")
      } catch (e) {
        // ignore errors
      }
      if (versionOnArtifactory && versionOnArtifactory.path) {
        error "Zowe version ${params.ZOWE_RELEASE_VERSION} already exists (${releaseFilePath})"
      } else {
        echo ">>>> Target artifactory folder ${releaseFilePath} doesn't exist, may proceed."
      }

      // check if tag already exists
      github.cloneRepository([
        email                      : lib.Constants.DEFAULT_GITHUB_ROBOT_EMAIL,
        usernamePasswordCredential : lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
        repository                 : ZOWE_INSTALL_PACKAGING_REPO,
        folder                     : '.tag',
        branch                     : 'master'
      ])
      if (github.tagExistsRemote("v${params.ZOWE_RELEASE_VERSION}")) {
        error "Repository tag v${params.ZOWE_RELEASE_VERSION} already exists."
      } else {
        echo ">>>> Repository tag v${params.ZOWE_RELEASE_VERSION} doesn't exist, may proceed."
      }

      // find the Zowe build number will be promoted
      releaseArtifacts['zowe'] = [:]
      releaseArtifacts['zowe']['target'] = "zowe-${params.ZOWE_RELEASE_VERSION}.pax".toString()
      releaseArtifacts['zowe']['buildName'] = params.ZOWE_BUILD_NAME
      releaseArtifacts['zowe']['buildNumber'] = params.ZOWE_BUILD_NUMBER

      // get zowe build source artifact
      releaseArtifacts['zowe']['source'] = pipeline.artifactory.getArtifact([
        'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_FILEPATTERN}",
        'build-name'   : releaseArtifacts['zowe']['buildName'],
        'build-number' : releaseArtifacts['zowe']['buildNumber']
      ])
      echo ">>> Found Zowe build ${releaseArtifacts['zowe']['source']['path']}."

      // try to get Zowe build commit hash
      def zoweBuildInfo = pipeline.artifactory.getBuildInfo(
        releaseArtifacts['zowe']['buildName'],
        releaseArtifacts['zowe']['buildNumber'],
        "zowe-install-packaging"
      )
      releaseArtifacts['zowe']['revision'] = zoweBuildInfo && zoweBuildInfo['vcsRevision']
      if (!("${releaseArtifacts['zowe']['revision']}" ==~ /^[0-9a-fA-F]{40}$/)) { // if it's a SHA-1 commit hash
        error "Cannot extract git revision from build \"${releaseArtifacts['zowe']['buildName']}/${releaseArtifacts['zowe']['buildNumber']}\""
      }
      echo ">>>> Build ${releaseArtifacts['zowe']['buildName']}/${releaseArtifacts['zowe']['buildNumber']} commit hash is ${releaseArtifacts['zowe']['revision']}, may proceed."
      echo ">>>> BuildInfo vcsUrl is ${zoweBuildInfo['vcsUrl']}, vcs revision is ${zoweBuildInfo['vcsRevision']}"

      // get SMP/e build
      try {
        def smpeTarSource = pipeline.artifactory.getArtifact([
          'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_SMPE_PAX_FILEPATTERN}",
          'build-name'   : releaseArtifacts['zowe']['buildName'],
          'build-number' : releaseArtifacts['zowe']['buildNumber']
        ])
        if (smpeTarSource['path']) {
          echo ">>> Found SMP/e build ${smpeTarSource['path']}."
          def FMID = smpeTarSource['path'].split('/').last().split('-').first()
          releaseArtifacts['smpe-zip'] = [:]
          releaseArtifacts['smpe-zip']['source'] = smpeTarSource
          releaseArtifacts['smpe-zip']['target'] = "zowe-smpe-package-${params.ZOWE_RELEASE_VERSION}.zip".toString()
        }

        try {
          def smpePtfPromote = pipeline.artifactory.getArtifact([
            'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_SMPE_PTF_PROMOTE_FILEPATTERN}",
            'build-name'   : releaseArtifacts['zowe']['buildName'],
            'build-number' : releaseArtifacts['zowe']['buildNumber']
          ])
          echo ">>> Found SMP/e promote tar ${smpePtfPromote['path']}."
          smpePtfPromoteTarPath = smpePtfPromote['path']
        } catch (e2) {
          echo ">>> no SMP/e promote tar found in the build, throwing error and exit"
          error "no SMP/e promote tar found in the build"
        }
      } catch (e1) {
        echo ">>> no SMP/e zip found in the build, throwing error and exit"
        error "no SMP/e zip found in the build"
      }

      // get Docker images - amd64
      try {
        def dockerImageAmd64 = pipeline.artifactory.getArtifact([
          'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_DOCKER_AMD64_FILEPATTERN}",
          'build-name'   : releaseArtifacts['zowe']['buildName'],
          'build-number' : releaseArtifacts['zowe']['buildNumber']
        ])
        if (dockerImageAmd64['path']) {
          echo ">>> Found Docker image amd64 version ${dockerImageAmd64['path']}."
          def FMID = dockerImageAmd64['path'].split('/').last().split('-').first()
          releaseArtifacts['docker-amd64'] = [:]
          releaseArtifacts['docker-amd64']['source'] = dockerImageAmd64
          releaseArtifacts['docker-amd64']['target'] = "server-bundle.amd64-${params.ZOWE_RELEASE_VERSION}.tar".toString()
        }
      } catch (e1) {
        echo ">>> no Docker image amd64 version found in the build."
      }
      // get Docker images with sources - amd64
      try {
        def dockerImageSourcesAmd64 = pipeline.artifactory.getArtifact([
          'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_DOCKER_AMD64_SOURCE_FILEPATTERN}",
          'build-name'   : releaseArtifacts['zowe']['buildName'],
          'build-number' : releaseArtifacts['zowe']['buildNumber']
        ])
        if (dockerImageSourcesAmd64['path']) {
          echo ">>> Found Docker image amd64 sources version ${dockerImageSourcesAmd64['path']}."
          def FMID = dockerImageSourcesAmd64['path'].split('/').last().split('-').first()
          releaseArtifacts['docker-amd64-sources'] = [:]
          releaseArtifacts['docker-amd64-sources']['source'] = dockerImageSourcesAmd64
          releaseArtifacts['docker-amd64-sources']['target'] = "server-bundle.sources.amd64-${params.ZOWE_RELEASE_VERSION}.tar".toString()
        }
      } catch (e1) {
        echo ">>> no Docker image amd64 sources version found in the build."
      }
      
      // get Docker images - s390x
      try {
        def dockerImageS390x = pipeline.artifactory.getArtifact([
          'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_DOCKER_S390X_FILEPATTERN}",
          'build-name'   : releaseArtifacts['zowe']['buildName'],
          'build-number' : releaseArtifacts['zowe']['buildNumber']
        ])
        if (dockerImageS390x['path']) {
          echo ">>> Found Docker image s390x version ${dockerImageS390x['path']}."
          def FMID = dockerImageS390x['path'].split('/').last().split('-').first()
          releaseArtifacts['docker-s390x'] = [:]
          releaseArtifacts['docker-s390x']['source'] = dockerImageS390x
          releaseArtifacts['docker-s390x']['target'] = "server-bundle.s390x-${params.ZOWE_RELEASE_VERSION}.tar".toString()
        }
      } catch (e1) {
        echo ">>> no Docker image s390x version found in the build."
      }
      // get Docker images with sources - s390x
      try {
        def dockerImageSourcesS390x = pipeline.artifactory.getArtifact([
          'pattern'      : "${params.ZOWE_BUILD_REPOSITORY}/${params.ZOWE_BUILD_PATH}/${ZOWE_RELEASE_DOCKER_S390X_SOURCE_FILEPATTERN}",
          'build-name'   : releaseArtifacts['zowe']['buildName'],
          'build-number' : releaseArtifacts['zowe']['buildNumber']
        ])
        if (dockerImageSourcesS390x['path']) {
          echo ">>> Found Docker image s390x sources version ${dockerImageSourcesS390x['path']}."
          def FMID = dockerImageSourcesS390x['path'].split('/').last().split('-').first()
          releaseArtifacts['docker-s390x-sources'] = [:]
          releaseArtifacts['docker-s390x-sources']['source'] = dockerImageSourcesS390x
          releaseArtifacts['docker-s390x-sources']['target'] = "server-bundle.sources.s390x-${params.ZOWE_RELEASE_VERSION}.tar".toString()
        }
      } catch (e1) {
        echo ">>> no Docker image s390x sources version found in the build."
      }

      
      // find the Zowe CLI build number will be promoted
      releaseArtifacts['cli'] = [:]
      releaseArtifacts['cli']['target'] = "zowe-cli-package-${params.ZOWE_RELEASE_VERSION}.zip".toString()
      releaseArtifacts['cli-plugins'] = [:]
      releaseArtifacts['cli-plugins']['target'] = "zowe-cli-plugins-${params.ZOWE_RELEASE_VERSION}.zip".toString()

      // CLI Core
      releaseArtifacts['cli']['buildName'] = params.ZOWE_CLI_BUILD_NAME
      releaseArtifacts['cli']['buildNumber'] = params.ZOWE_CLI_BUILD_NUMBER

    // CLI Plugins
      releaseArtifacts['cli-plugins']['buildName'] = params.ZOWE_CLI_BUILD_NAME
      releaseArtifacts['cli-plugins']['buildNumber'] = params.ZOWE_CLI_BUILD_NUMBER

      // get CLI CORE build source artifact
      releaseArtifacts['cli']['source'] = pipeline.artifactory.getArtifact([
        'pattern'      : "${params.ZOWE_CLI_BUILD_REPOSITORY}/${params.ZOWE_CLI_BUILD_PATH}/${ZOWE_RELEASE_CLI_CORE_FILEPATTERN}",
        'build-name'   : releaseArtifacts['cli']['buildName'],
        'build-number' : releaseArtifacts['cli']['buildNumber']
      ])
      echo ">>> Found Zowe CLI build ${releaseArtifacts['cli']['source']['path']}."

      // get CLI PLUGINS builds source artifact
      releaseArtifacts['cli-plugins']['source'] = pipeline.artifactory.getArtifact([
        'pattern'      : "${params.ZOWE_CLI_BUILD_REPOSITORY}/${params.ZOWE_CLI_BUILD_PATH}/${ZOWE_RELEASE_CLI_PLUGINS_FILEPATTERN}",
        'build-name'   : releaseArtifacts['cli-plugins']['buildName'],
        'build-number' : releaseArtifacts['cli-plugins']['buildNumber']
      ])
      echo ">>> Found Zowe CLI Plugins ${releaseArtifacts['cli-plugins']['source']['path']}."

      // try to get Zowe CLI CORE build commit hash
      def cliBuildInfo = pipeline.artifactory.getBuildInfo(
        releaseArtifacts['cli']['buildName'],
        releaseArtifacts['cli']['buildNumber'],
        'zowe-cli'
      )
      // releaseArtifacts['cli']['revision'] = cliBuildInfo && cliBuildInfo['vcsRevision']
      // if (!("${releaseArtifacts['cli']['revision']}" ==~ /^[0-9a-fA-F]{40}$/)) { // if it's a SHA-1 commit hash
      //   error "Cannot extract git revision from build \"${releaseArtifacts['cli']['buildName']}/${releaseArtifacts['cli']['buildNumber']}\""
      // }
      // echo ">>>> Build ${releaseArtifacts['cli']['buildName']}/${releaseArtifacts['cli']['buildNumber']} commit hash is ${releaseArtifacts['cli']['revision']}, may proceed."

      // try to get Zowe CLI Plugins build commit hash
      def cliPluginsBuildInfo = pipeline.artifactory.getBuildInfo(
        releaseArtifacts['cli-plugins']['buildName'],
        releaseArtifacts['cli-plugins']['buildNumber'],
        'zowe-cli'
      )
      // releaseArtifacts['cli-plugins']['revision'] = cliPluginsBuildInfo && cliPluginsBuildInfo['vcsRevision']
      // if (!("${releaseArtifacts['cli-plugins']['revision']}" ==~ /^[0-9a-fA-F]{40}$/)) { // if it's a SHA-1 commit hash
      //   error "Cannot extract git revision from build \"${releaseArtifacts['cli-plugins']['buildName']}/${releaseArtifacts['cli-plugins']['buildNumber']}\""
      // }
      // echo ">>>> Build ${releaseArtifacts['cli-plugins']['buildName']}/${releaseArtifacts['cli-plugins']['buildNumber']} commit hash is ${releaseArtifacts['cli-plugins']['revision']}, may proceed."
    }
  )

  pipeline.createStage(
    name          : "Generate Source Build",
    isSkippable   : true,
    stage         : {
      // files should be downloaded to .release/source_zip
      withCredentials([
        usernamePassword(
          credentialsId: lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
          passwordVariable: 'GITHUB_PASSWORD',
          usernameVariable: 'GITHUB_USERNAME'
        )
      ]) {
        sh './.dependency/prepare_source_zip.sh'
      }
      // zip files
      zip dir: '.release/source_zip', glob: '', zipFile: ".release/zowe_sources-${params.ZOWE_RELEASE_VERSION}.zip"
    },
    timeout: [time: 10, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Promote",
    isSkippable   : true,
    stage         : {
      // debug show releaseArtifacts
      writeJSON file: '.tmp-release-artifacts.json', json: releaseArtifacts, pretty: 2
      sh "set +x\n" +
          "echo All artifacts will be promoted:\n" +
          "echo ===============================================\n" +
          "cat .tmp-release-artifacts.json\n" +
          "echo\n" +
          "echo ===============================================\n"

      releaseArtifacts.each { artifact, info ->
        echo ">>> Promoting ${artifact} ..."
        pipeline.artifactory.promote([
          'source': info['source'],
          'targetPath': releaseFilePath,
          'targetName': info['target']
        ])
      }
    },
    timeout: [time: 10, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Download",
    isSkippable   : true,
    stage         : {
      // download all promoted artifacts into .release folder
      sh 'mkdir -p .release'
      def downloadArtifacts = readJSON(text:'{"files":[]}')
      def expected = releaseArtifacts.size()

      releaseArtifacts.each { artifact, info ->
        downloadArtifacts['files'].push([
          "pattern" : "${releaseFilePath}/${info['target']}".toString(),
          "target"  : '.release/',
          "flat"    : 'true'
        ])
      }

      // debug show downloadArtifacts
      writeJSON file: '.tmp-download-artifacts.json', json: downloadArtifacts, pretty: 2
      sh "set +x\n" +
          "echo All promoted artifacts:\n" +
          "echo ===============================================\n" +
          "cat .tmp-download-artifacts.json\n" +
          "echo\n" +
          "echo ==============================================="

        echo ">>> Downloading all promoted artifacts ..."
        pipeline.artifactory.download([
          'spec'     : '.tmp-download-artifacts.json',
          'expected' : expected
        ])
    },
    timeout: [time: 20, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Sign",
    isSkippable   : true,
    shouldExecute : {
      return isFormalRelease
    },
    stage         : {
      def signing = lib.package.Signing.new(this)
      def uploadArtifacts = readJSON(text:'{"files":[]}')

      // pload zowe_sources
      uploadArtifacts['files'].push([
        "pattern" : '.release/zowe_sources-*.zip',
        "target"  : releaseFilePath + '/'
      ])

      releaseArtifacts.each { artifact, info ->
        echo ">>> Signing ${info['target']} ..."
        signing.sign(".release/${info['target']}")
        uploadArtifacts['files'].push([
          "pattern" : ".release/${info['target']}.asc".toString(),
          "target"  : releaseFilePath + '/'
        ])

        echo ">>> Generate hash of ${info['target']} ..."
        signing.hash(".release/${info['target']}")
        uploadArtifacts['files'].push([
          "pattern" : ".release/${info['target']}.sha512".toString(),
          "target"  : releaseFilePath + '/'
        ])
      }

      // write code-signing-key-info.json
      def signingKeyId = signing.getSigningKey()
      sh "curl -o .release/code-signing-key-info.json https://raw.githubusercontent.com/zowe/zowe-install-packaging/master/signing_keys/${signingKeyId}.json"
      if (!fileExists('.release/code-signing-key-info.json')) {
        error "Failed to download code signing key info json"
      }
      uploadArtifacts['files'].push([
        "pattern" : '.release/code-signing-key-info.json',
        "target"  : releaseFilePath + '/'
      ])

      // write version, no need to upload to Artifactory
      writeFile file: '.release/version', text: params.ZOWE_RELEASE_VERSION

      // debug show uploadArtifacts
      writeJSON file: '.tmp-upload-artifacts.json', json: uploadArtifacts, pretty: 2
      sh "set +x\n" +
          "echo All signing results:\n" +
          "echo ===============================================\n" +
          "cat .tmp-upload-artifacts.json\n" +
          "echo\n" +
          "echo ===============================================\n"

      echo ">>> Uploading signing results ..."
      pipeline.artifactory.upload([spec: '.tmp-upload-artifacts.json'])
    },
    timeout: [time: 20, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Tag Branch",
    isSkippable   : true,
    shouldExecute : {
      return isFormalRelease && releaseArtifacts['zowe'] && releaseArtifacts['zowe']['revision']
    },
    stage         : {
      echo ">>> Tagging ${releaseArtifacts['zowe']['revision']} as v${params.ZOWE_RELEASE_VERSION} ..."

      github.cloneRepository([
        email                      : lib.Constants.DEFAULT_GITHUB_ROBOT_EMAIL,
        usernamePasswordCredential : lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
        repository                 : ZOWE_INSTALL_PACKAGING_REPO,
        folder                     : '.tag',
        branch                     : 'master'
      ])
      github.command('git checkout ' + releaseArtifacts['zowe']['revision'])
      def commit = github.getLastCommitHash()
      if (commit != releaseArtifacts['zowe']['revision']) {
        error "Failed to checkout ${releaseArtifacts['zowe']['revision']} for tagging."
      }
      github.tag("v${params.ZOWE_RELEASE_VERSION}")
    },
    timeout: [time: 10, unit: 'MINUTES']
  )

  pipeline.createStage(
    name          : "Promote PTF",
    isSkippable   : true,
    shouldExecute : {
      return isFormalRelease && releaseArtifacts['zowe'] && releaseArtifacts['zowe']['revision'] && smpePtfPromoteTarPath
    },
    stage         : {
      def gitPromotePtf = lib.scm.GitHub.new(this)
      def pathPromotePtfGit = '.promote-ptf'
      def pathPromotePtfTmp = '.release/promote-ptf'

      // clone staging branch
      gitPromotePtf.cloneRepository([
        email                      : lib.Constants.DEFAULT_GITHUB_ROBOT_EMAIL,
        usernamePasswordCredential : lib.Constants.DEFAULT_GITHUB_ROBOT_CREDENTIAL,
        repository                 : ZOWE_INSTALL_PACKAGING_REPO,
        folder                     : pathPromotePtfGit,
        branch                     : 'staging'
      ])

      // checkout new branch for PR
      gitPromotePtf.checkout("users/robot/promote-ptf-from-${currentBuild.number}", true)

      // download smpe-promote zip
      pipeline.artifactory.download([
        'specContent' : ([
          "{\"files\":[{",
          "\"pattern\":\"${smpePtfPromoteTarPath}\",",
          "\"target\":\"${pathPromotePtfTmp}/smpe-promote.tar\",",
          "\"flat\":\"true\"",
          "}]}",
        ].join("")),
        'expected'    : 1
      ])

      // run smpe-promote.sh
      def pwd = sh(script: 'pwd', returnStdout: true).trim()
      dir("${pathPromotePtfGit}/smpe/bld") {
        // use current branch version smpe-promote.sh
        sh "cp smpe-promote.sh smpe-promote.sh.bak && cp \"${pwd}/smpe/bld/smpe-promote.sh\" ."
        sh "./smpe-promote.sh -d -p \"${pwd}/${pathPromotePtfTmp}/smpe-promote.tar\""
        sh "mv smpe-promote.sh.bak smpe-promote.sh"
        // revert smpe-promote.sh, don't commit in this PR
      }

      // show change status
      echo gitPromotePtf.command('git status')
      // commit all changes
      gitPromotePtf.commit("Promote PTF after release v${params.ZOWE_RELEASE_VERSION}")
      // push before creating PR
      gitPromotePtf.push()

      // create PR
      def pullRequestId = gitPromotePtf.createPullRequest('staging', 'Automatically Promote PTF after release by zowe robot')
      echo "Pull request #${pullRequestId} is created."
    },
    timeout: [time: 10, unit: 'MINUTES']
  )

  pipeline.createStage(
    name             : "Publish Docker Image",
    isSkippable      : true,
    displayTimestamp : true,
    timeout          : [time: 120, unit: 'MINUTES'],
    shouldExecute    : {
      return isFormalRelease && (releaseArtifacts['docker-amd64'] || releaseArtifacts['docker-s390x'])
    },
    stage            : {
      withCredentials([usernamePassword(
        credentialsId: 'ZoweDockerhub',
        usernameVariable: 'USERNAME',
        passwordVariable: 'PASSWORD'
      )]) {
        def publishCommands = "docker login -u \"${USERNAME}\" -p \"${PASSWORD}\"\n"
        def manifestAmends = ""
        def manifestSourceAmends = ""
        // load images
        if (releaseArtifacts['docker-amd64']) {
          // the image in tar should always be ompzowe/server-bundle:amd64
          sh "docker load --input .release/server-bundle.amd64-*.tar"
          publishCommands += "docker push ompzowe/server-bundle:amd64\n"
          manifestAmends += " --amend ompzowe/server-bundle:amd64"
        }
        if (releaseArtifacts['docker-amd64-sources']) {
          // the image in tar should always be ompzowe/server-bundle:amd64-sources
          sh "docker load --input .release/server-bundle.sources.amd64-*.tar"
          publishCommands += "docker push ompzowe/server-bundle:amd64-sources\n"
          manifestSourceAmends += " --amend ompzowe/server-bundle:amd64-sources"
        }
        if (releaseArtifacts['docker-s390x']) {
          // the image in tar should always be ompzowe/server-bundle:s390x
          sh "docker load --input .release/server-bundle.s390x-*.tar"
          publishCommands += "docker push ompzowe/server-bundle:s390x\n"
          manifestAmends += " --amend ompzowe/server-bundle:s390x"
        }
        if (releaseArtifacts['docker-s390x-sources']) {
          // the image in tar should always be ompzowe/server-bundle:amd64-sources
          sh "docker load --input .release/server-bundle.sources.s390x-*.tar"
          publishCommands += "docker push ompzowe/server-bundle:s390x-sources\n"
          manifestSourceAmends += " --amend ompzowe/server-bundle:s390x-sources"
        }
        
        // publish images
        sh publishCommands +
          "export DOCKER_CLI_EXPERIMENTAL=enabled\n" +
          "docker manifest create ompzowe/server-bundle:latest ${manifestAmends}\n" +
          "docker manifest push ompzowe/server-bundle:latest\n" +
          "docker manifest create ompzowe/server-bundle:v${params.ZOWE_RELEASE_VERSION} ${manifestAmends}\n" +
          "docker manifest push ompzowe/server-bundle:v${params.ZOWE_RELEASE_VERSION}"
        // publish source images
        if (manifestSourceAmends) {
          sh publishCommands +
          "export DOCKER_CLI_EXPERIMENTAL=enabled\n" +
          "docker manifest create ompzowe/server-bundle:latest-sources ${manifestSourceAmends}\n" +
          "docker manifest push ompzowe/server-bundle:latest-sources\n" +
          "docker manifest create ompzowe/server-bundle:v${params.ZOWE_RELEASE_VERSION}-sources ${manifestSourceAmends}\n" +
          "docker manifest push ompzowe/server-bundle:v${params.ZOWE_RELEASE_VERSION}-sources"
        }
      }
    }
  )

  pipeline.createStage(
    name          : "Message",
    isSkippable   : true,
    stage         : {
      // update build description with release version
      currentBuild.description = "v${params.ZOWE_RELEASE_VERSION}".toString()

      def source = "Build #${params.ZOWE_BUILD_NUMBER}"

      def smpeMessage = ''
      if (releaseArtifacts['smpe-zip'] && releaseArtifacts['smpe-zip']['target']) {
smpeMessage = """
SMP/e build:
${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['smpe-zip']['target']}

"""
      }
def dockeramd64Message = ''
      if (releaseArtifacts['docker-amd64'] && releaseArtifacts['docker-amd64']['target']) {
dockeramd64Message = """
docker amd64 image:
${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['docker-amd64']['target']}

"""
      }
def dockers390xMessage = ''
      if (releaseArtifacts['docker-s390x'] && releaseArtifacts['docker-s390x']['target']) {
dockers390xMessage = """
docker s390x image:
${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['docker-s390x']['target']}

"""
      }

      def successMsg = """
*************************************************************************************************

${source} is promoted as Zowe v${params.ZOWE_RELEASE_VERSION}, you can download from:

${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['zowe']['target']}
${smpeMessage}
${dockeramd64Message}
${dockers390xMessage}
The CLI Core Package is published here:
${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['cli']['target']}

The CLI Plugins Package is published here:
${lib.Constants.DEFAULT_LFJ_ARTIFACTORY_URL}/${releaseFilePath}/${releaseArtifacts['cli-plugins']['target']}

*************************************************************************************************
      """
      echo successMsg
    },
    timeout: [time: 2, unit: 'MINUTES']
  )

  pipeline.end()
}