-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #74 from milvus-io/0.5.0
0.5.0 Release Former-commit-id: c6737497902ae6fc1f23918f1788214bc5a5b3fe
- Loading branch information
Showing
161 changed files
with
13,262 additions
and
1,203 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| pipeline { | ||
| agent none | ||
|
|
||
| options { | ||
| timestamps() | ||
| } | ||
|
|
||
| parameters{ | ||
| choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE' | ||
| string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true | ||
| string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true | ||
| string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true | ||
| string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true | ||
| string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true | ||
| } | ||
|
|
||
| environment { | ||
| PROJECT_NAME = "milvus" | ||
| LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase() | ||
| SEMVER = "${BRANCH_NAME}" | ||
| JOBNAMES = env.JOB_NAME.split('/') | ||
| PIPELINE_NAME = "${JOBNAMES[0]}" | ||
| } | ||
|
|
||
| stages { | ||
| stage("Ubuntu 18.04") { | ||
| environment { | ||
| OS_NAME = "ubuntu18.04" | ||
| PACKAGE_VERSION = VersionNumber([ | ||
| versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' | ||
| ]); | ||
| DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}" | ||
| } | ||
|
|
||
| stages { | ||
| stage("Run Build") { | ||
| agent { | ||
| kubernetes { | ||
| label 'build' | ||
| defaultContainer 'jnlp' | ||
| yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml' | ||
| } | ||
| } | ||
|
|
||
| stages { | ||
| stage('Build') { | ||
| steps { | ||
| container('milvus-build-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| stage('Code Coverage') { | ||
| steps { | ||
| container('milvus-build-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/coverage.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| stage('Upload Package') { | ||
| steps { | ||
| container('milvus-build-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| stage("Publish docker images") { | ||
| agent { | ||
| kubernetes { | ||
| label 'publish' | ||
| defaultContainer 'jnlp' | ||
| yamlFile 'ci/jenkins/pod/docker-pod.yaml' | ||
| } | ||
| } | ||
|
|
||
| stages { | ||
| stage('Publish') { | ||
| steps { | ||
| container('publish-images'){ | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| stage("Deploy to Development") { | ||
| agent { | ||
| kubernetes { | ||
| label 'dev-test' | ||
| defaultContainer 'jnlp' | ||
| yamlFile 'ci/jenkins/pod/testEnvironment.yaml' | ||
| } | ||
| } | ||
|
|
||
| stages { | ||
| stage("Deploy to Dev") { | ||
| steps { | ||
| container('milvus-test-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| stage("Dev Test") { | ||
| steps { | ||
| container('milvus-test-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| stage ("Cleanup Dev") { | ||
| steps { | ||
| container('milvus-test-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| post { | ||
| unsuccessful { | ||
| container('milvus-test-env') { | ||
| script { | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| timeout(time: 60, unit: 'MINUTES') { | ||
| dir ("ci/jenkins/scripts") { | ||
| sh "./build.sh -l" | ||
| withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { | ||
| sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c" | ||
| } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| try { | ||
| sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" | ||
| } catch (exc) { | ||
| def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true | ||
| if (!helmResult) { | ||
| sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" | ||
| } | ||
| throw exc | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| timeout(time: 60, unit: 'MINUTES') { | ||
| dir ("ci/jenkins/scripts") { | ||
| sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP" | ||
| // Set some env variables so codecov detection script works correctly | ||
| withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) { | ||
| sh 'curl -s https://codecov.io/bash | bash -s - -f output_new.info || echo "Codecov did not collect coverage reports"' | ||
| } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| try { | ||
| sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' | ||
| sh 'helm repo update' | ||
| dir ('milvus-helm') { | ||
| checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) | ||
| dir ("milvus-gpu") { | ||
| sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ." | ||
| } | ||
| } | ||
| } catch (exc) { | ||
| echo 'Helm running failed!' | ||
| sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" | ||
| throw exc | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| timeout(time: 5, unit: 'MINUTES') { | ||
| sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" | ||
| withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { | ||
| def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") | ||
| if (uploadStatus != 0) { | ||
| error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!") | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| container('publish-images') { | ||
| timeout(time: 15, unit: 'MINUTES') { | ||
| dir ("docker/deploy/${OS_NAME}") { | ||
| def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" | ||
|
|
||
| withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { | ||
| def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") | ||
|
|
||
| if (downloadStatus != 0) { | ||
| error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") | ||
| } | ||
| } | ||
| sh "tar zxvf ${binaryPackage}" | ||
| def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" | ||
|
|
||
| try { | ||
| def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") | ||
| if (isExistSourceImage == 0) { | ||
| def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") | ||
| } | ||
|
|
||
| def customImage = docker.build("${imageName}") | ||
|
|
||
| def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") | ||
| if (isExistTargeImage == 0) { | ||
| def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") | ||
| } | ||
|
|
||
| docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") { | ||
| customImage.push() | ||
| } | ||
| } catch (exc) { | ||
| throw exc | ||
| } finally { | ||
| def isExistSourceImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") | ||
| if (isExistSourceImage == 0) { | ||
| def removeSourceImageStatus = sh(returnStatus: true, script: "docker rmi ${imageName}") | ||
| } | ||
|
|
||
| def isExistTargeImage = sh(returnStatus: true, script: "docker inspect --type=image ${params.DOKCER_REGISTRY_URL}/${imageName} 2>&1 > /dev/null") | ||
| if (isExistTargeImage == 0) { | ||
| def removeTargeImageStatus = sh(returnStatus: true, script: "docker rmi ${params.DOKCER_REGISTRY_URL}/${imageName}") | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| timeout(time: 30, unit: 'MINUTES') { | ||
| dir ("tests/milvus_python_test") { | ||
| sh 'python3 -m pip install -r requirements.txt' | ||
| sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" | ||
| } | ||
| // mysql database backend test | ||
| load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" | ||
|
|
||
| if (!fileExists('milvus-helm')) { | ||
| dir ("milvus-helm") { | ||
| checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) | ||
| } | ||
| } | ||
| dir ("milvus-helm") { | ||
| dir ("milvus-gpu") { | ||
| sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml --namespace milvus ." | ||
| } | ||
| } | ||
| dir ("tests/milvus_python_test") { | ||
| sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| labels: | ||
| app: publish | ||
| componet: docker | ||
| spec: | ||
| containers: | ||
| - name: publish-images | ||
| image: registry.zilliz.com/library/docker:v1.0.0 | ||
| securityContext: | ||
| privileged: true | ||
| command: | ||
| - cat | ||
| tty: true | ||
| volumeMounts: | ||
| - name: docker-sock | ||
| mountPath: /var/run/docker.sock | ||
| volumes: | ||
| - name: docker-sock | ||
| hostPath: | ||
| path: /var/run/docker.sock | ||
|
|
Oops, something went wrong.