diff --git a/.circleci/config.yml b/.circleci/config.yml index 353aaa8..7abf730 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,21 @@ -version: 2 -jobs: - build: +version: 2.1 + +executors: + node-executor: working_directory: ~/phovea docker: - - image: caleydo/phovea_circleci_python:v3.0 - - image: docker:17.05.0-ce-git + - image: circleci/python:3.7.11-buster-node-browsers # for node version see Dockerfile on https://hub.docker.com/r/circleci/python + +orbs: + aws-ecr: circleci/aws-ecr@6.15.3 + +jobs: + build: + executor: node-executor steps: - checkout - - setup_remote_docker + - setup_remote_docker + - aws-ecr/ecr-login - run: name: Show Node.js and npm version command: | @@ -17,78 +25,43 @@ jobs: name: Show Python and pip version command: | python --version - pip --version + pip --version - restore_cache: - key: dependency-cache-{{ checksum "package.json" }} + key: deps1-{{ .Branch }}-{{ checksum "package.json" }} - run: name: Install npm dependencies command: npm install - save_cache: - key: dependency-cache-{{ checksum "package.json" }} - paths: - - ./node_modules - - run: - name: Show installed npm dependencies - command: npm list --depth=1 || true - - restore_cache: - key: awscli-1.11.113 - - run: - name: Install AWS CLI - command: | - virtualenv ~/venv - . ~/venv/bin/activate - pip install awscli==1.11.113 - - save_cache: - key: awscli-1.11.113 - paths: - - ~/venv - - run: - name: Login AWS ECR and DockerHub - command: | - . ~/venv/bin/activate - cat > ~/.dockercfg << EOF - { - "https://index.docker.io/v1/": { - "auth": "$DOCKER_AUTH" - } - } - EOF - export AWS_DEFAULT_REGION=eu-central-1 - login="$(aws ecr get-login --no-include-email)" - ${login} + key: deps1-{{ .Branch }}-{{ checksum "package.json" }} + paths: ./node_modules - deploy: name: Build and deploy command: | - . ~/venv/bin/activate - case $CIRCLE_BRANCH in + case "${CIRCLE_BRANCH}${CIRCLE_TAG}" in master) awsTag="latest" ;; *) - awsTag="${CIRCLE_BRANCH}" + awsTag="${CIRCLE_BRANCH//\//_}${CIRCLE_TAG}" # replace `/` with `_` in branch name ;; esac echo "using tag: --${awsTag}--" - node build.js --skipSaveImage --skipTests --noDefaultTags --pushExtra=${awsTag} --pushTo=922145058410.dkr.ecr.eu-central-1.amazonaws.com/caleydo + node build.js --injectVersion --skipSaveImage --noDefaultTags --pushExtra=${awsTag} --pushTo=922145058410.dkr.ecr.eu-central-1.amazonaws.com/caleydo - store_artifacts: path: build - prefix: build + destination: build - deploy: name: Cleanup untagged AWS repositories command: | - . ~/venv/bin/activate - export AWS_DEFAULT_REGION=eu-central-1 baseName=${CIRCLE_PROJECT_REPONAME%_product} # list repos filter to just the one of this product and delete untagged ones - aws ecr describe-repositories --output text | cut -f5 | grep "caleydo/${baseName}" | while read line; do aws ecr list-images --repository-name $line --filter tagStatus=UNTAGGED --query 'imageIds[*]' --output text | while read imageId; do aws ecr batch-delete-image --output text --repository-name $line --image-ids imageDigest=$imageId; done; done + aws ecr describe-repositories --output text | cut -f6 | grep "caleydo/${baseName}" | while read line; do aws ecr list-images --repository-name $line --filter tagStatus=UNTAGGED --query 'imageIds[*]' --output text | while read imageId; do aws ecr batch-delete-image --output text --repository-name $line --image-ids imageDigest=$imageId; done; done - deploy: name: Restart AWS task # assumes the task definition is called - command: | - . ~/venv/bin/activate - export AWS_DEFAULT_REGION=eu-central-1 # cleanup name by removing the _product suffix baseName=${CIRCLE_PROJECT_REPONAME%_product} - awsFamily="${baseName}-${CIRCLE_BRANCH}" + awsFamily="${baseName}-${CIRCLE_BRANCH//\//_}" # replace `/` with `_` in branch name echo "awsFamily --${awsFamily}--" tasksExists=$(aws --output text ecs list-task-definitions --family-prefix ${awsFamily}) echo "existsTaskDefinition? --${tasksExists}--" @@ -103,7 +76,7 @@ jobs: aws --output text ecs run-task --cluster caleydo --task-definition ${awsFamily} --started-by CircleCIAutoUpdate fi workflows: - version: 2 + version: 2.1 # build-nightly: # triggers: # - schedule: @@ -117,12 +90,16 @@ workflows: build-branch: jobs: - build: + context: + - org-global filters: tags: ignore: /^v.*/ build-tag: jobs: - build: + context: + - org-global filters: branches: ignore: /.*/ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..9836a90 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @thinkh diff --git a/.gitignore b/.gitignore index 2f10738..c65cefb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,29 @@ -/.tscache -/.idea -/build/ -/dist/ -node_modules/ -/src/**/*.js -/tests/**/*.js -*.map -*.css -*.log -/tmp*/ +# ignore all files and plugin repos of the workspace ... +.* +*.* +/*/ +* +package-lock.json + +# ... except all files relevant for the product build +!/.circleci/ +!/.circleci/**/* +!/.github/ +!/.github/**/* +!templates/ +!templates/web/ +!templates/web/**/* +!templates/api/ +!templates/api/**/* +!.editorconfig +!.gitattributes +!.yo-rc.json +!ISSUE_TEMPLATE.md +!LICENSE +!README.md +!build.js +!package.json +!phovea_product.* +!Jenkinsfile +!docker_script.sh +!docker-compose-patch.* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f9b94b0..0000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -language: node_js - -node_js: -- 6 - -services: -- docker - -before_install: -- export DISPLAY=:99.0 -- sh -e /etc/init.d/xvfb start -- if [[ `npm -v` != 3* ]]; then npm i -g npm@3; fi - -before_script: -- pip install --user awscli -- export PATH=$PATH:$HOME/.local/bin - -# build and push -#- $(aws ecr get-login --region eu-central-1) -#script: node build.js --skipTests --skipSaveImage --pushTo=$AWS_ECR_PREFIX --pushLatest --pushDaily - -script: node build.js --skipSaveImage - -deploy: - provider: releases - api_key: - secure: TK9/P34Bi3WuppiDrBCwVcn41yCBwmILaU8hXTBzUPbT7TbeFIwsC6/4CtH85Z+ZrUve4S5pTmWRNf2dQDxWw3uYu7+bJuemV2J1LHG76mognj+TNEiYxfLQUt3Gql4W7C7FcI4Rlx5/uMN9wY1wro8TWUBMwT6jjSrUWIvK3GXoojd5bHvJx07XpjWl9wCon4D0ruZiFoM2mdeP23lbc2GckETi32oEKswnQXxkMACmxbPzoWbvkxH4aK8Bt2Rj2sl2TbPhVkN6DAkHGkGAvLI+2/aRfG27+oo3OKsaDjbuGABct8TfZccJ970CbQ8kbnCjYxstvqkg1JWjF0W67sX/flBZZOEUA5l0OLWo6HqMGMxm7/lEQhIdPMsRmvXL+HVOxkMrB2dda58QzxVwiZp+rRqUaeabPZp8Kl5xodGrVxsBvxe6zAbJ5jCtCSumG6+kLyKI00/kYlghqQNrgUw0ZsYJlQ34h3lo/24QpaeyDpQoCkGWQgtgqiXGpeKSu7bCnOqIqAy3nbT9Utwj7K8gIasTG5idosEAz/THMampNbGDuyxxc340sYGNMg9Bhm1g2ILWRdtV470p5hwBtIDTKi3/PAizEO26+Wh0zI47Sg3ao57avcbCsTmzbZUeA5J4bojmchhJCHX8su9cSCGh/2fJA/1eBIgEvOQ8LNE= - file: build/* - on: - tags: true - -notifications: - slack: - secure: E8/1UIdHSczUbN+6i6gd1d5LM4vmLdwLQ30tpyjvnM0wvfDce76oPxLJAy240WJ5ybXRZUtNrttpVpt4tEXCy8aLFCmxD7s77rVloH+q1J8R/ptTFWZGhFGEujk1awEmVbzcWxJkV9/JENQaeGBKxwv8/EQwWwEkAb7p/+AJb9owmH88b3wUZUGHBWtbMiyyaF4Rm1Wg1stJB8Z1Ga7PRF4cqufTgcDdsCPVv9gAY+VxOIGqX/Vfuc9UWpUH8vq8lHUE7Inn5QS78kuFfSgLWga3H6Mu/Gko1XNlWk0QWWQBUvEZ6ZC6Wuo68KzvUjJHDTnx8WyfHue2JNHIslcX+eJq2WHLeEgM24VeNkILCGo/H/60NGHiSjrIv/Y9h6bQ9FDjo6TUyE4nbdPYN1RN9FQ5UbI9Y4Gi753H9mqnHWlEywBOzHxdZCAuz9Wh03CCF/blsvJ+Obbyo6Jrfe+g44jyi9kQdBNQ78qG6v4EXws8FiYao6x3PpgIwFix42Cpr+soAh5FpA3C1zHSAyZZpXF65/lrDl5yPNofK7Wy0B9bw+0I6Z/u7ZKFNVZXvYPGYvtUVcsALGBdmYc61+LCta36Po0KZseWVAlJj6QnOJDYzv0wvV/zsuf9A5KpYFGiqV9Q7zmtiO5FYF5sBy+lE7O9tHVO4O18IRndhRQgxhs= - on_success: change - on_failure: always diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 44e9742..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,46 +0,0 @@ -node { - stage('Checkout') { - checkout scm - } - - stage('Before Install') { - def nodeHome = tool 'node-v7' - env.PATH="${env.PATH}:${nodeHome}/bin" - def dockerHome = tool 'docker' - env.PATH="${env.PATH}:${dockerHome}/bin" - } - - stage('Install') { - sh 'node -v' - sh 'npm --version' - sh 'docker --version' - sh 'npm install' - } - - stage('Build') { - try { - withCredentials([usernameColonPassword(credentialsId: 'PHOVEA_GITHUB_CREDENTIALS', variable: 'PHOVEA_GITHUB_CREDENTIALS')]) { - docker.withRegistry("https://922145058410.dkr.ecr.eu-central-1.amazonaws.com", "ecr:eu-central-1:PHOVEA_AWS_CREDENTIALS") { - docker.withRegistry("", "PHOVEA_DOCKER_HUB_CREDENTIALS") { - wrap([$class: 'Xvfb']) { - sh 'node build.js --skipTests --skipSaveImage --noDefaultTags --pushExtra=latest --pushTo=922145058410.dkr.ecr.eu-central-1.amazonaws.com/caleydo' - } - } - } - } - currentBuild.result = "SUCCESS" - } catch (e) { - // if any exception occurs, mark the build as failed - currentBuild.result = 'FAILURE' - throw e - } finally { - // always clean up - sh 'npm prune' - sh 'rm node_modules -rf' - } - } - - stage('Post Build') { - archiveArtifacts artifacts: 'build/*' - } -} diff --git a/README.md b/README.md index d6e5a1c..8b68b43 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ -vega_clue_product [![Phovea][phovea-image]][phovea-url] [![NPM version][npm-image]][npm-url] [![Build Status][travis-image]][travis-url] [![Dependency Status][daviddm-image]][daviddm-url] +DEPRECATED: vega_clue_product ===================== +[![Phovea][phovea-image]][phovea-url] [![Build Status][circleci-image]][circleci-url] - +### DEPRECATION Information +Please note that this project has been archived and is no longer being maintained. There is an active development under https://github.com/datavisyn/tdp_core and we will also contribute our future changes to it. Installation ------------ @@ -36,9 +38,5 @@ This repository is part of **[Phovea](http://phovea.caleydo.org/)**, a platform [phovea-image]: https://img.shields.io/badge/Phovea-Product-FABC15.svg [phovea-url]: https://phovea.caleydo.org -[npm-image]: https://badge.fury.io/js/vega_clue_product.svg -[npm-url]: https://npmjs.org/package/vega_clue_product -[travis-image]: https://travis-ci.org/Caleydo/vega_clue_product.svg?branch=master -[travis-url]: https://travis-ci.org/Caleydo/vega_clue_product -[daviddm-image]: https://david-dm.org/Caleydo/vega_clue_product/status.svg -[daviddm-url]: https://david-dm.org/Caleydo/vega_clue_product +[circleci-image]: https://circleci.com/gh/Caleydo/vega_clue_product.svg?style=shield +[circleci-url]: https://circleci.com/gh/Caleydo/vega_clue_product diff --git a/build.js b/build.js index bef9fb1..92ea0cb 100644 --- a/build.js +++ b/build.js @@ -14,7 +14,7 @@ const quiet = argv.quiet !== undefined; const now = new Date(); const prefix = (n) => n < 10 ? ('0' + n) : n.toString(); -const buildId = `${now.getUTCFullYear()}${prefix(now.getUTCMonth())}${prefix(now.getUTCDate())}-${prefix(now.getUTCHours())}${prefix(now.getUTCMinutes())}${prefix(now.getUTCSeconds())}`; +const buildId = `${now.getUTCFullYear()}${prefix(now.getUTCMonth()+1)}${prefix(now.getUTCDate())}-${prefix(now.getUTCHours())}${prefix(now.getUTCMinutes())}${prefix(now.getUTCSeconds())}`; pkg.version = pkg.version.replace('SNAPSHOT', buildId); const env = Object.assign({}, process.env); const productName = pkg.name.replace('_product', ''); @@ -24,18 +24,17 @@ function showHelp(steps, chain) { possible options: * --quiet ... reduce log messages * --serial ... build elements sequentially - * --skipTests ... skip tests + * --skipTests ... skip tests: will set the environment variable PHOVEA_SKIP_TESTS * --injectVersion ... injects the product version into the package.json of the built component * --useSSH ... clone via ssh instead of https * --skipCleanUp ... skip cleaning up old docker images * --skipSaveImage ... skip saving the generated docker images * --pushTo ... push docker images to the given registry - * --noDefaultTags ... don't push generated default tag : + * --noDefaultTags ... do not push generated default tag : * --pushExtra ... push additional custom tag: e.g., --pushExtra=develop * --forceLabel ... force to use the label even only a single service exists * --dryRun ... just compute chain no execution * --help ... show this help message - arguments: (starting with --!) optional list of steps to execute in the given order (expert mode) by default the default chain is executed `); @@ -297,7 +296,6 @@ function yo(generator, options, cwd, args) { const yeoman = require('yeoman-environment'); // call yo internally const yeomanEnv = yeoman.createEnv([], {cwd, env}, quiet ? createQuietTerminalAdapter() : undefined); - const _args = Array.isArray(args) ? args.join(' ') : args || ''; return new Promise((resolve, reject) => { try { @@ -393,14 +391,31 @@ function patchWorkspace(p) { fs.writeFileSync(p.tmpDir + '/docker_script.sh', content); } + function injectVersion(targetPkgFile, targetVersion) { + if (fs.existsSync(targetPkgFile)) { + const ppkg = require(targetPkgFile); + ppkg.version = targetVersion; + console.log(`Write version ${targetVersion} into ${targetPkgFile}`); + fs.writeJSONSync(targetPkgFile, ppkg, {spaces: 2}); + } else { + console.warn(`Cannot inject version: ${targetPkgFile} not found`); + } + } + if (argv.injectVersion) { - const pkgfile = `${p.tmpDir}/${p.name}/package.json`; - if (fs.existsSync(pkgfile)) { - const ppkg = require(pkgfile); - ppkg.version = pkg.version; - fs.writeJSONSync(pkgfile, ppkg); + const targetPkgFile = `${p.tmpDir}/package.json`; + // inject version of product package.json into workspace package.json + injectVersion(targetPkgFile, pkg.version); + } else { + // read default app package.json + const defaultAppPkgFile = `${p.tmpDir}/${p.name}/package.json`; + if (fs.existsSync(defaultAppPkgFile)) { + const sourcePkg = require(defaultAppPkgFile); + const targetPkgFile = `${p.tmpDir}/package.json`; + // inject version of default app package.json into workspace package.json + injectVersion(targetPkgFile, sourcePkg.version); } else { - console.warn('cannot inject version, main package.json not found'); + console.warn(`Cannot read version from default app package.json: ${defaultAppPkgFile} not found`); } } @@ -416,6 +431,16 @@ function patchWorkspace(p) { `; fs.writeFileSync(p.tmpDir + '/phovea_registry.js', registry); } + //copy template files of product to workspace of product + if (fs.existsSync(`./templates/${p.type}/deploy/${p.label}`)) { + console.log(`Copy deploy files from`, `./templates/${p.type}/deploy/${p.label}`, `to`, `${p.tmpDir}/deploy/${p.label}`); + fs.copySync(`./templates/${p.type}/deploy/${p.label}`, `${p.tmpDir}/deploy/${p.label}`); + } else if (fs.existsSync(`./templates/${p.type}`)) { + console.log(`Copy deploy files from`, `./templates/${p.type}`, `to`, `${p.tmpDir}/`); + fs.copySync(`./templates/${p.type}`, `${p.tmpDir}/`); + } + + } function mergeCompose(composePartials) { @@ -608,7 +633,7 @@ function strObject(items) { } function buildDockerImage(p) { - const buildInSubDir = p.type === 'web' || p.type === 'static'; + const buildInSubDir = p.type === 'static'; let buildArgs = ''; // pass through http_proxy, no_proxy, and https_proxy env variables for (const key of Object.keys(process.env)) { @@ -618,26 +643,46 @@ function buildDockerImage(p) { buildArgs += ` --build-arg ${lkey}='${process.env[key]}'`; } } + const additionalType = (label, type) => { + return fs.existsSync(`./templates/${type}/deploy/${label}`); + } + let dockerFile; + // check if label exists and use type as fallback + if (additionalType(p.label, p.type) && (p.type === 'web' || p.type === 'api')) { + dockerFile = `deploy/${p.label}/Dockerfile`; + } else if (p.type === 'web' || p.type === 'api') { + dockerFile = `deploy/${p.type}/Dockerfile`; + } else { + dockerFile = `deploy/Dockerfile`; + } + console.log('use dockerfile: ' + dockerFile); // patch the docker file with the with an optional given baseImage - return Promise.resolve(patchDockerfile(p, `${p.tmpDir}${buildInSubDir ? '/' + p.name : ''}/deploy/Dockerfile`)) + return Promise.resolve(patchDockerfile(p, `${p.tmpDir}${buildInSubDir ? '/' + p.name : ''}/${dockerFile}`)) // create the container image - .then(() => docker(`${p.tmpDir}${buildInSubDir ? '/' + p.name : ''}`, `build -t ${p.image}${buildArgs} -f deploy/Dockerfile .`)) + .then(() => docker(`${p.tmpDir}${buildInSubDir ? '/' + p.name : ''}`, `build -t ${p.image}${buildArgs} -f ${dockerFile} .`)) // tag the container image .then(() => argv.pushExtra ? docker(`${p.tmpDir}`, `tag ${p.image} ${p.image.substring(0, p.image.lastIndexOf(':'))}:${argv.pushExtra}`) : null); } function createWorkspace(p) { - return yo('workspace', {noAdditionals: true, defaultApp: 'phovea'}, p.tmpDir) + return yo('workspace', {noAdditionals: true, defaultApp: p.name, addWorkspaceRepos: false}, p.tmpDir) .then(() => patchWorkspace(p)); } function installWebDependencies(p) { - return npm(p.additional.length > 0 ? p.tmpDir : (`${p.tmpDir}/${p.name}`), 'install'); + return npm(p.tmpDir, 'install'); +} + +function showWebDependencies(p) { + // `npm ls` fails if some peerDependencies are not installed + // since this function is for debug purposes only, we catch possible errors of `npm()` and resolve it with status code `0`. + return npm(p.tmpDir, 'list --depth=1') + .catch(() => Promise.resolve(0)); // status code = 0 } function cleanUpWebDependencies(p) { - return fs.emptyDirAsync(p.additional.length > 0 ? `${p.tmpDir}/node_modules` : (`${p.tmpDir}/${p.name}/node_modules`)); + return fs.emptyDirAsync(`${p.tmpDir}/node_modules` ); } function resolvePluginTypes(p) { @@ -650,21 +695,10 @@ function resolvePluginTypes(p) { return Promise.all([resolvePluginType(p, p.tmpDir)].concat(p.additional.map((pi) => resolvePluginType(pi, p.tmpDir)))); } -function testWebAdditionals(p) { - return Promise.all(p.additional.map((pi) => npm(p.tmpDir, `run test${pi.isHybridType ? ':web' : ''}:${pi.name}`))); -} - function buildWeb(p) { - const hasAdditional = p.additional.length > 0; - - let step; - if (hasAdditional) { - step = npm(p.tmpDir, `run dist${p.isHybridType ? ':web' : ''}:${p.name}`); - } else { - step = npm(`${p.tmpDir}/${p.name}`, `run dist${p.isHybridType ? ':web' : ''}`); - } + const step = npm(p.tmpDir, `run dist`); // move to target directory - return step.then(() => fs.renameAsync(`${p.tmpDir}/${p.name}/dist/${p.name}.tar.gz`, `./build/${p.label}.tar.gz`)); + return step.then(() => fs.renameAsync(`${p.tmpDir}/dist/bundles.tar.gz`, `./build/${p.label}.tar.gz`)); } function installPythonTestDependencies(p) { @@ -673,6 +707,12 @@ function installPythonTestDependencies(p) { .then(() => spawn('pip', 'install --no-cache-dir -r requirements_dev.txt', {cwd: p.tmpDir})); } +function showPythonTestDependencies(p) { + // since this function is for debug purposes only, we catch possible errors and resolve it with status code `0`. + return spawn('pip', 'list', {cwd: p.tmpDir}) + .catch(() => Promise.resolve(0)); // status code = 0 +} + function buildServer(p) { let act = npm(`${p.tmpDir}/${p.name}`, `run build${p.isHybridType ? ':python' : ''}`); for (const pi of p.additional) { @@ -685,11 +725,6 @@ function buildServer(p) { .then(() => fs.copyAsync(`${p.tmpDir}/${p.name}/build/source`, `${p.tmpDir}/build/source/`)) .then(() => Promise.all(p.additional.map((pi) => fs.copyAsync(`${p.tmpDir}/${pi.name}/build/source`, `${p.tmpDir}/build/source/`)))); - // copy main deploy thing and create a docker out of it - act = act - .then(() => fs.ensureDirAsync(`${p.tmpDir}/deploy`)) - .then(() => fs.copyAsync(`${p.tmpDir}/${p.name}/deploy`, `${p.tmpDir}/deploy/`)); - return act; } @@ -798,27 +833,31 @@ if (require.main === module) { } } - const needsWorskpace = (isWeb && hasAdditional) || isServer; - steps[`prepare:${suffix}`] = needsWorskpace ? () => catchProductBuild(p, createWorkspace(p)) : null; + const needsWorkspace = isWeb || isServer; + if(needsWorkspace) { + steps[`prepare:${suffix}`] = () => catchProductBuild(p, createWorkspace(p)); + } if (isWeb) { steps[`install:${suffix}`] = () => catchProductBuild(p, installWebDependencies(p)); + steps[`show:${suffix}`] = () => catchProductBuild(p, showWebDependencies(p)); } else { // server steps[`install:${suffix}`] = argv.skipTests ? () => null : () => catchProductBuild(p, installPythonTestDependencies(p)); + steps[`show:${suffix}`] = () => catchProductBuild(p, showPythonTestDependencies(p)); } - steps[`test:${suffix}`] = isWeb && hasAdditional ? () => catchProductBuild(p, resolvePluginTypes(p).then(() => testWebAdditionals(p))) : () => null; steps[`build:${suffix}`] = isWeb ? () => catchProductBuild(p, resolvePluginTypes(p).then(() => buildWeb(p))) : () => catchProductBuild(p, resolvePluginTypes(p).then(() => buildServer(p))); steps[`data:${suffix}`] = () => catchProductBuild(p, downloadServerDataFiles(p)); steps[`postbuild:${suffix}`] = isWeb ? () => catchProductBuild(p, cleanUpWebDependencies(p)) : () => null; steps[`image:${suffix}`] = () => catchProductBuild(p, buildDockerImage(p)); steps[`save:${suffix}`] = () => catchProductBuild(p, dockerSave(p.image, `build/${p.label}_image.tar.gz`)); - subSteps.push(`prepare:${suffix}`); - subSteps.push(`install:${suffix}`); - if (!argv.skipTests) { - subSteps.push(`test:${suffix}`); + if(needsWorkspace) { + subSteps.push(`prepare:${suffix}`); } + subSteps.push(`install:${suffix}`); + subSteps.push(`show:${suffix}`); subSteps.push(`build:${suffix}`); + if (isServer && p.data.length > 0) { subSteps.push(`data:${suffix}`); } @@ -838,7 +877,7 @@ if (require.main === module) { // create some meta steps { const stepNames = Object.keys(steps); - for (const meta of ['clone', 'prepare', 'build', 'test', 'postbuild', 'image', 'product', 'install']) { + for (const meta of ['clone', 'prepare', 'build', 'postbuild', 'image', 'product', 'install', 'show']) { const sub = stepNames.filter((d) => d.startsWith(`${meta}:`)); if (sub.length <= 0) { continue; @@ -895,4 +934,4 @@ if (require.main === module) { if (!argv.dryRun) { launch(); } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 47e6481..1e47d9b 100644 --- a/package.json +++ b/package.json @@ -1,24 +1,8 @@ { - "scripts": { - "build": "node build.js --skipTests", - "predist": "npm run build", - "dist": "mkdirp dist && cd build && tar cvzf ../dist/vega_clue_product.tar.gz *" - }, - "dependencies": { - "bluebird": "3.4.6", - "chalk": "1.1.3", - "fs-extra": "^1.0.0", - "generator-phovea": "^2.0.0", - "lodash": "4.17.14", - "mkdirp": "0.5.1", - "yamljs": "0.2.8", - "yargs-parser": "4.2.0", - "yeoman-environment": "1.6.6" - }, "name": "vega_clue_product", "description": "", "homepage": "https://phovea.caleydo.org", - "version": "1.0.0-SNAPSHOT", + "version": "2.0.0", "author": { "name": "The Caleydo Team", "email": "contact@caleydo.org", @@ -31,5 +15,20 @@ "repository": { "type": "git", "url": "https://github.com/Caleydo/vega_clue_product.git" + }, + "scripts": { + "build": "node build.js --skipTests", + "predist": "npm run build", + "dist": "mkdir dist && cd build && tar cvzf ../dist/vega_clue_product.tar.gz *" + }, + "dependencies": { + "bluebird": "3.7.2", + "chalk": "3.0.0", + "fs-extra": "^9.0.1", + "generator-phovea": "^8.0.0", + "lodash": "4.17.20", + "yamljs": "0.3.0", + "yargs-parser": "18.0.0", + "yeoman-environment": "2.8.0" } } diff --git a/phovea_product.json b/phovea_product.json index dc5cc50..4bba153 100644 --- a/phovea_product.json +++ b/phovea_product.json @@ -3,7 +3,7 @@ "type": "web", "label": "vega_clue", "repo": "Caleydo/vega_clue", - "branch": "master", + "branch": "v2.0.0", "additional": [ { "name": "phovea_core", @@ -18,7 +18,12 @@ { "name": "phovea_ui", "repo": "phovea/phovea_ui", - "branch": "v2.1.3" + "branch": "thinkh/provenance_retrieval" + }, + { + "name": "phovea_security_flask", + "repo": "phovea/phovea_security_flask", + "branch": "thinkh/provenance_retrieval" } ] }, @@ -26,35 +31,28 @@ "type": "api", "label": "vega_clue_server", "repo": "phovea/phovea_server", - "branch": "v2.4.0", + "branch": "v7.2.0", "additional": [ { "name": "phovea_security_flask", "repo": "phovea/phovea_security_flask", - "branch": "v2.3.0" + "branch": "thinkh/provenance_retrieval" }, { "name": "phovea_data_redis", "repo": "phovea/phovea_data_redis", - "branch": "v2.3.0" + "branch": "v8.0.0" }, { "name": "phovea_data_mongo", "repo": "phovea/phovea_data_mongo", - "branch": "v2.2.0" + "branch": "v8.0.0" }, { "name": "phovea_clue", "repo": "phovea/phovea_clue", "branch": "thinkh/provenance_retrieval" } - ], - "data": [ - { - "type": "repo", - "repo": "Caleydo/vega_clue", - "branch": "master" - } ] } ] diff --git a/templates/api/deploy/api/Dockerfile b/templates/api/deploy/api/Dockerfile new file mode 100644 index 0000000..85ea6e9 --- /dev/null +++ b/templates/api/deploy/api/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.7-buster + +LABEL maintainer="contact@caleydo.org" +WORKDIR /phovea + +# install dependencies last step such that everything before can be cached +COPY requirements*.txt docker_packages.txt docker_script*.sh _docker_data* ./ +RUN (!(test -s docker_packages.txt) || (apt-get update && \ + (cat docker_packages.txt | xargs apt-get install -y))) && \ + (pip install --no-cache-dir -r requirements.txt) +RUN (!(test -s docker_script.sh) || (bash ./docker_script.sh)) + +COPY ./build/source ./ + +# Create separate launch file (similar to phovea_server/__main__.py from repository root), because it is not included in build files +RUN printf "from phovea_server import launch\nlaunch.run()\n" > /phovea/launch.py + +ENV PHOVEA_ENV=production +ENV PHOVEA_SERVICE=api +ENV PHOVEA_CONFIG_PATH=config.json +CMD python launch.py --env ${PHOVEA_ENV} ${PHOVEA_SERVICE} +EXPOSE 80 diff --git a/templates/api/deploy/api/Dockerfile_dev b/templates/api/deploy/api/Dockerfile_dev new file mode 100644 index 0000000..cc8fe09 --- /dev/null +++ b/templates/api/deploy/api/Dockerfile_dev @@ -0,0 +1,44 @@ +FROM python:3.7-buster + +LABEL maintainer="contact@caleydo.org" + +ARG http_proxy +ARG HTTP_PROXY +ARG https_proxy +ARG HTTPS_PROXY +ARG no_proxy +ARG NO_PROXY + +WORKDIR /phovea + +# install dependencies last step such that everything before can be cached +COPY requirements*.txt docker_packages.txt docker_script*.sh _docker_data* ./ +RUN (!(test -s docker_packages.txt) || (apt-get -y update && cat docker_packages.txt | xargs apt-get install -y)) && \ + pip install --no-cache-dir -r requirements.txt && \ + (pip install --no-cache-dir -r requirements_dev.txt) +RUN (!(test -s docker_script.sh) || bash ./docker_script.sh) + +#### +# Environment mode (dev or prod) +#### +ENV PHOVEA_ENV=dev + +#### +# The name must match the registred command in /phovea_server/phovea_server/__init__.py +# Example: `registry.append('command', 'api', 'phovea_server.server', {'isDefault': True})` +#### +ENV PHOVEA_SERVICE=api + +#### +# The path to the phovea config.json +# In a local workspace setup the /config.json is used here. +#### +ENV PHOVEA_CONFIG_PATH=config.json + +#### +# Use `phovea_server` as entry point and add some arguments and the service as command. +# In a local workspace setup it will call the /phovea_server/__main__.py, which runs the /phovea_server/launcher.py +#### +CMD python phovea_server/__main__.py --use_reloader --env ${PHOVEA_ENV} ${PHOVEA_SERVICE} + +EXPOSE 80 diff --git a/templates/web/deploy/web/404.html b/templates/web/deploy/web/404.html new file mode 100644 index 0000000..45a4f42 --- /dev/null +++ b/templates/web/deploy/web/404.html @@ -0,0 +1,64 @@ + + + + + Error 404 + + + + + +
+

Error 404

+

Sorry, but the page you were trying to view does not exist. + It looks like this was the result of either a mistyped address or an out-of-date link. +

+
+ + diff --git a/templates/web/deploy/web/50x.html b/templates/web/deploy/web/50x.html new file mode 100644 index 0000000..9c77357 --- /dev/null +++ b/templates/web/deploy/web/50x.html @@ -0,0 +1,62 @@ + + + + + Error 50X + + + + + +
+

Error 50x

+

Something went wrong. Sorry for the inconvenience, we are working on it.

+
+ + \ No newline at end of file diff --git a/templates/web/deploy/web/Dockerfile b/templates/web/deploy/web/Dockerfile new file mode 100644 index 0000000..e75191f --- /dev/null +++ b/templates/web/deploy/web/Dockerfile @@ -0,0 +1,10 @@ +FROM nginx:alpine + +LABEL maintainer="contact@caleydo.org" + +ENV PHOVEA_API_SERVER=api +ENV PHOVEA_NGINX_PORT=80 +COPY ./deploy/web/nginx-default.conf /etc/nginx/conf.d/default.conf +CMD sed -i -e "s/PHOVEA_API_SERVER/${PHOVEA_API_SERVER-api}/g" /etc/nginx/conf.d/default.conf && sed -i -e "s/PHOVEA_NGINX_PORT/${PHOVEA_NGINX_PORT-api}/g" /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;' +COPY ./bundles /usr/share/nginx/html +COPY ./deploy/web/*.html /usr/share/nginx/html/ diff --git a/templates/web/deploy/web/nginx-default.conf b/templates/web/deploy/web/nginx-default.conf new file mode 100644 index 0000000..fc87236 --- /dev/null +++ b/templates/web/deploy/web/nginx-default.conf @@ -0,0 +1,82 @@ +# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the +# scheme used to connect to this server +map $http_x_forwarded_proto $proxy_x_forwarded_proto { +default $http_x_forwarded_proto; +'' $scheme; +} + +# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any +# Connection header that may have been passed to this server +map $http_upgrade $proxy_connection { +default upgrade; +'' close; +} + +map "${sent_http_etag}${sent_http_last_modified}${sent_http_cache_control}${request_method}${status}" $expires { +default off; +'GET200' 6h; +} + +server { + listen PHOVEA_NGINX_PORT; + server_name localhost; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + ssi on; + root /usr/share/nginx/html/; + index 50x.html; + internal; + } + + error_page 404 /404.html; + location = /404.html { + ssi on; + root /usr/share/nginx/html/; + index 404.html; + internal; + } + + location / { + expires modified +12h; + add_header Cache-Control public; + + root /usr/share/nginx/html; + index index.html index.htm; + } + + location ~ ^/(api|login|logout|loggedinas) { + + proxy_pass http://PHOVEA_API_SERVER; + # HTTP 1.1 support + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header Host $http_host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $proxy_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + + # Mitigate httpoxy attack (see README for details) + proxy_set_header Proxy ""; + + # timeout in 600sec = 10min + proxy_connect_timeout 600; + proxy_send_timeout 600; + proxy_read_timeout 600; + send_timeout 600; + client_max_body_size 64m; + + add_header Cache-Control private; + expires $expires; + + proxy_intercept_errors on; + } +} + +gzip on; +gzip_proxied no_etag; +gzip_types text/plain text/comma-separated-values image/svg+xml text/xml text/css application/x-javascript application/javascript application/xml application/xml+rss text/csv application/json text/javascript application/xhtml+xml; +gzip_vary on; +gzip_disable "MSIE [1-6]\.(?!.*SV1)"; diff --git a/templates/web/workspace.scss b/templates/web/workspace.scss new file mode 100644 index 0000000..ebaf5f4 --- /dev/null +++ b/templates/web/workspace.scss @@ -0,0 +1,24 @@ +@debug('import plugin scss variables'); + +// import all variables scss +// first default app +@import "~vega_clue/dist/scss/abstracts/variables"; + +// then going up in the dependency tree from the default app +@import "~phovea_clue/dist/scss/abstracts/variables"; +@import "~phovea_ui/dist/scss/abstracts/variables"; +@import "~phovea_security_flask/dist/scss/abstracts/variables"; + +// then all other plugins in this workspace (excluding other apps) +@import "~@fortawesome/fontawesome-free/scss/variables"; + +// import mixins +@import "~@fortawesome/fontawesome-free/scss/mixins"; + +@debug('import plugin main scss'); + +// all main scss (in opposite order of the variables scss +@import "~phovea_security_flask/dist/scss/main"; +@import "~phovea_ui/dist/scss/main"; +@import "~phovea_clue/dist/scss/main"; +@import "~vega_clue/dist/scss/main";