diff --git a/.dockerignore b/.dockerignore index 3e75f5950..d972cb1fd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,11 @@ -.git +* -# --- Below Should mirror .gitignore, without leading '/' --- # -.DS_Store -.*sw[op] -*.py[co] -*.egg-info -persistent -runtime -bootstrap.json -.cache -.coverage* -coverage.xml -htmlcov -node_modules +!api +!bin +!docker/uwsgi-config.ini +!docker/uwsgi-entrypoint.sh +!requirements.txt +!setup.py +!swagger/examples +!swagger/schemas +!tests/requirements.txt diff --git a/.gitignore b/.gitignore index 022b7b09b..5a352d41a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,14 @@ -.DS_Store -.*sw[op] -*.py[co] *.egg-info +*.py[co] +.*sw[op] +.cache +.DS_Store +/.coverage* +/bin/accesslog.csv +/htmlcov /persistent /runtime bootstrap.json -.cache -/.coverage* coverage.xml endpoints.json -/htmlcov -node_modules/ -/bin/accesslog.csv +node_modules diff --git a/.travis.yml b/.travis.yml index a3ce03413..5c8ff06e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,45 +1,59 @@ -sudo: required dist: trusty +sudo: required services: - - mongodb +- docker + env: global: - secure: HELJx6WPr+W2S0FV47KkRdlS9NCqlMcdRMK8xWgRqqrEPv24KEvNnHxCy0tRbzITqadYtyvI1MtqtmpG04uty8Gpkc7w6L6LMJ/OuLG0gVX7AnaUovYTlY04m1/L9oyzOrTDXk5J/BKbcyiz7uJtkTc/A8MMZAFfZh7hmhLID78= # BUILD_TRIGGER_URL - -install: - - bin/install-ubuntu.sh - - tests/bin/setup-integration-tests-ubuntu.sh + - DOCKER_DIR="$HOME/.cache/docker" + - secure: HhT1TdJcpqys8juVMw/DIZeK7oD4595TEKH5KlowH7MvwwFAUyQFb5W63F8dgk7elvRG+3fmga/m1JfXO+Iu7PVD912eiNDagW9aB3CEl3Z8zg+JUL8IjpMCkyKQDyJMnfOkrzdxdaqfOK+WmF+13f2qBu9Kc7wdXuzgHQrg4+0= # CI_REGISTRY_USER + - secure: hh7VDZnkxgl/vqHtS4IpXfIAckKpVQvoCzNW7fstr5Mcu8KNiCWIPgObBRm+m13aqpcFTMWQ6lT2kzORz2wWRbDeVhI1eGWOJswGNHPHZLO0Jaei6yfY2nY2mpxZbl+vdg00jkN64mi1ab3e++QgeLFruW0gyNefXX7E5L/mHTs= # CI_REGISTRY_PASS -script: - - SCITRAN_PERSISTENT_DB_PORT=27017 tests/bin/run-tests-ubuntu.sh +cache: + directories: + - $DOCKER_DIR -after_success: - - if [ "$TRAVIS_EVENT_TYPE" == "push" -o "$TRAVIS_TAG" ]; then - SSH_KEY_FILE=$(mktemp -p $HOME/.ssh/); +before_install: +- sudo apt-get update +- sudo apt-get -y install docker-ce realpath +- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASS - openssl aes-256-cbc -K $encrypted_55750ae1fbc7_key -iv $encrypted_55750ae1fbc7_iv -in .github_deploy_key.enc -out "$SSH_KEY_FILE" -d; +install: true - chmod 600 "$SSH_KEY_FILE" && printf "%s\n" \ - "Host github.com" \ - " IdentityFile $SSH_KEY_FILE" \ - " LogLevel ERROR" >> ~/.ssh/config; +script: +- test -f "$DOCKER_DIR/image.tar" && docker load -i "$DOCKER_DIR/image.tar" || true +- docker build -t core:base --target base . +- docker build -t core:dist --target dist --build-arg VCS_BRANCH="$TRAVIS_BRANCH" --build-arg VCS_COMMIT="$TRAVIS_COMMIT" . +- docker build -t core:testing --target testing . +- docker save -o "$DOCKER_DIR/image.tar" $(docker history -q core:base | grep -v '') $(docker history -q core:dist | grep -v '') +- ./tests/bin/docker-tests.sh --image core:testing - git config --global user.email "travis@travis-ci.org"; - git config --global user.name "Travis CI"; - git config --global push.default simple; - fi - - if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then - bash <(curl -s https://codecov.io/bash) -cF python; - fi - - if [ "$TRAVIS_TAG" ]; then - ./docker/build-trigger.sh Tag "$TRAVIS_TAG" "$BUILD_TRIGGER_URL"; - ./bin/push-docs.sh "$GIT_REMOTE" tags "$TRAVIS_TAG" "Travis Core Docs Build - ${TRAVIS_BUILD_NUMBER}"; - fi - - if [ "$TRAVIS_EVENT_TYPE" == "push" -a "$TRAVIS_BRANCH" == "master" ]; then - ./docker/build-trigger.sh Branch "$TRAVIS_BRANCH" "$BUILD_TRIGGER_URL"; - fi - - if [ "$TRAVIS_EVENT_TYPE" == "push" -a -z "$TRAVIS_TAG" ]; then - ./bin/push-docs.sh "$GIT_REMOTE" branches "$TRAVIS_BRANCH" "Travis Core Docs Build - ${TRAVIS_BUILD_NUMBER}"; - fi - +after_success: +- | + if [ "$TRAVIS_EVENT_TYPE" == "push" -o "$TRAVIS_TAG" ]; then + SSH_KEY_FILE=$(mktemp -p $HOME/.ssh/); + openssl aes-256-cbc -K $encrypted_55750ae1fbc7_key -iv $encrypted_55750ae1fbc7_iv -in .github_deploy_key.enc -out $SSH_KEY_FILE -d; + chmod 600 $SSH_KEY_FILE; + printf "%s\n" \ + "Host github.com" \ + " IdentityFile $SSH_KEY_FILE" \ + " LogLevel ERROR" >> ~/.ssh/config; + git config --global user.email "travis@travis-ci.org"; + git config --global user.name "Travis CI"; + git config --global push.default simple; + fi +- if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then + bash <(curl -s https://codecov.io/bash) -cF python; + fi +- if [ "$TRAVIS_TAG" ]; then + docker tag core:dist scitran/core:$TRAVIS_TAG; + docker push scitran/core:$TRAVIS_TAG; + ./bin/push-docs.sh "$GIT_REMOTE" tags "$TRAVIS_TAG" "Travis Core Docs Build - $TRAVIS_BUILD_NUMBER"; + elif [ "$TRAVIS_EVENT_TYPE" == "push" ]; then + ./bin/push-docs.sh "$GIT_REMOTE" branches "$TRAVIS_BRANCH" "Travis Core Docs Build - $TRAVIS_BUILD_NUMBER"; + fi +- if [ "$TRAVIS_EVENT_TYPE" == "push" -a "$TRAVIS_BRANCH" == "master" ]; then + docker tag core:dist scitran/core:latest; + docker push scitran/core:latest; + fi diff --git a/Dockerfile b/Dockerfile index 3813f9e9a..bee012ac3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,103 +1,90 @@ -# -# Image used for hosting scitran core with uwsgi. -# -# Example usage is in README.md -# - -FROM ubuntu:14.04 - - -# Install pre-requisites -RUN apt-get update \ - && apt-get install -y \ - build-essential \ - ca-certificates curl \ - libatlas3-base \ - numactl \ - python-dev \ - python-pip \ - libffi-dev \ - libssl-dev \ - libpcre3 \ - libpcre3-dev \ - git \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip - - -# Grab gosu for easy step-down from root in a docker-friendly manner -# https://github.com/tianon/gosu -# -# Alternate key servers are due to reliability issues with ha.pool.sks-keyservers.net -RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture)" \ - && curl -o /tmp/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture).asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; do \ - gpg --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \ - done \ - && gpg --batch --verify /tmp/gosu.asc /usr/local/bin/gosu \ - && rm -r "$GNUPGHOME" /tmp/gosu.asc \ - && chmod +x /usr/local/bin/gosu - - -# Setup environment -WORKDIR /var/scitran - -RUN mkdir -p \ - /var/scitran/config \ - /var/scitran/data \ - /var/scitran/code/api \ - /var/scitran/logs \ - /var/scitran/keys - -# Declaring a volume makes the intent to map externally explicit. This enables -# the contents to survive/persist across container versions, and easy access -# to the contents outside the container. -# -# Declaring the VOLUME in the Dockerfile guarantees the contents are empty -# for any new container that doesn't specify a volume map via 'docker run -v ' -# or similar option. -# -VOLUME /var/scitran/keys -VOLUME /var/scitran/data -VOLUME /var/scitran/logs - - -# Install pip modules -# -# Split this out for better cache re-use. -# -COPY requirements.txt docker/requirements-docker.txt /var/scitran/code/api/ - -RUN pip install --upgrade pip wheel setuptools \ - && pip install -r /var/scitran/code/api/requirements-docker.txt \ - && pip install -r /var/scitran/code/api/requirements.txt - -COPY tests /var/scitran/code/api/tests/ -RUN bash -e -x /var/scitran/code/api/tests/bin/setup-integration-tests-ubuntu.sh - - -# Copy full repo -# -COPY . /var/scitran/code/api/ +FROM ubuntu:14.04 as base +ENV TERM=xterm +RUN set -eux \ + && apt-get -yqq update \ + && apt-get -yqq install \ + build-essential \ + ca-certificates \ + curl \ + git \ + libatlas3-base \ + libffi-dev \ + libpcre3 \ + libpcre3-dev \ + libssl-dev \ + numactl \ + python-dev \ + python-pip \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -qq --upgrade pip setuptools wheel \ + && export GNUPGHOME="$(mktemp -d)" \ + && KEYSERVERS="\ + ha.pool.sks-keyservers.net \ + hkp://keyserver.ubuntu.com:80 \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + pgp.mit.edu" \ + && for server in $(shuf -e $KEYSERVERS); do \ + gpg --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || true; \ + done \ + && curl -LSso /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture)" \ + && curl -LSso /tmp/gosu.asc "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture).asc" \ + && gpg --batch --verify /tmp/gosu.asc /usr/local/bin/gosu \ + && chmod +x /usr/local/bin/gosu \ + && rm -rf "$GNUPGHOME" /tmp/gosu.asc \ + && mkdir -p \ + /var/scitran/code/api \ + /var/scitran/config \ + /var/scitran/data \ + /var/scitran/keys \ + /var/scitran/logs + +VOLUME ["/var/scitran/keys", "/var/scitran/data", "/var/scitran/logs"] +WORKDIR /var/scitran/code/api COPY docker/uwsgi-entrypoint.sh /var/scitran/ -COPY docker/uwsgi-config.ini /var/scitran/config/ - - +COPY docker/uwsgi-config.ini /var/scitran/config/ +ENTRYPOINT ["/var/scitran/uwsgi-entrypoint.sh"] +CMD ["uwsgi", "--ini=/var/scitran/config/uwsgi-config.ini", "--http=[::]:9000", \ + "--http-keepalive", "--so-keepalive", "--add-header", "Connection: Keep-Alive"] -# Inject build information into image so the source of the container can be -# determined from within it. -ARG BRANCH_LABEL=NULL -ARG COMMIT_HASH=0 -COPY docker/inject_build_info.sh / -RUN /inject_build_info.sh ${BRANCH_LABEL} ${COMMIT_HASH} \ - && rm /inject_build_info.sh +FROM base as dist +COPY requirements.txt /var/scitran/code/api/requirements.txt +RUN set -eux \ + && pip install -qq --requirement /var/scitran/code/api/requirements.txt -ENTRYPOINT ["/var/scitran/uwsgi-entrypoint.sh"] -CMD ["uwsgi", "--ini", "/var/scitran/config/uwsgi-config.ini", "--http", "0.0.0.0:8080", "--http-keepalive", "--so-keepalive", "--add-header", "Connection: Keep-Alive" ] +COPY . /var/scitran/code/api/ +RUN set -eux \ + && pip install -qq --no-deps --editable /var/scitran/code/api + +ARG VCS_BRANCH=NULL +ARG VCS_COMMIT=NULL +RUN set -eux \ + && /var/scitran/code/api/bin/build_info.sh $VCS_BRANCH $VCS_COMMIT > /var/scitran/version.json \ + && cat /var/scitran/version.json + + +FROM base as testing +ENV MONGO_MAJOR=3.2 \ + MONGO_VERSION=3.2.9 +RUN set -eux \ + && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 \ + && echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/$MONGO_MAJOR multiverse" > /etc/apt/sources.list.d/mongodb-org-$MONGO_MAJOR.list \ + && apt-get -yqq update \ + && apt-get -yqq install \ + mongodb-org=$MONGO_VERSION \ + mongodb-org-server=$MONGO_VERSION \ + mongodb-org-shell=$MONGO_VERSION \ + mongodb-org-mongos=$MONGO_VERSION \ + mongodb-org-tools=$MONGO_VERSION \ + && rm -rf /var/lib/apt/lists/* /var/lib/mongodb \ + && mkdir -p /data/db + +COPY --from=dist /usr/local /usr/local + +COPY tests/requirements.txt /var/scitran/code/api/tests/requirements.txt +RUN set -eux \ + && pip install -qq --requirement /var/scitran/code/api/tests/requirements.txt + +COPY --from=dist /var/scitran /var/scitran diff --git a/README.md b/README.md index 543332ab6..39cddbe08 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ API documentation for branches and tags can be found at `https://scitran.github. ### [Contributing](https://github.com/scitran/core/blob/master/CONTRIBUTING.md) -### [Testing](https://github.com/scitran/core/blob/master/TESTING.md) +### [Testing](https://github.com/scitran/core/blob/master/tests/README.md) ### [License](https://github.com/scitran/core/blob/master/LICENSE) diff --git a/TESTING.md b/TESTING.md deleted file mode 100644 index af53c2100..000000000 --- a/TESTING.md +++ /dev/null @@ -1,35 +0,0 @@ -## Run the tests - -### Ubuntu -Run automated tests: -``` -# Follow installation instructions in README first -. /runtime/bin/activate # Or wherever your scitran virtualenv is -./test/bin/setup-integration-tests-ubuntu.sh -./test/bin/run-tests-ubuntu.sh -``` -All tests are executed by default. Subsets can be run using the filtering options: - -* To run linting, use `--lint` (`-l`) -* To run unit tests, use `--unit` (`-u`) -* To run integration tests, use `--integ` (`-i`) -* To pass any arguments to `py.test`, use `-- PYTEST_ARGS` - -See [py.test usage](https://docs.pytest.org/en/latest/usage.html) for more. - -### Docker -Build scitran-core image and run automated tests in a docker container: -``` -./tests/bin/run-tests-docker.sh -``` -* To skip building the image, use `--no-build` (`-B`) -* To pass any arguments to `run-tests-ubuntu.sh`, use `-- TEST_ARGS` - - -#### Example -Without rebuilding the image, run only integration tests matching `foo`, use the highest verbosity level for test output and jump into a python debugger session in case an assertion fails: -``` -./tests/bin/run-tests-docker.sh -B -- -i -- -k foo -vvv --pdb -``` - -**NOTE:** The mongodb version is pinned via the `MONGO_VERSION` variable in `tests/bin/run-tests-docker.sh`. diff --git a/api/__init__.py b/api/__init__.py index 8b1378917..e69de29bb 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -1 +0,0 @@ - diff --git a/api/wsgi.py b/api/wsgi.py new file mode 100644 index 000000000..9951ff12f --- /dev/null +++ b/api/wsgi.py @@ -0,0 +1,3 @@ +from .web import start + +application = start.app_factory() diff --git a/bin/api.wsgi b/bin/api.wsgi deleted file mode 100644 index 5c2168a53..000000000 --- a/bin/api.wsgi +++ /dev/null @@ -1,4 +0,0 @@ -# vim: filetype=python -from api.web import start - -application = start.app_factory() diff --git a/bin/build_info.sh b/bin/build_info.sh new file mode 100755 index 000000000..a8c2d6245 --- /dev/null +++ b/bin/build_info.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +set -eu + +cat <&2 - exit 1 - ;; - -I|--no-install) - INSTALL_APP=0 - ;; - -r|--reload) - AUTO_RELOAD_INTERVAL=$2 - shift - ;; - -T|--no-testdata) - BOOTSTRAP_TESTDATA=0 - ;; - -U|--no-users) - BOOTSTRAP_USERS=0 - ;; - *) - echo "Invalid option: $key" >&2 - echo $USAGE >&2 - exit 1 - ;; - esac - shift -done - - -set -o allexport - -if [ "$CONFIG_FILE" != "" ]; then - EXISTING_ENV=$(env | grep "SCITRAN_" | cat) - source "$CONFIG_FILE" - eval "$EXISTING_ENV" -fi - -SCITRAN_RUNTIME_PATH=${SCITRAN_RUNTIME_PATH:-"./runtime"} -SCITRAN_RUNTIME_HOST=${SCITRAN_RUNTIME_HOST:-"127.0.0.1"} -SCITRAN_RUNTIME_PORT=${SCITRAN_RUNTIME_PORT:-"8080"} -SCITRAN_RUNTIME_UWSGI_INI=${SCITRAN_RUNTIME_UWSGI_INI:-""} -SCITRAN_RUNTIME_BOOTSTRAP=${SCITRAN_RUNTIME_BOOTSTRAP:-"./bootstrap.json"} -SCITRAN_RUNTIME_SSL_PEM=${SCITRAN_RUNTIME_SSL_PEM:-""} -SCITRAN_RUNTIME_COVERAGE=${SCITRAN_RUNTIME_COVERAGE:-"false"} - -SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$( openssl rand -base64 32 )} - -SCITRAN_PERSISTENT_PATH=${SCITRAN_PERSISTENT_PATH:-"./persistent"} -SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data" -SCITRAN_PERSISTENT_DB_PATH=${SCITRAN_PERSISTENT_DB_PATH:-"$SCITRAN_PERSISTENT_PATH/db"} -SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"} -SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"} - -if [ "$SCITRAN_RUNTIME_SSL_PEM" == "" ]; then - SCITRAN_SITE_API_URL="http://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api" - UWSGI_SOCKET_ARG="--http" - UWSGI_SOCKET_VALUE="$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT" -else - SCITRAN_SITE_API_URL="https://$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT/api" - UWSGI_SOCKET_ARG="--https" UWSGI_SOCKET_VALUE="$SCITRAN_RUNTIME_HOST:$SCITRAN_RUNTIME_PORT,$SCITRAN_RUNTIME_SSL_PEM,$SCITRAN_RUNTIME_SSL_PEM" -fi - -set +o allexport - - -if [ $INSTALL_APP -eq 1 ]; then - ./bin/install-dev-osx.sh -fi - - -clean_up () { - kill $MONGOD_PID || true - kill $UWSGI_PID || true - deactivate || true - wait 2> /dev/null -} -trap clean_up EXIT - - -source "$SCITRAN_RUNTIME_PATH/bin/activate" - - -# Launch MongoDB -ulimit -n 1024 -mkdir -p "$SCITRAN_PERSISTENT_DB_PATH" -mongod \ - --port $SCITRAN_PERSISTENT_DB_PORT \ - --dbpath "$SCITRAN_PERSISTENT_DB_PATH" \ - --smallfiles \ - & -MONGOD_PID=$! - - -# Launch uWSGI -if [ "$SCITRAN_RUNTIME_UWSGI_INI" == "" ]; then - uwsgi \ - "$UWSGI_SOCKET_ARG" "$UWSGI_SOCKET_VALUE" \ - --master --die-on-term \ - --home "$SCITRAN_RUNTIME_PATH" \ - --wsgi-file "bin/api.wsgi" \ - --py-autoreload $AUTO_RELOAD_INTERVAL \ - --logformat '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)" request_id=%(request_id)' & - UWSGI_PID=$! -else - uwsgi --ini "$SCITRAN_RUNTIME_UWSGI_INI" & - UWSGI_PID=$! -fi - - -echo "Waiting for API to become available" -until $(curl --output /dev/null --silent --head --fail --insecure "$SCITRAN_SITE_API_URL"); do - sleep 1 -done - - -# Bootstrap users -if [ $BOOTSTRAP_USERS -eq 1 ]; then - if [ -f "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" ]; then - echo "Users previously bootstrapped. Remove $SCITRAN_PERSISTENT_DB_PATH to re-bootstrap." - else - echo "Bootstrapping users" - PYTHONPATH=. bin/load_users_drone_secret.py \ - --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" \ - "$SCITRAN_SITE_API_URL" \ - "$SCITRAN_RUNTIME_BOOTSTRAP" - - echo "Bootstrapped users" - touch "$SCITRAN_PERSISTENT_DB_PATH/.bootstrapped" - fi -else - echo "NOT bootstrapping users" -fi - - -# Boostrap test data -TESTDATA_REPO="https://github.com/scitran/testdata.git" -if [ $BOOTSTRAP_TESTDATA -eq 1 ]; then - if [ -f "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" ]; then - echo "Data previously bootstrapped. Remove $SCITRAN_PERSISTENT_DATA_PATH to re-bootstrap." - else - if [ ! -d "$SCITRAN_PERSISTENT_PATH/testdata" ]; then - echo "Cloning testdata to $SCITRAN_PERSISTENT_PATH/testdata" - git clone --single-branch $TESTDATA_REPO $SCITRAN_PERSISTENT_PATH/testdata - else - echo "Updating testdata in $SCITRAN_PERSISTENT_PATH/testdata" - git -C $SCITRAN_PERSISTENT_PATH/testdata pull - fi - echo "Ensuring reaper is up to date with master branch" - pip install --upgrade --upgrade-strategy only-if-needed git+https://github.com/scitran/reaper.git - echo "Bootstrapping testdata" - folder_sniper --yes --insecure --secret "$SCITRAN_CORE_DRONE_SECRET" "$SCITRAN_PERSISTENT_PATH/testdata" $SCITRAN_SITE_API_URL - echo "Bootstrapped testdata" - touch "$SCITRAN_PERSISTENT_DATA_PATH/.bootstrapped" - fi -else - echo "NOT bootstrapping testdata" -fi - -wait diff --git a/docker/build-trigger.sh b/docker/build-trigger.sh deleted file mode 100755 index 8c012e8e8..000000000 --- a/docker/build-trigger.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Triggers an auto-build on Docker Hub for the given source control reference. -# -# Example usage: ./build-trigger Tag 1.0.0 https://registry.hub.docker.com/u/scitran/reaper/trigger/11111111-2222-3333-4444-abcdefabcdef/ - -set -e - -if [ $# -ne 3 ] ; then - >&2 echo "Usage: $( basename $0 ) " - exit 1 -fi - -SOURCE_CONTROL_REF_TYPE="${1}" -SOURCE_CONTROL_REF_NAME="${2}" -TRIGGER_URL="${3}" - -if [ -z "${SOURCE_CONTROL_REF_TYPE}" ] ; then - >&2 echo "Source control reference type not provided. Skipping build trigger." - exit 1 -fi - -if [ -z "${SOURCE_CONTROL_REF_NAME}" ] ; then - >&2 echo "Source control tag name not provided. Skipping build trigger." - exit 1 -fi - -TRIGGER_PAYLOAD="{\"source_type\": \"${SOURCE_CONTROL_REF_TYPE}\", \"source_name\": \"${SOURCE_CONTROL_REF_NAME}\"}" -curl -H "Content-Type: application/json" --data "${TRIGGER_PAYLOAD}" -X POST "${TRIGGER_URL}" ->&2 echo ->&2 echo "Docker Hub build for ${SOURCE_CONTROL_REF_TYPE} '${SOURCE_CONTROL_REF_NAME}' triggered." diff --git a/docker/inject_build_info.sh b/docker/inject_build_info.sh deleted file mode 100755 index cd68e40be..000000000 --- a/docker/inject_build_info.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -( - -set -e - -# Set cwd -unset CDPATH -cd "$( dirname "${BASH_SOURCE[0]}" )" - -# Dump the build info into version.json so it can be displayed in the footer -# of the site pages. - -# { -# "commit": "5683785e8cd6efdfd794a79828b2cccd2424ed21", -# "timestamp": "January 12, 2016 at 2:46:23 PM CST", -# "branch": "ng-constant" -# } - - - BRANCH_NAME=${1} - COMMIT_HASH=${2} - BUILD_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - echo "{ - \"commit\": \"${COMMIT_HASH}\", - \"timestamp\": \"${BUILD_TIMESTAMP}\", - \"branch\": \"${BRANCH_NAME}\" -}" > version.json - -cat version.json - -) diff --git a/docker/requirements-docker.txt b/docker/requirements-docker.txt deleted file mode 100644 index ac39e009c..000000000 --- a/docker/requirements-docker.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Service and support dependencies -uWSGI==2.0.11 -ipython==4.0.2 diff --git a/docker/uwsgi-config.ini b/docker/uwsgi-config.ini index 53467c7ec..669b63901 100644 --- a/docker/uwsgi-config.ini +++ b/docker/uwsgi-config.ini @@ -1,8 +1,6 @@ [uwsgi] -wsgi-file = bin/api.wsgi -chdir=code/api -pythonpath=code/data -master = True -die-on-term = True +die-on-term = true +master = true +module = api.wsgi processes = 4 threads = 2 diff --git a/docker/uwsgi-entrypoint.sh b/docker/uwsgi-entrypoint.sh index 4e071a07e..aff9b13ec 100755 --- a/docker/uwsgi-entrypoint.sh +++ b/docker/uwsgi-entrypoint.sh @@ -1,48 +1,22 @@ -#!/bin/bash +#!/usr/bin/env sh -# FDM-831 workaround -# https://github.com/docker/compose/issues/2454 -# remove after docker 1.10.0 becomes minim supported version. -# -# If /etc/hosts has lines starting with tab, it is corrupted, -# exit to allow docker to restart. -grep -P "^\t" /etc/hosts -if [ "$?" == 0 ] ; then - echo "Host mapping in /etc/hosts is buggy, fail contain start." - exit 1 -fi - - -set -e -set -x +set -eu -export PYTHONPATH=/var/scitran/code/api +# /etc/hosts is corrupted if it has lines starting with tab. +# Exit to allow docker to restart. +if grep -P "^\t" /etc/hosts; then + echo "Host mapping in /etc/hosts is buggy, fail contain start." + exit 1 +fi export SCITRAN_PERSISTENT_PATH=/var/scitran/data export SCITRAN_PERSISTENT_DATA_PATH=/var/scitran/data -# Get the RunAs user from the owner of the mapped folder. -# This is a compromise to get permissions to work well with -# host mapped volumes with docker-machine on OSX and production -# without the vbox driver layer. -RUNAS_USER=$(ls -ld "${SCITRAN_PERSISTENT_DATA_PATH}" | awk '{print $3}') - - -if [ "${1:0:1}" = '-' ]; then - set -- uwsgi "$@" -fi - -# run $PRE_RUNAS_CMD as root if provided. Useful for things like JIT pip insalls. -if [ ! -z "${PRE_RUNAS_CMD}" ]; then - ${PRE_RUNAS_CMD} -fi - -if [ "$1" = 'uwsgi' ]; then - - exec gosu ${RUNAS_USER} "$@" -fi +# Set RUNAS_USER based on the owner of the persistent data path. +RUNAS_USER=$(stat -c '%U' $SCITRAN_PERSISTENT_DATA_PATH) -gosu ${RUNAS_USER} "$@" +# Run $PRE_RUNAS_CMD as root if provided. Useful for things like JIT pip installs. +[ -n "${PRE_RUNAS_CMD:-}" ] && eval $PRE_RUNAS_CMD -result=$? -echo "Exit code was $result" +# Use exec to keep PID and use gosu to step-down from root. +exec gosu $RUNAS_USER "$@" diff --git a/requirements.txt b/requirements.txt index e544505b0..1e14080a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ backports.tempfile==1.0 django>=1.11.0,<1.12.0 elasticsearch==5.3.0 enum==0.4.6 +ipython==4.0.2 jsonschema==2.6.0 Markdown==2.6.5 pymongo==3.2 -pyOpenSSL >=17.1.0,<18.0 python-dateutil==2.4.2 pytz==2015.7 -requests==2.9.1 +requests[security]==2.18.4 rfc3987==1.3.4 strict-rfc3339==0.7 unicodecsv==0.9.0 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..4436dba18 --- /dev/null +++ b/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup, find_packages + + +requirements = open('requirements.txt').readlines() +dependency_links = [r for r in requirements if r.startswith('git+')] +install_requires = [r for r in requirements if not r.startswith('git+')] +tests_require = open('tests/requirements.txt').readlines() + + +setup( + name = 'core', + version = '1.0.0', + description = 'Scitran Core API', + author = 'Gunnar Schaefer', + author_email = 'gsfr@flywheel.io', + maintainer = 'Megan Henning, Ambrus Simon', + maintainer_email = 'meganhenning@flywheel.io, ambrussimon@invenshure.com', + url = 'https://github.com/scitran/core', + license = 'MIT', + packages = find_packages(), + dependency_links = dependency_links, + install_requires = install_requires, + tests_require = tests_require, +) diff --git a/.pylintrc b/tests/.pylintrc similarity index 100% rename from .pylintrc rename to tests/.pylintrc diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..f43ba9aa9 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,24 @@ +## Testing +Build scitran-core and run automated tests in a docker container: +``` +./tests/bin/docker-tests.sh +``` + +* To skip building the image, use `--no-build` (`-B`) +* All tests (unit, integration and linting) are executed by default +* To pass any arguments to `py.test`, use `-- PYTEST_ARGS` + * To run only a subset of test, use the [keyword expression filter](https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests) `-k` + * To see `print` output during tests, increase verbosity with `-vvv` + * To get a debugger session on failures, use [`--pdb`](https://docs.pytest.org/en/latest/usage.html#dropping-to-pdb-python-debugger-on-failures) + +See [py.test usage](https://docs.pytest.org/en/latest/usage.html) for more. + +### Example +The most common use case is adding a new (still failing) test, and wanting to +* (re-)run it as fast as possible (`-B` and `-k foo`) +* see output from quick and dirty `print` statements in the test (`-vvv`) +* get into an interactive pdb session to inspect what went wrong (`--pdb`) + +``` +./tests/bin/docker-tests.sh -B -- -k foo -vvv --pdb +``` diff --git a/tests/bin/docker-tests.sh b/tests/bin/docker-tests.sh new file mode 100755 index 000000000..a2b0390b9 --- /dev/null +++ b/tests/bin/docker-tests.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env sh + +set -eu +unset CDPATH +cd "$( dirname "$0" )/../.." + + +USAGE=" +Usage: + $0 [OPTION...] [-- PYTEST_ARGS...] + +Build scitran/core image and run tests in a Docker container. +Also displays coverage report and saves HTML under htmlcov/ + +Options: + -h, --help Print this help and exit + + -B, --no-build Skip rebuilding default Docker image + --image IMAGE Use custom Docker image + --shell Enter shell instead of running tests + + -- PYTEST_ARGS Arguments passed to py.test + +" + + +main() { + local DOCKER_IMAGE= + local PYTEST_ARGS= + local RUN_SHELL= + + while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + log "$USAGE" + exit 0 + ;; + -B|--no-build) + DOCKER_IMAGE="scitran/core:testing" + ;; + --image) + DOCKER_IMAGE="$2" + shift + ;; + --shell) + RUN_SHELL=true + ;; + --) + shift + PYTEST_ARGS="$@" + break + ;; + *) + log "Invalid argument: $1" + log "$USAGE" + exit 1 + ;; + esac + shift + done + + # Docker build + if [ -z "${DOCKER_IMAGE}" ]; then + log "Building scitran/core:testing ..." + docker build -t scitran/core:testing . + else + docker tag "$DOCKER_IMAGE" "scitran/core:testing" + fi + + log "Cleaning pyc and previous coverage results ..." + # Run within container to avoid permission problems + docker run --rm \ + --name core-test-cleanup \ + --volume $(pwd):/var/scitran/code/api \ + scitran/core:testing \ + sh -c " + find . -type d -name __pycache__ -exec rm -rf {} \;; + find . -type f -name '*.pyc' -delete; + rm -rf .coverage htmlcov; + " + + trap clean_up EXIT + docker network create core-test + + # Launch core test service (includes mongo) + docker run -d \ + --name core-test-service \ + --network core-test \ + --volume $(pwd)/api:/var/scitran/code/api/api \ + --volume $(pwd)/tests:/var/scitran/code/api/tests \ + --env PRE_RUNAS_CMD='[ "$1" = uwsgi ] && mongod > /dev/null 2>&1 &' \ + --env SCITRAN_CORE_DRONE_SECRET=secret \ + --env SCITRAN_RUNTIME_COVERAGE=true \ + --env SCITRAN_CORE_ACCESS_LOG_ENABLED=true \ + scitran/core:testing \ + uwsgi --ini /var/scitran/config/uwsgi-config.ini --http [::]:9000 \ + --processes 1 --threads 1 --enable-threads \ + --http-keepalive --so-keepalive --add-header "Connection: Keep-Alive" \ + --logformat '[%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) request_id=%(request_id)' + + # Run core test cmd + local CORE_TEST_CMD + [ $RUN_SHELL ] && CORE_TEST_CMD=bash || \ + CORE_TEST_CMD="tests/bin/tests.sh -- $PYTEST_ARGS" + docker run -it \ + --name core-test-runner \ + --network core-test \ + --volume $(pwd)/api:/var/scitran/code/api/api \ + --volume $(pwd)/tests:/var/scitran/code/api/tests \ + --env SCITRAN_SITE_API_URL=http://core-test-service:9000/api \ + --env SCITRAN_CORE_DRONE_SECRET=secret \ + --env SCITRAN_PERSISTENT_DB_URI=mongodb://core-test-service:27017/scitran \ + --env SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://core-test-service:27017/logs \ + scitran/core:testing \ + $CORE_TEST_CMD +} + + +clean_up() { + local TEST_RESULT_CODE=$? + set +e + + log "INFO: Test return code = $TEST_RESULT_CODE" + if [ "${TEST_RESULT_CODE}" = "0" ]; then + log "INFO: Collecting coverage..." + + # Copy unit test coverage + docker cp core-test-runner:/var/scitran/code/api/.coverage .coverage.unit-tests + + # Save integration test coverage + docker wait $(docker stop core-test-service) + docker cp core-test-service:/var/scitran/code/api/.coverage.integration-tests . + + # Combine unit/integ coverage and report/grenerate html + docker run --rm \ + --name core-test-coverage \ + --volume $(pwd):/var/scitran/code/api \ + scitran/core:testing \ + sh -c ' + coverage combine; + coverage report --skip-covered --show-missing; + coverage html; + ' + else + log "INFO: Printing container logs..." + docker logs core-test-service + log "ERROR: Test return code = $TEST_RESULT_CODE. Container logs printed above." + fi + + # Spin down core service + docker rm --force --volumes core-test-runner + docker rm --force --volumes core-test-service + docker network rm core-test + exit $TEST_RESULT_CODE +} + + +log() { + printf "\n%s\n" "$@" >&2 +} + + +main "$@" diff --git a/tests/bin/run-tests-docker.sh b/tests/bin/run-tests-docker.sh deleted file mode 100755 index a755c4357..000000000 --- a/tests/bin/run-tests-docker.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -set -eu -unset CDPATH -cd "$( dirname "${BASH_SOURCE[0]}" )/../.." - - -function usage() { -cat >&2 < 0 ]]; do - case "$1" in - -B|--no-build) DOCKER_BUILD=false; ;; - -h|--help) usage; exit 0 ;; - --) TEST_ARGS="${@:2}"; break ;; - *) echo "Invalid argument: $1" >&2; usage; exit 1 ;; - esac - shift - done - - if ${DOCKER_BUILD}; then - echo "Building scitran-core:run-tests ..." - docker build -t scitran-core:run-tests . - fi - - trap clean_up EXIT - - docker network create scitran-core-test-network - - # Launch Mongo instance - docker run -d \ - --name scitran-core-test-mongo \ - --network scitran-core-test-network \ - mongo:${MONGO_VERSION} - - # Execute tests - docker run -it \ - --name scitran-core-test-uwsgi \ - --network scitran-core-test-network \ - -e SCITRAN_PERSISTENT_DB_URI=mongodb://scitran-core-test-mongo:27017/scitran \ - -e SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://scitran-core-test-mongo:27017/logs \ - -v $(pwd):/var/scitran/code/api \ - --entrypoint bash \ - scitran-core:run-tests \ - /var/scitran/code/api/tests/bin/run-tests-ubuntu.sh \ - $TEST_ARGS -} - - -function clean_up() { - local TEST_RESULT_CODE=$? - set +e - - # Copy coverage file to host for possible further reporting - docker cp scitran-core-test-uwsgi:/var/scitran/code/api/.coverage .coverage - docker cp scitran-core-test-uwsgi:/var/scitran/code/api/endpoints.json endpoints.json - - # Spin down dependencies - docker rm -f -v scitran-core-test-uwsgi - docker rm -f -v scitran-core-test-mongo - docker network rm scitran-core-test-network - exit $TEST_RESULT_CODE -} - - -main "$@" diff --git a/tests/bin/run-tests-ubuntu.sh b/tests/bin/run-tests-ubuntu.sh deleted file mode 100755 index 443fa304f..000000000 --- a/tests/bin/run-tests-ubuntu.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env bash -set -eu -unset CDPATH -cd "$( dirname "${BASH_SOURCE[0]}" )/../.." - - -function usage() { -cat >&2 < 0 ]]; do - case "$1" in - -l|--lint) RUN_ALL=false; RUN_LINT=true ;; - -u|--unit) RUN_ALL=false; RUN_UNIT=true ;; - -i|--integ) RUN_ALL=false; RUN_INTEG=true ;; - -h|--help) usage; exit 0 ;; - --) PYTEST_ARGS="${@:2}"; break ;; - *) echo "Invalid argument: $1" >&2; usage; exit 1 ;; - esac - shift - done - - if ${RUN_ALL}; then - # No filtering options used, run everything by default - RUN_LINT=true - RUN_UNIT=true - RUN_INTEG=true - elif ${RUN_LINT} && ${RUN_UNIT} && ${RUN_INTEG}; then - # All filtering options were used, the same as none - RUN_ALL=true - fi - - trap clean_up EXIT - - # Remove __pycache__ directories for issue with __file__ attribute due to - # running the tests on the host creating bytecode files hich have a - # mismatched __file__ attribute when loaded in docker container - rm -rf tests/unit_tests/python/__pycache__ - rm -rf tests/integration_tests/python/__pycache__ - - export PYTHONPATH="$(pwd)" - export SCITRAN_SITE_API_URL="http://localhost:8081/api" - export SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"} - export SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"} - export SCITRAN_PERSISTENT_DB_LOG_URI=${SCITRAN_PERSISTENT_DB_LOG_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/logs"} - export SCITRAN_PERSISTENT_PATH=`mktemp -d` - export SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data" - export SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$( openssl rand -base64 32 )} - - if ${RUN_LINT}; then - echo "Running pylint ..." - # TODO Enable Refactor and Convention reports - # TODO Move --disable into rc - pylint --reports=no --disable=C,R,W0312,W0141,W0110 api - - # echo "Running pep8 ..." - # pep8 --max-line-length=150 --ignore=E402 api - fi - - if ${RUN_UNIT}; then - echo "Running unit tests ..." - rm -f .coverage - py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS - fi - - if ${RUN_INTEG}; then - echo "Spinning up dependencies ..." - uwsgi --http "localhost:8081" --master --http-keepalive \ - --so-keepalive --add-header "Connection: Keep-Alive" \ - --processes 1 --threads 1 \ - --enable-threads \ - --wsgi-file bin/api.wsgi \ - --die-on-term \ - --logformat '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)" request_id=%(request_id)' \ - --env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \ - --env "SCITRAN_PERSISTENT_DB_LOG_URI=$SCITRAN_PERSISTENT_DB_LOG_URI" \ - --env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \ - --env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" \ - --env "SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET" \ - --env "SCITRAN_RUNTIME_COVERAGE=true" \ - --env "SCITRAN_COLLECT_ENDPOINTS=true" \ - --env "SCITRAN_CORE_ACCESS_LOG_ENABLED=true" & - export API_PID=$! - - echo "Connecting to API" - until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do - printf '.' - sleep 1 - done - - echo "Running integration tests ..." - py.test tests/integration_tests/python $PYTEST_ARGS - fi -} - - -function clean_up () { - local TEST_RESULT_CODE=$? - set +e - - echo - echo "Test return code = $TEST_RESULT_CODE" - - if [[ -n "${API_PID:-}" ]]; then - # Killing uwsgi - kill $API_PID - wait 2> /dev/null - fi - - if ${RUN_ALL} && [[ "${TEST_RESULT_CODE}" == "0" ]]; then - echo - echo "UNIT TEST COVERAGE:" - coverage report --skip-covered - echo - echo "OVERALL COVERAGE:" - coverage combine - coverage report --show-missing - coverage html - else - echo "Some tests were skipped or failed, skipping coverage report" - fi - - exit $TEST_RESULT_CODE -} - - -main "$@" diff --git a/tests/bin/setup-integration-tests-ubuntu.sh b/tests/bin/setup-integration-tests-ubuntu.sh deleted file mode 100755 index 7719063cc..000000000 --- a/tests/bin/setup-integration-tests-ubuntu.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eu -unset CDPATH -cd "$( dirname "${BASH_SOURCE[0]}" )/../.." - -sudo pip install -U -r "tests/integration_tests/requirements-integration-test.txt" - diff --git a/tests/bin/tests.sh b/tests/bin/tests.sh new file mode 100755 index 000000000..d6ed0ad9f --- /dev/null +++ b/tests/bin/tests.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env sh + +set -eu +unset CDPATH +cd "$( dirname "$0" )/../.." + + +USAGE=" +Usage: + $0 [-- PYTEST_ARGS...] + +Runs all tests (unit, integ and linting) if no options are provided. + +Assumes running in a scitran/core:testing container or that core and all +of its dependencies are installed the same way as in the Dockerfile. + +Options: + -h, --help Print this help and exit + -- PYTEST_ARGS Arguments passed to py.test + +Envvars (required for integration tests): + SCITRAN_SITE_API_URL URI to a running core instance (including /api) + SCITRAN_CORE_DRONE_SECRET API shared secret + SCITRAN_PERSISTENT_DB_URI Mongo URI to the scitran DB + SCITRAN_PERSISTENT_DB_LOG_URI Mongo URI to the scitran log DB + +" + + +main() { + export PYTHONDONTWRITEBYTECODE=1 + local PYTEST_ARGS= + + while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + log "$USAGE" + exit 0 + ;; + --) + shift + PYTEST_ARGS="$@" + break + ;; + *) + log "Invalid argument: $1" + log "$USAGE" >&2 + exit 1 + ;; + esac + shift + done + + log "Running unit tests ..." + py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS + + log "Running integration tests ..." + py.test tests/integration_tests/python $PYTEST_ARGS + + log "Running pylint ..." + # TODO Enable Refactor and Convention reports + # TODO Move --disable into rc + pylint --rcfile=tests/.pylintrc --jobs=4 --reports=no --disable=C,R,W0312,W0141,W0110 api + + # log "Running pep8 ..." + # pep8 --max-line-length=150 --ignore=E402 api +} + + +log() { + printf "\n%s\n" "$@" >&2 +} + + +main "$@" diff --git a/tests/integration_tests/.npmignore b/tests/integration_tests/.npmignore deleted file mode 100644 index 72e8ffc0d..000000000 --- a/tests/integration_tests/.npmignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/tests/integration_tests/package.json b/tests/integration_tests/package.json deleted file mode 100644 index ea666c398..000000000 --- a/tests/integration_tests/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "scitran-core-integration-tests", - "version": "1.0.0", - "description": "SciTran Core integation test node dependencies", - "dependencies": { - "abao":"git+https://github.com/flywheel-io/abao.git#better-jsonschema-ref", - "chai": "~3.5.0" - }, - "bin":{ - "abao":"node_modules/.bin/abao" - } -} diff --git a/tests/integration_tests/python/test_download.py b/tests/integration_tests/python/test_download.py index 78d1c9d6b..55a6846be 100644 --- a/tests/integration_tests/python/test_download.py +++ b/tests/integration_tests/python/test_download.py @@ -110,7 +110,7 @@ def test_download(data_builder, file_form, as_admin, api_db): # Try to perform the download from a different IP update_result = api_db.downloads.update_one( {'_id': ticket}, - {'$set': {'ip': '0.0.0.0'}}) + {'$set': {'ip': '255.255.255.255'}}) assert update_result.modified_count == 1 r = as_admin.get('/download', params={'ticket': ticket}) diff --git a/tests/integration_tests/requirements-integration-test.txt b/tests/requirements.txt similarity index 94% rename from tests/integration_tests/requirements-integration-test.txt rename to tests/requirements.txt index 53c9cec21..8fb00ca5f 100644 --- a/tests/integration_tests/requirements-integration-test.txt +++ b/tests/requirements.txt @@ -9,4 +9,5 @@ pytest-mock==1.6.0 pytest-watch==3.8.0 pytest==2.8.5 requests_mock==1.3.0 +six>=1.11.0 testfixtures==4.10.1