Skip to content

Commit

Permalink
Merge pull request #224 from CybercentreCanada/hotfix/uwsgi
Browse files Browse the repository at this point in the history
hotfix/uwsgi
  • Loading branch information
cccs-sgaron authored Jul 15, 2021
2 parents 5f5631d + 4ad83d6 commit a9eb963
Show file tree
Hide file tree
Showing 6 changed files with 270 additions and 265 deletions.
2 changes: 1 addition & 1 deletion docker/socketio/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -yy build-essential libldap2-dev libsasl2-
# Install assemblyline UI into local so it merges new and old packages
USER assemblyline
RUN touch /tmp/before-pip
RUN pip install --no-cache-dir --user assemblyline-ui==$version
RUN pip install --no-cache-dir --user assemblyline-ui[socketio]==$version

# Remove files that existed before the pip install so that our copy command below doesn't take a snapshot of
# files that already exist in the base image
Expand Down
11 changes: 10 additions & 1 deletion docker/ui/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,13 @@ COPY --chown=assemblyline:assemblyline --from=builder /var/lib/assemblyline/.loc

# Switch back to assemblyline and run the app
USER assemblyline
CMD ["gunicorn", "assemblyline_ui.patched:app", "--config=python:assemblyline_ui.gunicorn_config", "--worker-class", "gevent"]

ENV UWSGI_MAX_REQUESTS=${MAX_REQUESTS:-1000}
ENV UWSGI_MAX_REQUESTS_DELTA=${MAX_REQUESTS_JITTER:-100}
ENV UWSGI_WORKERS=${WORKERS:-2}
ENV UWSGI_THREADS=${THREADS:-25}
ENV UWSGI_PROTOCOL=${PROTOCOL:-http}
ENV UWSGI_SOCKET=0.0.0.0:${PORT:-5000}
ENV UWSGI_BUFFER_SIZE=${BUFFER_SIZE:-65535}

CMD ["uwsgi", "--master", "--disable-logging", "--enable-threads", "--die-on-term", "--module", "assemblyline_ui.app:app"]
323 changes: 160 additions & 163 deletions pipelines/azure-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,173 +6,170 @@ trigger:
pr: none

pool:
vmImage: 'ubuntu-18.04'
vmImage: "ubuntu-18.04"

resources:
containers:
- container: redis
image: redis
ports:
- 6379:6379
- container: elasticsearch
image: cccs/elasticsearch:7.8.0
env:
ES_JAVA_OPTS: "-Xms256m -Xmx512m"
DISCOVERY_TYPE: 'single-node'
ports:
- 9200:9200
- container: minio
image: cccs/minio
env:
MINIO_ACCESS_KEY: al_storage_key
MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
ports:
- 9000:9000


stages:
- stage: build
jobs:
- job: build_package
displayName: Build Package
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: {versionSpec: 3.8}
- script: |
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
echo "Building $VERSION On branch $BRANCH"
if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
echo $VERSION > assemblyline_ui/VERSION
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
python setup.py bdist_wheel
- publish: $(System.DefaultWorkingDirectory)/dist/
artifact: dist
- publish: $(System.DefaultWorkingDirectory)/pipelines/
artifact: pipelines
- publish: $(System.DefaultWorkingDirectory)/test/
artifact: test
- stage: test
jobs:
- job: run_test
strategy:
matrix:
python3_7:
python.version: '3.7'
Python3_8:
python.version: '3.8'
timeoutInMinutes: 10
services:
elasticsearch: elasticsearch
redis: redis
minio: minio
steps:
- checkout: none
- task: UsePythonVersion@0
displayName: Set python version
inputs:
versionSpec: '$(python.version)'
- download: current
artifact: dist
- download: current
artifact: pipelines
- download: current
artifact: test
- script: |
set -x # Echo commands before they are run
sudo apt-get update
sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev libldap2-dev libsasl2-dev wget
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$VERSION" == *stable* ]]; then
sudo -E env "PATH=$PATH" python -m pip install "assemblyline" "assemblyline-core"
else
sudo -E env "PATH=$PATH" python -m pip install --pre "assemblyline" "assemblyline-core"
fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-ui==${VERSION}"
sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -r $(Pipeline.Workspace)/test/requirements.txt
sudo mkdir -p /etc/assemblyline/
sudo mkdir -p /var/cache/assemblyline/
sudo mkdir -p /var/lib/assemblyline/
sudo chmod a+rw /var/cache/assemblyline/
sudo chmod a+rw /var/lib/assemblyline/
sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
displayName: Install package
- script: |
python -m assemblyline_ui.app &
python -m assemblyline_ui.socketsrv &
sudo docker run -d --name frontend --network host --restart on-failure cccs/assemblyline-ui-frontend &
sudo docker run -d --name nginx --network host --restart on-failure -e "FRONTEND_HOST=localhost" -e "UI_HOST=localhost" -e "SOCKET_HOST=localhost" -e "TEMPLATE=minimal" -e "FQDN=localhost" cccs/nginx-ssl-frontend
wget https://localhost --no-check-certificate --timeout=2 --retry-on-http-error=502 --waitretry=10 --retry-connrefused
pytest -rsx -vv
workingDirectory: $(Pipeline.Workspace)/test
displayName: Test
- stage: deploy
jobs:
- job: deploy
displayName: Deploy packages and containers
variables:
- group: deployment-information
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: {versionSpec: 3.8}
- download: current
artifact: dist
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to Test PyPI
- container: redis
image: redis
ports:
- 6379:6379
- container: elasticsearch
image: cccs/elasticsearch:7.8.0
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to PyPI
ES_JAVA_OPTS: "-Xms256m -Xmx512m"
DISCOVERY_TYPE: "single-node"
ports:
- 9200:9200
- container: minio
image: cccs/minio
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- task: Docker@2
displayName: Login to docker hub
inputs:
command: login
containerRegistry: dockerhub
- task: Docker@2
displayName: Login to github packages
inputs:
command: login
containerRegistry: github-packages-sa
- script: |
set -xv # Echo commands before they are run
export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
export VERSION=${TAG/stable}
export VERSION=${VERSION/beta/b}
export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
until sudo env "PATH=$PATH" python -m pip download "assemblyline-ui==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
MINIO_ACCESS_KEY: al_storage_key
MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
ports:
- 9000:9000

for IMAGE in "cccs/assemblyline-ui" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-ui"
do
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker/ui
docker push $IMAGE --all-tags
done
for IMAGE in "cccs/assemblyline-socketio" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-socketio"
do
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker/socketio
docker push $IMAGE --all-tags
done
displayName: Deploy to Docker Hub
stages:
- stage: build
jobs:
- job: build_package
displayName: Build Package
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: { versionSpec: 3.8 }
- script: |
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
echo "Building $VERSION On branch $BRANCH"
if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
echo $VERSION > assemblyline_ui/VERSION
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
python setup.py bdist_wheel
- publish: $(System.DefaultWorkingDirectory)/dist/
artifact: dist
- publish: $(System.DefaultWorkingDirectory)/pipelines/
artifact: pipelines
- publish: $(System.DefaultWorkingDirectory)/test/
artifact: test
- stage: test
jobs:
- job: run_test
strategy:
matrix:
python3_7:
python.version: "3.7"
Python3_8:
python.version: "3.8"
timeoutInMinutes: 10
services:
elasticsearch: elasticsearch
redis: redis
minio: minio
steps:
- checkout: none
- task: UsePythonVersion@0
displayName: Set python version
inputs:
versionSpec: "$(python.version)"
- download: current
artifact: dist
- download: current
artifact: pipelines
- download: current
artifact: test
- script: |
set -x # Echo commands before they are run
sudo apt-get update
sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev libldap2-dev libsasl2-dev wget
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$VERSION" == *stable* ]]; then
sudo -E env "PATH=$PATH" python -m pip install "assemblyline" "assemblyline-core"
else
sudo -E env "PATH=$PATH" python -m pip install --pre "assemblyline" "assemblyline-core"
fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-ui[test,socketio]==${VERSION}"
sudo mkdir -p /etc/assemblyline/
sudo mkdir -p /var/cache/assemblyline/
sudo mkdir -p /var/lib/assemblyline/
sudo chmod a+rw /var/cache/assemblyline/
sudo chmod a+rw /var/lib/assemblyline/
sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
displayName: Install package
- script: |
python -m assemblyline_ui.app &
python -m assemblyline_ui.socketsrv &
sudo docker run -d --name frontend --network host --restart on-failure cccs/assemblyline-ui-frontend &
sudo docker run -d --name nginx --network host --restart on-failure -e "FRONTEND_HOST=localhost" -e "UI_HOST=localhost" -e "SOCKET_HOST=localhost" -e "TEMPLATE=minimal" -e "FQDN=localhost" cccs/nginx-ssl-frontend
wget https://localhost --no-check-certificate --timeout=2 --retry-on-http-error=502 --waitretry=10 --retry-connrefused
pytest -rsx -vv
workingDirectory: $(Pipeline.Workspace)/test
displayName: Test
- stage: deploy
jobs:
- job: deploy
displayName: Deploy packages and containers
variables:
- group: deployment-information
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: { versionSpec: 3.8 }
- download: current
artifact: dist
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to Test PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- task: Docker@2
displayName: Login to docker hub
inputs:
command: login
containerRegistry: dockerhub
- task: Docker@2
displayName: Login to github packages
inputs:
command: login
containerRegistry: github-packages-sa
- script: |
set -xv # Echo commands before they are run
export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
export VERSION=${TAG/stable}
export VERSION=${VERSION/beta/b}
export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
until sudo env "PATH=$PATH" python -m pip download "assemblyline-ui==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
for IMAGE in "cccs/assemblyline-ui" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-ui"
do
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker/ui
docker push $IMAGE --all-tags
done
for IMAGE in "cccs/assemblyline-socketio" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-socketio"
do
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker/socketio
docker push $IMAGE --all-tags
done
displayName: Deploy to Docker Hub
Loading

0 comments on commit a9eb963

Please sign in to comment.