Kill rabbitmq v3 (#1891)

Kill rabbitmq v3
oaeproject · Nov 11, 2019 · f54b9af · f54b9af
2 parents 3ef169d + 0c47252
commit f54b9af
Show file tree

Hide file tree

Showing 84 changed files with 4,546 additions and 4,710 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,79 +1,137 @@
-version: 2
-jobs:
-  test:
+aliases:
+  - &defaults
     docker:
-      - image: "alpine:3.8"
+      - image: "alpine:3.9"
+    parallelism: 4
     environment:
       TMP: /root/tmp
+      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+      REDIS_HOST: oae-redis
+      REDIS_PORT: 6379
     working_directory: ~/Hilary
+  - &install_system_dependencies
+    run:
+      name: Install system dependencies
+      command: |
+        apk add --update --no-cache curl git openssh-client docker py-pip python-dev libffi-dev openssl-dev gcc libc-dev make
+  - &update_config
+    run:
+      name: Adjusting Hilary configuration for tests to run
+      command: |
+        printf "\nconfig.ui.path = './3akai-ux';" >> config.js
+        printf "\nconfig.cassandra.hosts = ['oae-cassandra'];" >> config.js
+        printf "\nconfig.cassandra.timeout = 9000;" >> config.js
+        printf "\nconfig.redis.host = 'oae-redis';" >> config.js
+        printf "\nconfig.search.hosts[0].host = 'oae-elasticsearch';" >> config.js
+        printf "\nconfig.mq.host = 'oae-redis';" >> config.js
+        printf "\nconfig.etherpad.hosts[0].host = 'oae-etherpad';" >> config.js
+        printf "\nconfig.ethercalc[0].host = 'oae-ethercalc';" >> config.js
+        printf "\nconfig.previews.enabled = true;" >> config.js
+        printf "\nconfig.email.debug = false;" >> config.js
+        printf "\nconfig.email.transport = 'sendmail';" >> config.js
+        printf "\nconfig.previews.office.binary = '/usr/bin/soffice';" >> config.js
+        printf "\nconfig.previews.screenShotting.binary = '/usr/bin/chromium-browser';" >> config.js
+        printf "\nconfig.previews.screenShotting.sandbox = '--no-sandbox';" >> config.js
+  - &copy_artifacts
+    run:
+      name: Copy logs to host
+      command: |
+        docker cp oae-hilary:/usr/src/Hilary/tests.log reports/logs
+  - &store_results
+    store_test_results:
+      path: reports
+  - &upload_logs
+    store_artifacts:
+      path: reports/logs/tests.log
+  - &run_sequential_tests
+    run:
+      name: Run tests sequentially and upload coverage
+      command: |
+        docker-compose run --rm oae-hilary "yarn run test-with-coverage"
+  - &run_parallel_tests
+    run:
+      name: Run tests in parallel
+      command: |
+        docker-compose run --rm oae-hilary "yarn run test-module $(circleci tests glob 'packages/oae-*/tests' | circleci tests split | tr '\n' ' ')"
+  - &run_linting
+    run:
+      name: Run linting test
+      command: |
+        docker-compose run --rm oae-hilary "yarn run lint-module $(circleci tests glob 'packages/**/*.js' | circleci tests split | tr '\n' ' ')"
+  - &install_dependencies
+    run:
+      name: Install Hilary dependencies
+      command: |
+        docker-compose run --rm oae-hilary 'cd 3akai-ux && npx lerna bootstrap'
+        docker-compose run --rm oae-hilary 'npx lerna bootstrap'
+  - &copy_code
+    run:
+      name: Copy code
+      command: |
+        addgroup -g 1000 node
+        adduser -u 1000 -G node -s /bin/sh -D node
+        chown -R node:node .
+        docker cp /root/Hilary oae-hilary:/usr/src
+        mkdir -p reports/logs
+  - &create_containers
+    run:
+      name: Create the containers
+      command: docker-compose up --no-start --build oae-cassandra oae-redis oae-elasticsearch oae-hilary oae-ethercalc
+  - &launch_containers
+    run:
+      name: Start the containers
+      command: |
+        docker-compose up -d oae-cassandra oae-redis oae-elasticsearch oae-etherpad oae-ethercalc
+  - &install_docker_compose
+    run:
+      name: Install docker-compose
+      command: |
+        pip install docker-compose~=1.23.2
+
+version: 2.1
+jobs:
+  test:
+    <<: *defaults
     steps:
       - setup_remote_docker
-      - run:
-          name: Install dependencies
-          command: |
-            apk add --update --no-cache curl git openssh-client
+      - *install_system_dependencies
       - checkout
-      - run:
-          name: Checkout submodules
-          command: git submodule update --init --recursive
-      - run:
-          name: Creating folders for file storage and tmp files
-          command: |
-            mkdir -p ../files
-            mkdir -p ../tmp
-      - run:
-          name: Setting up volumes in docker-compose
-          command: |
-            sed -i -e 's/- \/src\/Hilary/- \/root\/Hilary/g' docker-compose.yml
-            sed -i -e 's/- \/src\/files/- \/root\/files/g' docker-compose.yml
-            sed -i -e 's/\/src\/tmp/\/root\/tmp/g' docker-compose.yml
-      - run:
-          name: Adjusting Hilary configuration for tests to run
-          command: |
-            printf "\nconfig.ui.path = './3akai-ux';" >> config.js
-            printf "\nconfig.cassandra.hosts = ['oae-cassandra'];" >> config.js
-            printf "\nconfig.cassandra.timeout = 9000;" >> config.js
-            printf "\nconfig.redis.host = 'oae-redis';" >> config.js
-            printf "\nconfig.search.hosts[0].host = 'oae-elasticsearch';" >> config.js
-            printf "\nconfig.mq.connection.host = ['oae-rabbitmq'];" >> config.js
-            printf "\nconfig.etherpad.hosts[0].host = 'oae-etherpad';" >> config.js
-            printf "\nconfig.ethercalc[0].host = 'oae-ethercalc';" >> config.js
-            printf "\nconfig.previews.enabled = true;" >> config.js
-            printf "\nconfig.email.debug = false;" >> config.js
-            printf "\nconfig.email.transport = 'sendmail';" >> config.js
-            printf "\nconfig.previews.office.binary = '/usr/bin/soffice';" >> config.js
-            printf "\nconfig.previews.screenShotting.binary = '/usr/bin/chromium-browser';" >> config.js
-      - run:
-          name: Install docker and docker-compose
-          command: |
-            apk add --update --no-cache docker py-pip python-dev libffi-dev openssl-dev gcc libc-dev make
-            pip install docker-compose~=1.23.2
-      - run:
-          name: Create the containers
-          command: docker-compose up --no-start --build oae-cassandra oae-redis oae-rabbitmq oae-elasticsearch oae-hilary oae-ethercalc
-      - run:
-          name: Start the containers we need
-          command: |
-            docker-compose up -d oae-cassandra oae-redis oae-rabbitmq oae-elasticsearch
-            sleep 25s
-            docker-compose up -d oae-etherpad oae-ethercalc
-      - run:
-          name: Install Hilary dependencies
-          command: |
-            addgroup -g 1000 node
-            adduser -u 1000 -G node -s /bin/sh -D node
-            chown -R node:node .
-            docker cp /root/Hilary oae-hilary:/usr/src
-            rm yarn.lock
-            docker-compose run --rm oae-hilary 'cd 3akai-ux && npx lerna bootstrap'
-            docker-compose run --rm oae-hilary 'npx lerna bootstrap'
-      - run:
-          name: Run tests
-          command: |
-            docker-compose run --rm oae-hilary "yarn run test-with-coverage"
+      - run: git submodule sync
+      - run: git submodule update --init
+      - *update_config
+      - *install_docker_compose
+      - *create_containers
+      - *launch_containers
+      - *copy_code
+      - *install_dependencies
+      - *run_linting
+      - *run_parallel_tests # doesn't include coverage
+      - *copy_artifacts
+      - *store_results
+      - *upload_logs
+  coverage:
+    <<: *defaults
+    steps:
+      - setup_remote_docker
+      - *install_system_dependencies
+      - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
+      - *update_config
+      - *install_docker_compose
+      - *create_containers
+      - *launch_containers
+      - *copy_code
+      - *install_dependencies
+      - *run_linting
+      - *run_sequential_tests # includes coverage
+      - *copy_artifacts
+      - *store_results
+      - *upload_logs
+
 workflows:
   version: 2
-  test-all:
+  lint-and-test:
     jobs:
       - test
   scheduled:
@@ -84,4 +142,4 @@ workflows:
             branches:
               only: master
     jobs:
-      - test
+      - coverage
diff --git a/Dockerfile b/Dockerfile
@@ -32,22 +32,23 @@ LABEL Name=OAE-Hilary
 LABEL Author=ApereoFoundation
 LABEL [email protected]
 
+# Install system dependencies
 RUN apk --update --no-cache add \
-    git \
-		python \
-    ghostscript \
-    graphicsmagick
+      git \
+      python \
+      ghostscript \
+      graphicsmagick
 
-# Installs the 3.8 Chromium package.
+# Installs the 3.9 Chromium package.
 RUN apk update && apk upgrade && \
-    echo @3.8 http://nl.alpinelinux.org/alpine/v3.8/community >> /etc/apk/repositories && \
-    echo @3.8 http://nl.alpinelinux.org/alpine/v3.8/main >> /etc/apk/repositories && \
-    apk add --no-cache \
-      chromium@3.8 \
-      nss@3.8 \
-      freetype@3.8 \
-      harfbuzz@3.8 \
-      ttf-freefont@3.8
+      echo @3.9 http://nl.alpinelinux.org/alpine/v3.9/community >> /etc/apk/repositories && \
+      echo @3.9 http://nl.alpinelinux.org/alpine/v3.9/main >> /etc/apk/repositories && \
+      apk add --no-cache \
+      chromium@3.9 \
+      nss@3.9 \
+      freetype@3.9 \
+      harfbuzz@3.9 \
+      ttf-freefont@3.9
 
 # Tell Puppeteer to skip installing Chrome. We'll be using the installed package.
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
@@ -59,26 +60,30 @@ RUN apk add --no-cache libreoffice openjdk8-jre
 RUN apk --update --no-cache add build-base libgit2-dev
 RUN ln -s /usr/lib/libcurl.so.4 /usr/lib/libcurl-gnutls.so.4
 
-# Set the base directory
-ENV HILARY_DIR /usr/src/Hilary
-RUN mkdir -p ${HILARY_DIR} \
-    && chown -R node:node ${HILARY_DIR} \
-    && chmod -R 755 ${HILARY_DIR}
+# Set the Hilary directory
+ENV CODE_DIR /usr/src
+ENV HILARY_DIR ${CODE_DIR}/Hilary
+RUN mkdir -p ${HILARY_DIR}
 WORKDIR ${HILARY_DIR}
 
+# Set the right permissions for Hilary
+RUN chown -R node:node ${CODE_DIR} \
+      && chmod -R 755 ${CODE_DIR}
+
 # Create the temp directory for Hilary
 ENV TMP_DIR /tmp
-RUN mkdir -p ${TMP_DIR} \
-    && chown -R node:node ${TMP_DIR} \
-    && chmod -R 755 ${TMP_DIR} \
-    && export TMP=${TMP_DIR}
+RUN mkdir -p ${TMP_DIR}
+RUN chown -R node:node ${TMP_DIR} \
+      && chmod -R 755 ${TMP_DIR} \
+      && export TMP=${TMP_DIR}
 
 # Expose ports for node server
 EXPOSE 2000
 EXPOSE 2001
 
+# Change user from now on
 USER node
 
 # Run the app - you may override CMD via docker run command line instruction
 ENTRYPOINT ["/bin/sh", "-c"]
-CMD ["nodemon -L app.js | bunyan"]
+CMD ["node app.js | bunyan"]
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@ Hilary is the back-end for the [Open Academic Environment](http://www.oaeproject
 ## Build status
 
 <!-- current project status -->
+
 [![CircleCI](https://circleci.com/gh/oaeproject/Hilary/tree/master.svg?style=shield)](https://circleci.com/gh/oaeproject/Hilary/tree/master)
 [![Code Climate](https://codeclimate.com/github/oaeproject/Hilary/badges/gpa.svg)](https://codeclimate.com/github/oaeproject/Hilary)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/8a6104cadb6b442596c418534cf97db3)](https://www.codacy.com/app/brecke/Hilary?utm_source=github.com&utm_medium=referral&utm_content=oaeproject/Hilary&utm_campaign=Badge_Grade)
@@ -17,8 +18,10 @@ Hilary is the back-end for the [Open Academic Environment](http://www.oaeproject
 [![Known Vulnerabilities](https://snyk.io/test/github/oaeproject/Hilary/badge.svg)](https://snyk.io/test/github/oaeproject/Hilary)
 
 <!-- standards used in project -->
+
 [![datree-badge](https://s3.amazonaws.com/catalog.static.datree.io/datree-badge-28px.svg)](https://datree.io/?src=badge)
 
+[![lerna](https://img.shields.io/badge/maintained%20with-lerna-cc00ff.svg)](https://lerna.js.org/)
 ![code style](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)
 [![XO code style](https://img.shields.io/badge/code_style-XO-5ed9c7.svg)](https://github.com/xojs/xo)
 [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://conventionalcommits.org)

diff --git a/config.js b/config.js
@@ -261,11 +261,11 @@ config.search = {
  * @param  {Boolean}    [purgeQueuesOnStartup]  If `true`, the application will **delete** all messages in a queue when a worker is first bound. This setting only takes effect if the NODE_ENV environment variable is not set to `production` to indicate a production environment. Default: `false`
  */
 config.mq = {
-  connection: {
-    host: [LOCALHOST],
-    port: 5672
-  },
-  purgeQueuesOnStartup: false
+  host: LOCALHOST,
+  port: 6379,
+  pass: '',
+  dbIndex: 0,
+  purgeQueuesOnStartup: true
 };
 
 /**

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -42,7 +42,6 @@ services:
     #   - "tenant1.oae.com:172.20.0.9"
     # - "any.other.host.oae.com:172.20.0.9"
     image: hilary:latest
-    restart: always
     networks:
       - my_network
     ports:
@@ -64,7 +63,6 @@ services:
   oae-redis:
     container_name: oae-redis
     image: redis:3.2.8-alpine
-    restart: always
     ports:
       - 6379:6379
     tty: false
@@ -73,27 +71,16 @@ services:
   oae-elasticsearch:
     container_name: oae-elasticsearch
     image: oaeproject/oae-elasticsearch-docker
-    restart: always
     networks:
       - my_network
     ports:
       - 9200:9200
     tty: false
     volumes:
       - ../data/elasticsearch:/usr/share/elasticsearch/data
-  oae-rabbitmq:
-    container_name: oae-rabbitmq
-    image: rabbitmq:3.6.6-management-alpine
-    restart: always
-    ports:
-      - 5672:5672
-    networks:
-      - my_network
-    tty: false
   oae-nginx:
     container_name: oae-nginx
     image: nginx:stable-alpine
-    restart: always
     networks:
       my_network:
         ipv4_address: 172.20.0.9 # this is needed because of multi-tenancy
@@ -114,7 +101,6 @@ services:
   oae-cassandra:
     container_name: oae-cassandra
     image: cassandra:2.1.21
-    restart: always
     networks:
       - my_network
     ports:
@@ -125,17 +111,15 @@ services:
       - ../data/cassandra:/var/lib/cassandra
   oae-etherpad:
     container_name: oae-etherpad
-    image: oaeproject/oae-etherpad-docker
-    restart: always
+    image: oaeproject/oae-etherpad-docker:latest
     networks:
       - my_network
     ports:
       - 9001:9001
     tty: false
   oae-ethercalc:
     container_name: oae-ethercalc
-    image: oaeproject/oae-ethercalc-docker
-    restart: always
+    image: oaeproject/oae-ethercalc-docker:latest
     networks:
       - my_network
     ports: