Update testing (#1010)

* update pipelines to expand on the tests and test upgrade from 3.6 to 3.7
phpList · Jan 21, 2024 · bcdd6e3 · bcdd6e3
1 parent 54becd4
commit bcdd6e3
Show file tree

Hide file tree

Showing 35 changed files with 100,801 additions and 426 deletions.
diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml
@@ -40,7 +40,7 @@ jobs:
       - name: Setup Packages
         run: |
           cd $GITHUB_WORKSPACE
-          sudo apt update && sudo apt install -y rsync
+          sudo apt update && sudo apt install -y rsync postfix
 
       - name: Cache Composer dependencies
         uses: actions/cache@v2
@@ -88,6 +88,7 @@ jobs:
           cp -fv tests/ci/config.php public_html/lists/config/config.php
           mkdir -p output/screenshots
           mkdir -p build/mails
+          smtp-sink -c -d "output/mails/%Y%m%d%H/%M."  2500 1024 &
           ./bin/start-selenium > output/selenium.log 2>&1 &
           sleep 5
           sudo php -S 0.0.0.0:80 -t public_html > /dev/null 2>&1 &
@@ -117,7 +118,7 @@ jobs:
         run: |
           cd $GITHUB_WORKSPACE/tests
           ../vendor/bin/behat -n -fprogress -p chrome --strict --stop-on-failure --tags=@initialise
-          ../vendor/bin/behat -n -fprogress -p chrome --strict --stop-on-failure --tags="~@initialise && ~@wip"
+          ../vendor/bin/behat -n -fprogress -p chrome --strict --stop-on-failure --tags="~@initialise && ~@wip && ~@behattest"
 
       - name: Upload the screenshots
         if: always()

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,7 +1,7 @@
 
 name: CI
 
-on: [pull_request]
+on: [pull_request, push]
 
 jobs:
   test:
@@ -35,7 +35,7 @@ jobs:
         run: |
           cd $GITHUB_WORKSPACE
           sudo apt update
-          sudo apt install rsync
+          sudo apt -y install rsync postfix
 
       # - name: Cache Composer dependencies
       #   uses: actions/cache@v2
@@ -69,8 +69,7 @@ jobs:
       - name: Create Database
         run: |
           sudo systemctl start mysql.service
-          sudo mysql -proot  -e 'drop database if exists phplistdb'
-          sudo mysqladmin -proot create phplistdb
+          sudo mysql -proot  -e 'drop database if exists phplistdb; create database phplistdb;'
           sudo mysql -proot -e 'create user phplist@"%" identified by "phplist"; grant all on phplistdb.* to phplist@"%"'
 
       - name: Set bootlist theme
@@ -88,7 +87,8 @@ jobs:
           cp -fv tests/ci/config.php public_html/lists/config/config.php
           mkdir -p output/screenshots
           touch output/screenshots/README.md
-          mkdir -p build/mails
+          mkdir -p output/mails
+          smtp-sink -c -d "output/mails/%Y%m%d%H/%M."  2500 1024 &
           ./bin/start-selenium > output/selenium.log 2>&1 &
           sleep 5
           sudo php -S 0.0.0.0:80 -t public_html > /dev/null 2>&1 &
@@ -102,7 +102,7 @@ jobs:
         run: |
           cd $GITHUB_WORKSPACE/tests
           ../vendor/bin/behat -p chrome -f progress --stop-on-failure --tags=@initialise
-          ../vendor/bin/behat -p chrome -f progress --tags="~@initialise && ~@wip"
+          ../vendor/bin/behat -p chrome -f progress --tags="~@initialise && ~@wip && ~@behattest"
 
       - name: Run BDD Tests CLI
         run: |
@@ -115,6 +115,14 @@ jobs:
           cd $GITHUB_WORKSPACE/tests
           ../vendor/bin/behat -p chrome --tags="~@initialise && ~@wip"
 
+      - name: copy screenshots
+        if: always()
+        run: |
+          cd $GITHUB_WORKSPACE
+          images=$(ls -l /tmp/*.png 2>/dev/null | wc -l)
+          [[ $images -gt 0 ]] && cp -v /tmp/*.png output/screenshots
+          exit 0;
+
       - name: Upload the screenshots
         if: always()
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml
@@ -27,7 +27,7 @@ jobs:
         run: |
           cd $GITHUB_WORKSPACE
           sudo apt update
-          sudo apt install rsync
+          sudo apt install -y rsync postfix
 
       # - name: Cache Composer dependencies
       #   uses: actions/cache@v2
@@ -82,6 +82,7 @@ jobs:
           mkdir -p output/screenshots
           touch output/screenshots/README.md
           mkdir -p build/mails
+          smtp-sink -c -d "output/mails/%Y%m%d%H/%M."  2500 1024 &
           ./bin/start-selenium > output/selenium.log 2>&1 &
           sleep 5
           sudo php -S 0.0.0.0:80 -t public_html > /dev/null 2>&1 &

diff --git a/.gitignore b/.gitignore
@@ -17,4 +17,8 @@ public_html/lists/admin/plugins/
 public_html/lists/admin/ui/dressprow
 public_html/lists/admin/ui/dressprow-ph
 public_html/lists/admin/ui/phplist-ui-*
-
+features/*
+tests/output/
+*.env
+.local
+public_html/images/
diff --git a/Dockerfile.release b/Dockerfile.release
@@ -3,9 +3,7 @@ FROM debian:buster-slim
 
 LABEL maintainer="[email protected]" 
 
-RUN apt-get update && apt-get upgrade -y
-
-RUN apt-get install -y apt-utils \
+RUN apt-get update && apt-get upgrade -y && apt-get install -y apt-utils \
     apache2 php-mysql \
     libapache2-mod-php php-curl php-gd \
     cron php-imap php-xml php-zip php-mbstring

diff --git a/TESTING.md b/TESTING.md
@@ -4,104 +4,45 @@
 
 [Behat](http://behat.org/en/latest/) is used to execute User Acceptance Tests written in [Gherkin](https://github.com/cucumber/cucumber/wiki/Gherkin). These tests are stored in `tests/features/`.
 
+### Developing tests with Docker
 
+To develop acceptance tests locally, install Docker. 
+You also need the "make" command in Linux (eg apt install make)
 
-### Developing tests with Vagrant
-
-To develop acceptance tests locally, install Vagrant. This allows you to run tests against 
-different versions of PHP.
-Find the line "PHPVERSION=7.4" and change it to the version of PHP you want to use.
 Put @wip in the feature you are working on, so that it doesn't interfere with Github CI.
 
-Then run
-```sh
-vagant up --provision
-```
-
-Once it has run once, you can fire off further runs with
-
-```sh
-vagant ssh
-cd /vagrant/tests
-sudo su
-make test
-```
-
-You can also use the following commands:
-
-- make verbosetest - show progress during testing, useful to identify issues in your tests
-- make test-wip - only run the features marked "@wip"
-- make testall - run all tests
-
-
-
-### Execute Acceptance tests
-
-#### Install composer globally
-
-```sh
-$ curl -sS https://getcomposer.org/installer | php
-$ mv composer.phar /usr/local/bin/composer
-```
-
-You can read the official documentation [here](https://getcomposer.org/doc/00-intro.md#installation-linux-unix-osx)
+### Running the tests locally
 
-#### Install Behat
+- cd tests
+- cp .env.dist .env
+- make test
 
-To install behat and other dependencies run the following command on the project root: 
+This will build a Behat container and spin up some docker containers to run the tests against the phpList container.
 
-```sh
-$ composer install
-```
+It will then follow the progress in the Behat container. You can get out of there, by pressing crtl-C
 
-run the following command:
+It will run the tests from your local repository, so if you make changes, you can re-run them by running "make" again,
+or you can leave the containers running and enter the behat one with
 
-```sh
-$ cp default.behat.yml behat.yml
-```
+docker exec -it behat bash
 
-Edit _behat.yml_ file and customize your `base_url`. The `base_url` determines which phpList server you wish to test. This could be a local development copy running on your machine, or a remotely hosted server. 
+and then run the tests with, eg 
 
-#### Database setup
+vendor/bin/behat --tags="@wip"
 
-Create a database, but keep it empty
+### viewing phpList itself
 
-```sh
-$ mysqladmin -uroot -p create phplisttestdb
-$ mysqladmin -uroot -p -e "grant all on phplisttestdb.* to phplist@localhost identified by 'testpassword'"
-```
+You can load phpList with http://localhost/lists/admin/
 
-Edit your phplist `config.php` [file](https://www.phplist.org/manual/ch028_installation.xhtml#edit-the-phplist-config-php-file) to use these details
-
-#### Run The tests
-
-### First time
-
-Some tests can only be run once before they change the system state upon which they depend. These tests should only be run once, or only in automated testing environments like Github actions. Execute these and all other tests together by running the following command from within your phpList 3 code root directory:
-
-```sh
-$ vendor/bin/behat
-```
-
-### Every other time
-
-Usually you will want to avoid running the first time tests and run all the others instead. To do so execute the following command from within your phpList 3 code root directory:
-
-```sh
-$ vendor/bin/behat --tags '~@initialise'
-```
-
-This will run all tests except for those with the `initialise` tag.
+Or if you want a different port, change that in the .env file
 
 ### Errors
 
-When you run it locally or using Vagrant, and a test fails, it will drop a screenshot of the page where it failed in
+When you run it locally, and a test fails, it will drop a screenshot of the page where it failed in
 tests/output/screenshots
 
 That way you can find out why it failed and update your test.
 
-
-
 #### TODO
 
 Add writing tests documentation.

diff --git a/scripts/testimport-admin.txt b/scripts/testimport-admin.txt
@@ -0,0 +1,101 @@
+email	loginname	password	First Name	Last Name
+[email protected]	Test Admin 20		Test admin First Name 20	Test admin Last name20
+[email protected]	Test Admin 21		Test admin First Name 21	Test admin Last name21
+[email protected]	Test Admin 22		Test admin First Name 22	Test admin Last name22
+[email protected]	Test Admin 23		Test admin First Name 23	Test admin Last name23
+[email protected]	Test Admin 24		Test admin First Name 24	Test admin Last name24
+[email protected]	Test Admin 25		Test admin First Name 25	Test admin Last name25
+[email protected]	Test Admin 26		Test admin First Name 26	Test admin Last name26
+[email protected]	Test Admin 27		Test admin First Name 27	Test admin Last name27
+[email protected]	Test Admin 28		Test admin First Name 28	Test admin Last name28
+[email protected]	Test Admin 29		Test admin First Name 29	Test admin Last name29
+[email protected]	Test Admin 30		Test admin First Name 30	Test admin Last name30
+[email protected]	Test Admin 31		Test admin First Name 31	Test admin Last name31
+[email protected]	Test Admin 32		Test admin First Name 32	Test admin Last name32
+[email protected]	Test Admin 33		Test admin First Name 33	Test admin Last name33
+[email protected]	Test Admin 34		Test admin First Name 34	Test admin Last name34
+[email protected]	Test Admin 35		Test admin First Name 35	Test admin Last name35
+[email protected]	Test Admin 36		Test admin First Name 36	Test admin Last name36
+[email protected]	Test Admin 37		Test admin First Name 37	Test admin Last name37
+[email protected]	Test Admin 38		Test admin First Name 38	Test admin Last name38
+[email protected]	Test Admin 39		Test admin First Name 39	Test admin Last name39
+[email protected]	Test Admin 40		Test admin First Name 40	Test admin Last name40
+[email protected]	Test Admin 41		Test admin First Name 41	Test admin Last name41
+[email protected]	Test Admin 42		Test admin First Name 42	Test admin Last name42
+[email protected]	Test Admin 43		Test admin First Name 43	Test admin Last name43
+[email protected]	Test Admin 44		Test admin First Name 44	Test admin Last name44
+[email protected]	Test Admin 45		Test admin First Name 45	Test admin Last name45
+[email protected]	Test Admin 46		Test admin First Name 46	Test admin Last name46
+[email protected]	Test Admin 47		Test admin First Name 47	Test admin Last name47
+[email protected]	Test Admin 48		Test admin First Name 48	Test admin Last name48
+[email protected]	Test Admin 49		Test admin First Name 49	Test admin Last name49
+[email protected]	Test Admin 50		Test admin First Name 50	Test admin Last name50
+[email protected]	Test Admin 51		Test admin First Name 51	Test admin Last name51
+[email protected]	Test Admin 52		Test admin First Name 52	Test admin Last name52
+[email protected]	Test Admin 53		Test admin First Name 53	Test admin Last name53
+[email protected]	Test Admin 54		Test admin First Name 54	Test admin Last name54
+[email protected]	Test Admin 55		Test admin First Name 55	Test admin Last name55
+[email protected]	Test Admin 56		Test admin First Name 56	Test admin Last name56
+[email protected]	Test Admin 57		Test admin First Name 57	Test admin Last name57
+[email protected]	Test Admin 58		Test admin First Name 58	Test admin Last name58
+[email protected]	Test Admin 59		Test admin First Name 59	Test admin Last name59
+[email protected]	Test Admin 60		Test admin First Name 60	Test admin Last name60
+[email protected]	Test Admin 61		Test admin First Name 61	Test admin Last name61
+[email protected]	Test Admin 62		Test admin First Name 62	Test admin Last name62
+[email protected]	Test Admin 63		Test admin First Name 63	Test admin Last name63
+[email protected]	Test Admin 64		Test admin First Name 64	Test admin Last name64
+[email protected]	Test Admin 65		Test admin First Name 65	Test admin Last name65
+[email protected]	Test Admin 66		Test admin First Name 66	Test admin Last name66
+[email protected]	Test Admin 67		Test admin First Name 67	Test admin Last name67
+[email protected]	Test Admin 68		Test admin First Name 68	Test admin Last name68
+[email protected]	Test Admin 69		Test admin First Name 69	Test admin Last name69
+[email protected]	Test Admin 70		Test admin First Name 70	Test admin Last name70
+[email protected]	Test Admin 71		Test admin First Name 71	Test admin Last name71
+[email protected]	Test Admin 72		Test admin First Name 72	Test admin Last name72
+[email protected]	Test Admin 73		Test admin First Name 73	Test admin Last name73
+[email protected]	Test Admin 74		Test admin First Name 74	Test admin Last name74
+[email protected]	Test Admin 75		Test admin First Name 75	Test admin Last name75
+[email protected]	Test Admin 76		Test admin First Name 76	Test admin Last name76
+[email protected]	Test Admin 77		Test admin First Name 77	Test admin Last name77
+[email protected]	Test Admin 78		Test admin First Name 78	Test admin Last name78
+[email protected]	Test Admin 79		Test admin First Name 79	Test admin Last name79
+[email protected]	Test Admin 80		Test admin First Name 80	Test admin Last name80
+[email protected]	Test Admin 81		Test admin First Name 81	Test admin Last name81
+[email protected]	Test Admin 82		Test admin First Name 82	Test admin Last name82
+[email protected]	Test Admin 83		Test admin First Name 83	Test admin Last name83
+[email protected]	Test Admin 84		Test admin First Name 84	Test admin Last name84
+[email protected]	Test Admin 85		Test admin First Name 85	Test admin Last name85
+[email protected]	Test Admin 86		Test admin First Name 86	Test admin Last name86
+[email protected]	Test Admin 87		Test admin First Name 87	Test admin Last name87
+[email protected]	Test Admin 88		Test admin First Name 88	Test admin Last name88
+[email protected]	Test Admin 89		Test admin First Name 89	Test admin Last name89
+[email protected]	Test Admin 90		Test admin First Name 90	Test admin Last name90
+[email protected]	Test Admin 91		Test admin First Name 91	Test admin Last name91
+[email protected]	Test Admin 92		Test admin First Name 92	Test admin Last name92
+[email protected]	Test Admin 93		Test admin First Name 93	Test admin Last name93
+[email protected]	Test Admin 94		Test admin First Name 94	Test admin Last name94
+[email protected]	Test Admin 95		Test admin First Name 95	Test admin Last name95
+[email protected]	Test Admin 96		Test admin First Name 96	Test admin Last name96
+[email protected]	Test Admin 97		Test admin First Name 97	Test admin Last name97
+[email protected]	Test Admin 98		Test admin First Name 98	Test admin Last name98
+[email protected]	Test Admin 99		Test admin First Name 99	Test admin Last name99
+[email protected]	Test Admin 100		Test admin First Name 100	Test admin Last name100
+[email protected]	Test Admin 101		Test admin First Name 101	Test admin Last name101
+[email protected]	Test Admin 102		Test admin First Name 102	Test admin Last name102
+[email protected]	Test Admin 103		Test admin First Name 103	Test admin Last name103
+[email protected]	Test Admin 104		Test admin First Name 104	Test admin Last name104
+[email protected]	Test Admin 105		Test admin First Name 105	Test admin Last name105
+[email protected]	Test Admin 106		Test admin First Name 106	Test admin Last name106
+[email protected]	Test Admin 107		Test admin First Name 107	Test admin Last name107
+[email protected]	Test Admin 108		Test admin First Name 108	Test admin Last name108
+[email protected]	Test Admin 109		Test admin First Name 109	Test admin Last name109
+[email protected]	Test Admin 110		Test admin First Name 110	Test admin Last name110
+[email protected]	Test Admin 111		Test admin First Name 111	Test admin Last name111
+[email protected]	Test Admin 112		Test admin First Name 112	Test admin Last name112
+[email protected]	Test Admin 113		Test admin First Name 113	Test admin Last name113
+[email protected]	Test Admin 114		Test admin First Name 114	Test admin Last name114
+[email protected]	Test Admin 115		Test admin First Name 115	Test admin Last name115
+[email protected]	Test Admin 116		Test admin First Name 116	Test admin Last name116
+[email protected]	Test Admin 117		Test admin First Name 117	Test admin Last name117
+[email protected]	Test Admin 118		Test admin First Name 118	Test admin Last name118
+[email protected]	Test Admin 119		Test admin First Name 119	Test admin Last name119