tests/add api json validation (#2047)

* add zod api json validation

* update test workflow

* fix deprecated function

* add pro plugin status

* removed test.only

* add visual tests

* add new api endpoints tests

* remove log line

* update lint

* update test config

* fix failed tests

* update yml

* update yml

* fix some minor issues

* fix lint

.github/workflows/e2e_tests.yml → .github/workflows/e2e_api_tests.yml

@@ -1,12 +1,17 @@
-name: End-to-End Tests Lite
+name: E2E_API Tests
 
 on:
     schedule:
-        - cron: '0 8 * * *'
+        # workflow will run daily at 8:00 AM BDT (2:00 AM UTC)
+        - cron: '0 2 * * *'
+    # workflow will trigger on pull request
     pull_request:
+        branches: [develop]
+    # workflow will trigger on push to develop branch
     push:
-        branches:
-            - develop
+        branches: [develop]
+
+    # workflow can be manually triggered
     workflow_dispatch:
 
 # Cancels all previous workflow runs for pull requests that have not completed.
@@ -15,12 +20,14 @@ concurrency:
     group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
     cancel-in-progress: true
 
+# workflow permissions
 permissions:
     checks: write
-    contents: read
+    contents: write
     issues: write
     pull-requests: write
 
+# workflow environment variables
 env:
     ADMIN: admin
     ADMIN_PASSWORD: password
@@ -39,16 +46,14 @@ env:
     DB_PREFIX: wp
     PR_NUMBER: ${{ github.event.number }}
     SHA: ${{ github.event.pull_request.head.sha }}
-    SYSTEM_INFO: ./tests/pw/systemInfo.json
+    SYSTEM_INFO: ./tests/pw/playwright/systemInfo.json
     API_TEST_RESULT: ./tests/pw/playwright-report/api/summary-report/results.json
     E2E_TEST_RESULT: ./tests/pw/playwright-report/e2e/summary-report/results.json
 
 jobs:
     tests:
-        name: e2e tests
+        name: e2e_api tests
         runs-on: ubuntu-latest
-        # strategy:
-        #   fail-fast: false
 
         steps:
             - name: Checkout testing repo
@@ -102,7 +107,6 @@ jobs:
               run: |
                   cd tests/pw
                   echo "The value of test DB_PORT is ${{ env.DB_PORT }}"
-                  echo "The value of test DB_PORT is ${{ env.GMAP }}"
 
             # Set permalink structure
             - name: Set Permalink structure
@@ -146,10 +150,17 @@ jobs:
                   cd tests/pw
                   npm run pw:browser-with-deps
 
+            # Install only the OS dependencies if cache hit  not needed
+            - name: Install Playwright OS dependencies
+              if: steps.playwright-cache.outputs.cache-hit == 'true'
+              run: |
+                cd tests/pw
+                npm run pw:deps-only
+
             # Run e2e tests
             - name: 🧪 Running the e2e tests
               id: e2e-test
-              if: steps.pw-install.outcome == 'success'
+              if: success()
               timeout-minutes: 40
               run: |
                   cd tests/pw 
@@ -158,7 +169,7 @@ jobs:
             # Run API tests
             - name: 🧪 Running the api tests
               id: api-test
-              if: always()
+              if: always() && steps.db-port.outcome == 'success'
               timeout-minutes: 5
               run: |
                   cd tests/pw 
@@ -175,25 +186,26 @@ jobs:
                       const script = require("./tests/pw/utils/gitTestSummary.ts")
                       return await script({github, context, core})
 
-            # Find PR comment
-            - name: Find PR comment by github-actions[bot]
-              uses: peter-evans/find-comment@v2
-              id: find-comment
-              if: github.event_name == 'pull_request'
-              with:
-                  issue-number: ${{ github.event.pull_request.number }}
-                  comment-author: 'github-actions[bot]'
-                  body-includes: Tests Summary
-
-            # Post test summary as PR comment
-            - name: Create or update PR comment
-              uses: peter-evans/create-or-update-comment@v2
-              if: github.event_name == 'pull_request'
-              with:
-                  comment-id: ${{ steps.find-comment.outputs.comment-id }}
-                  issue-number: ${{ github.event.pull_request.number }}
-                  body: ${{ steps.prepare-test-summary.outputs.result }}
-                  edit-mode: replace
+            # # Find PR comment
+            # - name: Find PR comment by github-actions[bot]
+            #   uses: peter-evans/find-comment@v2
+            #   id: find-comment
+            #   if: github.event_name == 'pull_request'
+            #   with:
+            #       issue-number: ${{ github.event.pull_request.number }}
+            #       comment-author: 'github-actions[bot]'
+            #       body-includes: Tests Summary
+
+            # # Post test summary as PR comment
+            # - name: Create or update PR comment
+            #   uses: peter-evans/create-or-update-comment@v2
+            #   if: github.event_name == 'pull_request'
+            #   with:
+            #       comment-id: ${{ steps.find-comment.outputs.comment-id }}
+            #       issue-number: ${{ github.event.pull_request.number }}
+            #       body: ${{ steps.prepare-test-summary.outputs.result }}
+            #       reactions: hooray
+            #       edit-mode: replace
 
             # Upload artifacts
             - name: Archive test artifacts (screenshots, HTML snapshots, Reports)

tests/pw/.env.example

@@ -13,7 +13,7 @@ DOKAN_PRO=true                          [Dokan pro active status]
 BASE_URL=https://example.com            [Base URL of the test site]
 CI=true                                 [CI/CD environment indicator]
 SLOWMO=10                               [Slow down test execution by provided seconds] [optional]
-SETUP=true                              [Whether not to run setup tests before actual tests] [optional]
+NO_SETUP=true                           [Whether not to run setup tests before actual tests] [optional]
 
 # Database Configuration                
 DB_HOST_NAME=localhost                  [Database server hostname or IP address]

tests/pw/.gitignore

@@ -174,6 +174,7 @@ playwright/
 test-results/
 playwright-report/
 playwright/.cache/
+visual.spec.ts-snapshots/
 storageState.json
 adminStorageState.json
 customerStorageState.json

tests/pw/README.MD

@@ -124,7 +124,7 @@ DOKAN_PRO=true                          [Dokan pro active status]
 BASE_URL=https://example.com            [Base URL of the test site]
 CI=true                                 [CI/CD environment indicator]
 SLOWMO=10                               [Slow down test execution by provided seconds] [optional]
-SETUP=true                              [Whether not to run setup tests before actual tests] [optional]
+NO_SETUP=true                           [Whether not to run setup tests before actual tests] [optional]
 
 # Database Configuration                
 DB_HOST_NAME=localhost                  [Database server hostname or IP address]

tests/pw/api.config.ts

@@ -1,11 +1,12 @@
-import { defineConfig } from '@playwright/test';
+import { defineConfig, expect } from '@playwright/test';
+import { customExpect } from '@utils/pwMatchers';
 import 'dotenv/config';
 
 export default defineConfig({
     testDir: './tests/api' /* test directory */,
     outputDir: 'playwright/api/test-artifacts/' /* Folder for test artifacts such as screenshots, videos, traces, etc. */,
     globalSetup: './global-setup' /* Path to the global setup file. This file will be required and run before all the tests. */,
-    globalTeardown: './global-teardown' /* Path to the global teardown file. This file will be required and run after all the tests. */,
+    // globalTeardown: './global-teardown' /* Path to the global teardown file. This file will be required and run after all the tests. */,
     globalTimeout: process.env.CI ? 20 * (60 * 1000) : 20 * (60 * 1000) /* Maximum time in milliseconds the whole test suite can run */,
     maxFailures: process.env.CI ? 30 : 30 /* The maximum number of test failures for the whole test suite run. After reaching this number, testing will stop and exit with an error. */,
     timeout: process.env.CI ? 5 * 1000 : 10 * 1000 /* Maximum time one test can run for. */,
@@ -17,21 +18,21 @@ export default defineConfig({
     // forbidOnly     : !!process.env.CI, 	/* Fail the build on CI if you accidentally left testonly in the source code. */
     repeatEach: 1 /* The number of times to repeat each test, useful for debugging flaky tests. */,
     retries: process.env.CI ? 1 : 0 /* The maximum number of retry attempts given to failed tests.  */,
-    workers: process.env.CI ? 1 : 1 /* Opt out of parallel tests on CI. */,
+    workers: process.env.CI ? 4 : 4 /* Opt out of parallel tests on CI. */,
     reportSlowTests: { max: 3, threshold: 10 } /* Whether to report slow test files. Pass null to disable this feature. */,
     reporter: process.env.CI
         ? [
+              ['github'],
               ['html', { open: 'never', outputFolder: 'playwright-report/api/html/html-report-api' }],
-              ['junit', { outputFile: 'playwright-report/api/junit-report/api-results.xml' }],
+              //   ['junit', { outputFile: 'playwright-report/api/junit-report/api-results.xml' }],
               ['list', { printSteps: true }],
               ['./utils/summaryReporter.ts', { outputFile: 'playwright-report/api/summary-report/results.json' }],
           ]
         : [
               ['html', { open: 'never', outputFolder: 'playwright-report/api/html/html-report-api' }],
-              ['junit', { outputFile: 'playwright-report/api/junit-report/api-results.xml' }],
+              //   ['junit', { outputFile: 'playwright-report/api/junit-report/api-results.xml' }],
               ['list', { printSteps: true }],
               ['./utils/summaryReporter.ts', { outputFile: 'playwright-report/api/summary-report/results.json' }],
-              //   ['allure-playwright', { detail: true, outputFolder: 'playwright-report/api/allure/allure-report', suiteTitle: false }],
           ],
 
     use: {
@@ -47,17 +48,33 @@ export default defineConfig({
     projects: [
         // Api project
 
+        // global_setup
+        {
+            name: 'global_setup',
+            testMatch: /global\.setup\.ts/,
+        },
+
         // api_setup
         {
             name: 'api_setup',
-            testMatch: /.*\.setup\.ts/,
+            // testMatch: /.*\.setup\.ts/,
+            testMatch: /.*\.setup\.spec\.ts/,
         },
 
         // api_tests
         {
             name: 'api_tests',
             testMatch: /.*\.spec\.ts/,
-            // dependencies: process.env.SETUP ? [] : ['api_setup'] /* whether not to run setup tests before running actual tests */,
+            dependencies: process.env.NO_SETUP ? [] : ['api_setup'] /* whether not to run setup tests before running actual tests */,
+            // teardown: 'global_teardown',
+        },
+
+        // global_teardown
+        {
+            name: 'global_teardown',
+            testMatch: /global\.teardown\.ts/,
         },
     ],
 });
+
+expect.extend(customExpect);

tests/pw/e2e.config.ts

@@ -1,16 +1,22 @@
-import { defineConfig, devices } from '@playwright/test';
+import { defineConfig, devices, expect } from '@playwright/test';
+import { customExpect } from '@utils/pwMatchers';
 import 'dotenv/config';
 
 export default defineConfig({
     testDir: './tests/e2e' /* test directory */,
     outputDir: 'playwright/e2e/test-artifacts/' /* Folder for test artifacts such as screenshots, videos, traces, etc. */,
     globalSetup: './global-setup' /* Path to the global setup file. This file will be required and run before all the tests. */,
-    globalTeardown: './global-teardown' /* Path to the global teardown file. This file will be required and run after all the tests. */,
+    // globalTeardown: './global-teardown' /* Path to the global teardown file. This file will be required and run after all the tests. */,
     globalTimeout: process.env.CI ? 40 * (60 * 1000) : 20 * (60 * 1000) /* Maximum time in milliseconds the whole test suite can run */,
     maxFailures: process.env.CI ? 30 : 30 /* The maximum number of test failures for the whole test suite run. After reaching this number, testing will stop and exit with an error. */,
     timeout: process.env.CI ? 40 * 1000 : 35 * 1000 /* Maximum time one test can run for. */,
     expect: {
         timeout: 10 * 1000 /* Maximum time expect() should wait for the condition to be met.  For example in `await expect(locator).toHaveText();`*/,
+        toHaveScreenshot: {
+            maxDiffPixelRatio: 0.2,
+            maxDiffPixels: 500,
+            threshold: 0.5,
+        },
     } /* Configuration for the expect assertion library */,
     preserveOutput: 'always' /* Whether to preserve test output in the testConfig.outputDir. Defaults to 'always'. */,
     // fullyParallel  : true, 	/* Run tests in files in parallel */
@@ -21,17 +27,17 @@ export default defineConfig({
     reportSlowTests: { max: 3, threshold: 25 } /* Whether to report slow test files. Pass null to disable this feature. */,
     reporter: process.env.CI
         ? [
+              ['github'],
               ['html', { open: 'never', outputFolder: 'playwright-report/e2e/html/html-report-e2e' }],
-              ['junit', { outputFile: 'playwright-report/e2e/junit-report/e2e-results.xml' }],
+              //   ['junit', { outputFile: 'playwright-report/e2e/junit-report/e2e-results.xml' }],
               ['list', { printSteps: true }],
               ['./utils/summaryReporter.ts', { outputFile: 'playwright-report/e2e/summary-report/results.json' }],
           ]
         : [
               ['html', { open: 'never', outputFolder: 'playwright-report/e2e/html/html-report-e2e' }],
-              ['junit', { outputFile: 'playwright-report/e2e/junit-report/e2e-results.xml' }],
+              //   ['junit', { outputFile: 'playwright-report/e2e/junit-report/e2e-results.xml' }],
               ['list', { printSteps: true }],
               ['./utils/summaryReporter.ts', { outputFile: 'playwright-report/e2e/summary-report/results.json' }],
-              // ['allure-playwright',	{ detail: true, outputFolder: 'playwright-report/e2e/allure/allure-report', suiteTitle: false }]
           ],
 
     use: {
@@ -40,16 +46,15 @@ export default defineConfig({
         actionTimeout: 15 * 1000 /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */,
         navigationTimeout: 20 * 1000 /* Maximum time each navigation such as 'goto()' can take. */,
         baseURL: process.env.BASE_URL ? process.env.BASE_URL : 'http://localhost:9999' /* Base URL */,
-        // browserName      : 'chromium', /* Name of the browser that runs tests. */
+        // browserName: 'chromium' /* Name of the browser that runs tests. */,
         bypassCSP: true /* Toggles bypassing page's Content-Security-Policy. */,
-        // channel          : 'chrome', /* Browser distribution channel. */
-        // colorScheme      : 'dark', /* Emulates 'prefers-colors-scheme' media feature, supported values are 'light', 'dark', 'no-preference' */
+        // channel: 'chrome' /* Browser distribution channel. */,
+        // colorScheme: 'dark' /* Emulates 'prefers-colors-scheme' media feature, supported values are 'light', 'dark', 'no-preference' */,
         headless: process.env.CI ? !!process.env.CI : false /* Whether to run tests on headless or non-headless mode */,
         ignoreHTTPSErrors: true /* Whether to ignore HTTPS errors during navigation. */,
-        // trace            : 'retain-on-failure', /* Record trace only when retrying a test for the first time. */
         trace: 'on-first-retry' /* Record trace only when retrying a test for the first time. */,
         screenshot: 'only-on-failure' /* Capture screenshot after each test failure. */,
-        // video            : 'retain-on-failure', /* Record video only when retrying a test for the first time. */
+        // video: 'retain-on-failure' /* Record video only when retrying a test for the first time. */,
         video: 'on-first-retry' /* Record video only when retrying a test for the first time. */,
         // viewport         : { width: 1280, height: 720 }, /* Size of viewport */
         launchOptions: { slowMo: process.env.SLOWMO ? Number(process.env.SLOWMO) * 1000 : 0 /* whether to slow down test execution by provided seconds */ },
@@ -61,14 +66,15 @@ export default defineConfig({
         // e2e_setup
         {
             name: 'e2e_setup',
-            testMatch: /.*\.setup\.ts/,
+            // testMatch: /.*\.setup\.ts/,
+            testMatch: /.*\.setup\.spec\.ts/,
         },
 
         // e2e_tests
         {
             name: 'e2e_tests',
             testMatch: /.*\.spec\.ts/,
-            // dependencies: process.env.SETUP ? [] : ['e2e_setup'] /* whether not to run setup tests before running actual tests */,
+            // dependencies: process.env.NO_SETUP ? [] : ['e2e_setup'] /* whether not to run setup tests before running actual tests */,
         },
 
         // local site setup project
@@ -79,3 +85,5 @@ export default defineConfig({
         },
     ],
 });
+
+expect.extend(customExpect);

tests/pw/global-setup.ts

@@ -16,7 +16,7 @@ async function globalSetup(config: FullConfig) {
         }
         console.log('retrying...');
     }
-    console.log('ServerUrl:', process.env.SERVER_URL);
+    console.log('ServerUrl:', serverUrl);
 
     console.log('Global Setup Finished!');
 }

tests/pw/global-teardown.ts

@@ -6,7 +6,7 @@ import { payloads } from '@utils/payloads';
 async function globalSetup() {
     console.log('Global Teardown running....');
 
-    const systemInfo = 'systemInfo.json';
+    const systemInfo = 'playwright/systemInfo.json';
 
     // get test environment info
     if (!helpers.fileExists(systemInfo)) {