Functional tests fixed and enhanced

.github/workflows/functional-tests.yml

@@ -3,7 +3,7 @@ name: Functional Tests
 on:
   pull_request:
     branches: [main, develop]
-    types: [opened]
+    types: [opened, edited]
   workflow_dispatch:
 
 env:
@@ -56,7 +56,7 @@ jobs:
         working-directory: ${{ env.working-directory }}
         run: |
           touch .env
-          echo PUBLIC_BACKEND_API_URL=http://127.0.0.1:8000/api >> .env
+          echo PUBLIC_BACKEND_API_URL=http://localhost:8000/api >> .env
 
       - name: Create backend environment variables file
         working-directory: ${{ env.backend-directory }}
@@ -70,7 +70,7 @@ jobs:
           echo POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} >> .env
           echo DB_HOST=localhost >> .env
           echo CISO_ASSISTANT_SUPERUSER_EMAIL='' >> .env
-          echo CISO_ASSISTANT_URL=http://127.0.0.1:4173 >> .env
+          echo CISO_ASSISTANT_URL=http://localhost:4173 >> .env
       - name: Run migrations
         working-directory: ${{ env.backend-directory }}
         run: |
@@ -91,6 +91,6 @@ jobs:
         with:
           name: functional-tests-report
           path: |
-            ${{ env.working-directory }}/tests/reports/
             ${{ env.working-directory }}/tests/results/
-          retention-days: 30
+            ${{ env.working-directory }}/tests/reports/
+          retention-days: 5

.github/workflows/startup-tests.yml

@@ -69,7 +69,7 @@ jobs:
           echo POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} >> .env
           echo DB_HOST=localhost >> .env
           echo CISO_ASSISTANT_SUPERUSER_EMAIL='' >> .env
-          echo CISO_ASSISTANT_URL=http://127.0.0.1:4173 >> .env
+          echo CISO_ASSISTANT_URL=http://localhost:4173 >> .env
       - name: Run migrations
         working-directory: ${{ env.backend-directory }}
         run: |
@@ -90,6 +90,6 @@ jobs:
         with:
           name: startup-tests-report
           path: |
-            ${{ env.working-directory }}/tests/reports/
             ${{ env.working-directory }}/tests/results/
-          retention-days: 30
+            ${{ env.working-directory }}/tests/reports/
+          retention-days: 5

frontend/.gitignore

@@ -18,5 +18,7 @@ vite.config.js.timestamp-*
 .turbo
 .vercel
 .test-tmp
+.testhistory
+.testbackendoutput.out
 symlink-from
 coverage/**

frontend/package.json

@@ -6,20 +6,20 @@
 		"dev": "vite dev",
 		"build": "paraglide-js compile --project ./project.inlang && vite build",
 		"preview": "vite preview",
-		"test:e2e": "playwright test",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
 		"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
 		"test": "vitest",
 		"test:ci": "vitest run",
 		"test:ui": "vitest --ui",
+		"test:e2e": "ARGS=\"$npm_config_args\" docker compose -f ./tests/docker-compose.e2e-tests.yml up --force-recreate --build -V",
 		"coverage": "vitest run --coverage",
 		"lint": "prettier --plugin-search-dir . --check . && eslint .",
 		"format": "prettier --plugin-search-dir . --write .",
 		"postinstall": "paraglide-js compile --project ./project.inlang"
 	},
 	"devDependencies": {
+		"@playwright/test": "^1.40.1",
 		"@inlang/paraglide-js": "1.2.5",
-		"@playwright/test": "^1.28.1",
 		"@skeletonlabs/skeleton": "^2.3.0",
 		"@skeletonlabs/tw-plugin": "^0.2.2",
 		"@sveltejs/adapter-auto": "^3.0.0",

frontend/playwright.config.ts

@@ -4,43 +4,50 @@ import { devices } from '@playwright/test';
 const config: PlaywrightTestConfig = {
 	webServer: {
 		command: 'npm run build && npm run preview',
-		port: 4173
+		port: 4173,
+		reuseExistingServer: !process.env.CI,
 	},
 	testDir: 'tests',
 	outputDir: 'tests/results',
 	fullyParallel: true,
 	forbidOnly: !!process.env.CI,
 	retries: process.env.CI ? 2 : 1,
-	workers: process.env.CI ? 2 : 2,
+	workers: process.env.CI ? 1 : 2,
 	globalTimeout: 60 * 60 * 1000,
-	timeout: 60 * 1000,
+	timeout: 50 * 1000,
+	expect : {
+		timeout: 7 * 1000
+	},
 	reporter: [
 		[process.env.CI ? 'github' : 'list'],
-		['html', { open: 'never', outputFolder: 'tests/reports' }]
+		['html', { 
+			open: process.env.CI ? 'never' : process.env.DOCKER ? 'always' : 'on-failure',
+			outputFolder: 'tests/reports',
+			host: process.env.DOCKER ? '0.0.0.0' : 'localhost'
+		}]
 	],
 	use: {
-		// launchOptions: {
-		// 	slowMo: 1000,
-		// },
 		screenshot: 'only-on-failure',
-		video: 'retain-on-failure',
-		trace: 'retain-on-failure',
+		video: process.env.CI ? 'retain-on-failure' : 'on',
+		trace: process.env.CI ? 'retain-on-failure' : 'on',
 		contextOptions: {
-			recordVideo: { dir: 'tests/results/videos' }
+			recordVideo: { dir: "tests/results/videos"}
 		}
 	},
 	projects: [
 		{
 			name: 'chromium',
-			use: { ...devices['Desktop Chrome'] }
+			use: { ...devices['Desktop Chrome'] },
 		},
 		{
 			name: 'firefox',
-			use: { ...devices['Desktop Firefox'] }
-		}
+			use: { ...devices['Desktop Firefox'] },
+		},
 		// {
 		// 	name: 'webkit',
 		// 	use: { ...devices['Desktop Safari'] },
+		// 	name: 'webkit',
+		// 	use: { ...devices['Desktop Safari'] },
 		// }
 	]
 };

frontend/src/lib/components/ModelTable/Search.svelte

@@ -6,6 +6,7 @@
 
 <input
 	class="input bg-surface-50 max-w-2xl"
+	data-testid="search-input"
 	placeholder="Search..."
 	bind:value
 	on:input={() => handler.search(value)}

frontend/src/lib/components/ModelTable/Th.svelte

@@ -12,6 +12,7 @@
 	on:click={() => handler.sort(orderBy)}
 	class:active={$sorted.identifier === identifier}
 	class={_class}
+	data-testid="tableheader"
 >
 	<div class="flex items-center h-full">
 		<slot />

frontend/src/lib/components/TableRowActions/TableRowActions.svelte

@@ -64,15 +64,15 @@
 	<slot name="body" />
 	{#if !hasBody}
 		{#if displayDetail}
-			<a href={detailURL} class="unstyled cursor-pointer hover:text-primary-500"
+			<a href={detailURL} class="unstyled cursor-pointer hover:text-primary-500" data-testid="tablerow-detail-button"
 				><i class="fa-solid fa-eye" /></a
 			>
 		{/if}
 		{#if displayEdit}
 			<a
 				href={editURL}
 				on:click={stopPropagation}
-				class="unstyled cursor-pointer hover:text-primary-500"
+				class="unstyled cursor-pointer hover:text-primary-500" data-testid="tablerow-edit-button"
 				><i class="fa-solid fa-pen-to-square" /></a
 			>
 		{/if}
@@ -83,7 +83,7 @@
 					stopPropagation(_);
 				}}
 				on:keydown={(_) => modalConfirmDelete(row.meta.id, row.name ?? Object.values(row)[0])}
-				class="cursor-pointer hover:text-primary-500"><i class="fa-solid fa-trash" /></button
+				class="cursor-pointer hover:text-primary-500" data-testid="tablerow-delete-button"><i class="fa-solid fa-trash" /></button
 			>
 		{/if}
 	{/if}

frontend/src/routes/(app)/[model=urlmodel]/[id=uuid]/+page.svelte

@@ -172,16 +172,17 @@
 		<div class="flex flex-col space-y-2">
 			{#each Object.entries(data.data).filter(([key, _]) => !['id', 'is_published'].includes(key)) as [key, value]}
 				<div class="flex flex-col">
-					<div class="text-sm font-medium text-gray-800 capitalize-first">
+					<div class="text-sm font-medium text-gray-800 capitalize-first"
+					data-testid="{key.replace('_', '-')}-field-title">
 						{key.replace('_', ' ')}
 					</div>
 					<ul class="text-sm">
-						<li class="text-gray-600 list-none">
+						<li class="text-gray-600 list-none" data-testid={!(value instanceof Array) ? key.replace('_', '-') + "-field-value" : null}>
 							{#if value}
 								{#if Array.isArray(value)}
 									<ul>
 										{#each value as val}
-											<li>
+											<li data-testid={key.replace('_', '-') + "-field-value"}>
 												{#if val.str && val.id}
 													{@const itemHref = `/${
 														URL_MODEL_MAP[data.urlModel]['foreignKeyFields']?.find(
@@ -219,6 +220,7 @@
 			<a
 				href={`${$page.url.pathname}/edit?next=${$page.url.pathname}`}
 				class="btn variant-filled-primary h-fit"
+				data-testid="edit-button"
 				><i class="fa-solid fa-pen-to-square mr-2" /> Edit</a
 			>
 		{/if}

frontend/src/routes/(app)/compliance-assessments/[id=uuid]/+page.svelte

@@ -85,15 +85,15 @@
 		<div class="flex flex-col space-y-2">
 			{#each Object.entries(data.compliance_assessment).filter( ([key, _]) => ['name', 'description', 'project', 'framework'].includes(key) ) as [key, value]}
 				<div class="flex flex-col">
-					<div class="text-sm font-medium text-gray-800 capitalize-first">
+					<div class="text-sm font-medium text-gray-800 capitalize-first" data-testid={key.replaceAll('_', '-') + "-field-title"}>
 						{#if key === 'urn'}
 							URN
 						{:else}
 							{key.replace('_', ' ')}
 						{/if}
 					</div>
 					<ul class="text-sm">
-						<li class="text-gray-600 list-none">
+						<li class="text-gray-600 list-none" data-testid={key.replaceAll('_', '-') + "-field-value"}>
 							{#if value}
 								{#if Array.isArray(value)}
 									<ul>
@@ -144,6 +144,7 @@
 				<a
 					href={`${$page.url.pathname}/edit?next=${$page.url.pathname}`}
 					class="btn variant-filled-primary h-fit"
+					data-testid="edit-button"
 					><i class="fa-solid fa-pen-to-square mr-2" /> Edit</a
 				>
 			{/if}

frontend/src/routes/(app)/evidences/[id=uuid]/+page.svelte

@@ -71,21 +71,22 @@
 		<div class="flex flex-col space-y-2">
 			{#each Object.entries(data.evidence).filter( ([key, _]) => ['name', 'description', 'folder', 'attachment', 'link', 'comment'].includes(key) ) as [key, value]}
 				<div class="flex flex-col">
-					<div class="text-sm font-medium text-gray-800 capitalize-first">
+					<div class="text-sm font-medium text-gray-800 capitalize-first"
+					data-testid={key.replace('_', '-') + "-field-title"}>
 						{#if key === 'urn'}
 							URN
 						{:else}
 							{key.replace('_', ' ')}
 						{/if}
 					</div>
 					<ul class="text-sm">
-						<li class="text-gray-600 list-none">
+						<li class="text-gray-600 list-none" data-testid={!Array.isArray(value) || value.length <=0 ? key.replace('_', '-') + "-field-value" : null}>
 							{#if value}
 								{#if Array.isArray(value)}
 									<ul>
 										{#if value.length > 0}
 											{#each value as val}
-												<li>
+												<li data-testid={key.replace('_', '-') + "-field-value"}>
 													{#if val.str && val.id}
 														{@const itemHref = `/${
 															URL_MODEL_MAP[data.URLModel]['foreignKeyFields']?.find(
@@ -127,6 +128,7 @@
 				<a
 					href={`${$page.url.pathname}/edit?next=${$page.url.pathname}`}
 					class="btn variant-filled-primary h-fit"
+					data-testid="edit-button"
 					><i class="fa-solid fa-pen-to-square mr-2" /> Edit</a
 				>
 			{/if}
@@ -160,10 +162,11 @@
 	{#if data.evidence.attachment}
 		<div class="card px-6 py-4 bg-white flex flex-col shadow-lg space-y-4">
 			<div class="flex flex-row justify-between">
-				<h4 class="h4 font-semibold">{data.evidence.attachment}</h4>
+				<h4 class="h4 font-semibold" data-testid="attachment-name-title">{data.evidence.attachment}</h4>
 				<div class="space-x-2">
 					<a href={`./${data.evidence.id}/attachment`} class="btn variant-filled-primary h-fit"
-						><i class="fa-solid fa-download mr-2" /> Download</a
+						data-testid="attachment-download-button"	
+					><i class="fa-solid fa-download mr-2" /> Download</a
 					>
 					<button
 						on:click={(_) => {

frontend/src/routes/(app)/risk-assessments/[id=uuid]/+page.svelte

@@ -187,14 +187,15 @@
 					<a
 						href="/risk-assessments/{risk_assessment.id}/edit?next=/risk-assessments/{risk_assessment.id}"
 						class="btn variant-filled-primary"
+						data-testid="edit-button"
 					>
 						<i class="fa-solid fa-edit mr-2" />
 						Edit</a
 					>
 				{/if}
 			</div>
 			<div class="container w-1/3">
-				<div id="name" class="text-lg font-semibold">
+				<div id="name" class="text-lg font-semibold" data-testid="name-field-value">
 					{#if risk_assessment.is_draft}
 						<span class="badge bg-blue-200">Draft</span>
 					{/if}
@@ -220,14 +221,14 @@
 			</div>
 			<div class="container w-2/3">
 				<div class="text-sm">
-					<span class="font-semibold">Risk matrix:</span>
-					<a href="/risk-matrices/{risk_assessment.risk_matrix.id}" class="anchor"
+					<span class="font-semibold" data-testid="risk-matrix-field-title">Risk matrix:</span>
+					<a href="/risk-matrices/{risk_assessment.risk_matrix.id}" class="anchor" data-testid="risk-matrix-field-value"
 						>{risk_assessment.risk_matrix.name}</a
 					>
 				</div>
 				<br />
-				<div class="text-sm"><span class="font-semibold">Description:</span></div>
-				<div class="text-sm">{risk_assessment.description ?? '-'}</div>
+				<div class="text-sm"><span class="font-semibold" data-testid="description-field-title">Description:</span></div>
+				<div class="text-sm" data-testid="description-field-value">{risk_assessment.description ?? '-'}</div>
 			</div>
 		</div>
 	</div>

frontend/src/routes/(app)/risk-scenarios/[id=uuid]/edit/+page.svelte

@@ -295,10 +295,11 @@
 		<div class="flex flex-row justify-between space-x-4">
 			<button
 				class="btn bg-gray-400 text-white font-semibold w-full"
+				data-testid="cancel-button"
 				type="button"
 				on:click={cancel}>Cancel</button
 			>
-			<button class="btn variant-filled-primary font-semibold w-full">Save</button>
+			<button class="btn variant-filled-primary font-semibold w-full" data-testid="save-button">Save</button>
 		</div>
 	</SuperForm>
 </div>

frontend/tests/Dockerfile

@@ -0,0 +1,12 @@
+# The image version should match the playwright version in the package.json
+FROM mcr.microsoft.com/playwright:v1.40.1
+WORKDIR /app
+
+ARG PUBLIC_BACKEND_API_URL
+ENV PUBLIC_BACKEND_API_URL=$PUBLIC_BACKEND_API_URL
+
+COPY ../package*.json .
+RUN npm ci
+RUN npx playwright install --with-deps
+COPY .. .
+EXPOSE 9323