From 1d50f957cfa0369d69000ba7af6c1671007d3dd3 Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 09:56:15 +0200 Subject: [PATCH 1/6] feat(FilesTotalDownloadSize): upgrade js-dataverse version --- dev-env/.env | 2 +- package-lock.json | 8 ++++---- package.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev-env/.env b/dev-env/.env index a7a8a8574..05d4bebe8 100644 --- a/dev-env/.env +++ b/dev-env/.env @@ -1,4 +1,4 @@ POSTGRES_VERSION=13 DATAVERSE_DB_USER=dataverse SOLR_VERSION=9.3.0 -REGISTRY=ghcr.io +REGISTRY=docker.io diff --git a/package-lock.json b/package-lock.json index aef2e8157..0cecf950f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@faker-js/faker": "7.6.0", - "@iqss/dataverse-client-javascript": "2.0.0-pr93.8996380", + "@iqss/dataverse-client-javascript": "2.0.0-pr97.418bf5e", "@iqss/dataverse-design-system": "*", "@istanbuljs/nyc-config-typescript": "1.0.2", "@tanstack/react-table": "8.9.2", @@ -3588,9 +3588,9 @@ }, "node_modules/@iqss/dataverse-client-javascript": { "name": "@IQSS/dataverse-client-javascript", - "version": "2.0.0-pr93.8996380", - "resolved": "https://npm.pkg.github.com/download/@IQSS/dataverse-client-javascript/2.0.0-pr93.8996380/e2b872620f16a08f5cb3e94dbf7afd0afc55e1d2", - "integrity": "sha512-hlSKtjPhpASVZD5RQsk2sRTrLWPbucY7G7uDcjWIWpg0NiHxhvz3Tny5q8yIAzQ44v+8K/n5K0fr8gv9PUBeiQ==", + "version": "2.0.0-pr97.418bf5e", + "resolved": "https://npm.pkg.github.com/download/@IQSS/dataverse-client-javascript/2.0.0-pr97.418bf5e/0d604232d6f567b41328143c2d5d27513d069380", + "integrity": "sha512-wKRR1ORFkFSoFoCrWsCQvcm1vExIIrVN3pJbIl9Nk8lopP7zkUEB1xcAU+UVShbGktIZIa20TQMFl0ONafDiZw==", "license": "MIT", "dependencies": { "@types/node": "^18.15.11", diff --git a/package.json b/package.json index ca0daa72c..fc806aa14 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "@faker-js/faker": "7.6.0", - "@iqss/dataverse-client-javascript": "2.0.0-pr93.8996380", + "@iqss/dataverse-client-javascript": "2.0.0-pr97.418bf5e", "@iqss/dataverse-design-system": "*", "@istanbuljs/nyc-config-typescript": "1.0.2", "@tanstack/react-table": "8.9.2", From b996808c69f60809513670bef9ad14c3060902b7 Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 09:56:58 +0200 Subject: [PATCH 2/6] feat(FilesTotalDownloadSize): add search criteria to get the total size --- src/files/domain/repositories/FileRepository.ts | 3 ++- .../domain/useCases/getFilesTotalDownloadSize.ts | 6 ++++-- src/files/infrastructure/FileJSDataverseRepository.ts | 11 +++++++++-- src/sections/dataset/dataset-files/useFiles.tsx | 4 ++-- src/stories/files/FileMockLoadingRepository.ts | 4 +++- src/stories/files/FileMockNoDataRepository.ts | 5 ++++- src/stories/files/FileMockNoFiltersRepository.ts | 4 +++- src/stories/files/FileMockRepository.ts | 5 ++++- 8 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/files/domain/repositories/FileRepository.ts b/src/files/domain/repositories/FileRepository.ts index 4ce01579a..976fd5a66 100644 --- a/src/files/domain/repositories/FileRepository.ts +++ b/src/files/domain/repositories/FileRepository.ts @@ -19,7 +19,8 @@ export interface FileRepository { ) => Promise getFilesTotalDownloadSizeByDatasetPersistentId: ( datasetPersistentId: string, - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + criteria?: FileCriteria ) => Promise getUserPermissionsById: (id: number) => Promise } diff --git a/src/files/domain/useCases/getFilesTotalDownloadSize.ts b/src/files/domain/useCases/getFilesTotalDownloadSize.ts index 81c930559..c95bb93ba 100644 --- a/src/files/domain/useCases/getFilesTotalDownloadSize.ts +++ b/src/files/domain/useCases/getFilesTotalDownloadSize.ts @@ -1,13 +1,15 @@ import { FileRepository } from '../repositories/FileRepository' import { DatasetVersion } from '../../../dataset/domain/models/Dataset' +import { FileCriteria } from '../models/FileCriteria' export async function getFilesTotalDownloadSize( fileRepository: FileRepository, datasetPersistentId: string, - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + criteria?: FileCriteria ): Promise { return fileRepository - .getFilesTotalDownloadSizeByDatasetPersistentId(datasetPersistentId, datasetVersion) + .getFilesTotalDownloadSizeByDatasetPersistentId(datasetPersistentId, datasetVersion, criteria) .catch((error: Error) => { throw new Error(error.message) }) diff --git a/src/files/infrastructure/FileJSDataverseRepository.ts b/src/files/infrastructure/FileJSDataverseRepository.ts index a3ef1e93c..783cb998e 100644 --- a/src/files/infrastructure/FileJSDataverseRepository.ts +++ b/src/files/infrastructure/FileJSDataverseRepository.ts @@ -90,10 +90,17 @@ export class FileJSDataverseRepository implements FileRepository { getFilesTotalDownloadSizeByDatasetPersistentId( datasetPersistentId: string, - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + criteria: FileCriteria = new FileCriteria() ): Promise { return getDatasetFilesTotalDownloadSize - .execute(datasetPersistentId, datasetVersion.toString(), FileDownloadSizeMode.ARCHIVAL) + .execute( + datasetPersistentId, + datasetVersion.toString(), + FileDownloadSizeMode.ARCHIVAL, + DomainFileMapper.toJSFileSearchCriteria(criteria), + includeDeaccessioned + ) .catch((error: ReadError) => { throw new Error(error.message) }) diff --git a/src/sections/dataset/dataset-files/useFiles.tsx b/src/sections/dataset/dataset-files/useFiles.tsx index 476d9cd20..538da0921 100644 --- a/src/sections/dataset/dataset-files/useFiles.tsx +++ b/src/sections/dataset/dataset-files/useFiles.tsx @@ -88,14 +88,14 @@ export function useFiles( ]) useEffect(() => { - getFilesTotalDownloadSize(filesRepository, datasetPersistentId, datasetVersion) + getFilesTotalDownloadSize(filesRepository, datasetPersistentId, datasetVersion, criteria) .then((filesTotalDownloadSize: number) => { setFilesTotalDownloadSize(filesTotalDownloadSize) }) .catch((error) => { console.error('There was an error getting the files total download size', error) }) - }, [filesRepository, datasetPersistentId, datasetVersion]) + }, [filesRepository, datasetPersistentId, datasetVersion, criteria]) return { files, diff --git a/src/stories/files/FileMockLoadingRepository.ts b/src/stories/files/FileMockLoadingRepository.ts index f505c33ef..eff4bd897 100644 --- a/src/stories/files/FileMockLoadingRepository.ts +++ b/src/stories/files/FileMockLoadingRepository.ts @@ -47,7 +47,9 @@ export class FileMockLoadingRepository implements FileRepository { // eslint-disable-next-line unused-imports/no-unused-vars datasetPersistentId: string, // eslint-disable-next-line unused-imports/no-unused-vars - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + // eslint-disable-next-line unused-imports/no-unused-vars + criteria?: FileCriteria ): Promise { return new Promise((resolve) => { setTimeout(() => { diff --git a/src/stories/files/FileMockNoDataRepository.ts b/src/stories/files/FileMockNoDataRepository.ts index 46fd0332a..ba7c10a3b 100644 --- a/src/stories/files/FileMockNoDataRepository.ts +++ b/src/stories/files/FileMockNoDataRepository.ts @@ -5,6 +5,7 @@ import { FilesCountInfoMother } from '../../../tests/component/files/domain/mode import { FileUserPermissions } from '../../files/domain/models/FileUserPermissions' import { FileUserPermissionsMother } from '../../../tests/component/files/domain/models/FileUserPermissionsMother' import { DatasetVersion } from '../../dataset/domain/models/Dataset' +import { FileCriteria } from '../../files/domain/models/FileCriteria' export class FileMockNoDataRepository implements FileRepository { getAllByDatasetPersistentId( @@ -37,7 +38,9 @@ export class FileMockNoDataRepository implements FileRepository { // eslint-disable-next-line unused-imports/no-unused-vars datasetPersistentId: string, // eslint-disable-next-line unused-imports/no-unused-vars - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + // eslint-disable-next-line unused-imports/no-unused-vars + criteria?: FileCriteria ): Promise { return new Promise((resolve) => { setTimeout(() => { diff --git a/src/stories/files/FileMockNoFiltersRepository.ts b/src/stories/files/FileMockNoFiltersRepository.ts index 1566b876d..eec689eaa 100644 --- a/src/stories/files/FileMockNoFiltersRepository.ts +++ b/src/stories/files/FileMockNoFiltersRepository.ts @@ -49,7 +49,9 @@ export class FileMockNoFiltersRepository implements FileRepository { // eslint-disable-next-line unused-imports/no-unused-vars datasetPersistentId: string, // eslint-disable-next-line unused-imports/no-unused-vars - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + // eslint-disable-next-line unused-imports/no-unused-vars + criteria?: FileCriteria ): Promise { return new Promise((resolve) => { setTimeout(() => { diff --git a/src/stories/files/FileMockRepository.ts b/src/stories/files/FileMockRepository.ts index 85ceafe96..63e1499fa 100644 --- a/src/stories/files/FileMockRepository.ts +++ b/src/stories/files/FileMockRepository.ts @@ -7,6 +7,7 @@ import { FilePaginationInfo } from '../../files/domain/models/FilePaginationInfo import { FileUserPermissionsMother } from '../../../tests/component/files/domain/models/FileUserPermissionsMother' import { FileUserPermissions } from '../../files/domain/models/FileUserPermissions' import { DatasetVersion } from '../../dataset/domain/models/Dataset' +import { FileCriteria } from '../../files/domain/models/FileCriteria' export class FileMockRepository implements FileRepository { // eslint-disable-next-line unused-imports/no-unused-vars @@ -39,7 +40,9 @@ export class FileMockRepository implements FileRepository { // eslint-disable-next-line unused-imports/no-unused-vars datasetPersistentId: string, // eslint-disable-next-line unused-imports/no-unused-vars - datasetVersion: DatasetVersion + datasetVersion: DatasetVersion, + // eslint-disable-next-line unused-imports/no-unused-vars + criteria?: FileCriteria ): Promise { return new Promise((resolve) => { setTimeout(() => { From 0437248b2af06ce24730a76ccb29fb899878547d Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 10:11:00 +0200 Subject: [PATCH 3/6] feat(FilesTotalDownloadSize): add unit test for the search criteria --- .../dataset-files/DatasetFiles.spec.tsx | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/component/sections/dataset/dataset-files/DatasetFiles.spec.tsx b/tests/component/sections/dataset/dataset-files/DatasetFiles.spec.tsx index 1e394acc6..f6ac50b7c 100644 --- a/tests/component/sections/dataset/dataset-files/DatasetFiles.spec.tsx +++ b/tests/component/sections/dataset/dataset-files/DatasetFiles.spec.tsx @@ -443,5 +443,32 @@ describe('DatasetFiles', () => { new FilePaginationInfo(5, 10, 200) ) }) + + it('calls getFilesTotalDownloadSizeByDatasetPersistentId with the correct parameters when applying search file criteria', () => { + cy.customMount( + + + + ) + + cy.findByRole('button', { name: 'File Type: All' }).click() + cy.findByText('Image (485)').should('exist').click() + cy.get('table > thead > tr > th > input[type=checkbox]').click() + cy.findByRole('button', { name: 'Select all 200 files in this dataset.' }).click() + cy.findByText( + 'The overall size of the files selected (19.4 KB) for download exceeds the zip limit of 1.0 B. Please unselect some files to continue.' + ).should('exist') + + cy.wrap(fileRepository.getFilesTotalDownloadSizeByDatasetPersistentId).should( + 'be.calledWith', + datasetPersistentId, + datasetVersion, + new FileCriteria().withFilterByType('image') + ) + }) }) }) From 00c315c730a683f72e065fd6ce5757c7413d12db Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 10:33:11 +0200 Subject: [PATCH 4/6] feat(FilesTotalDownloadSize): add integration test for the files search criteria --- .../files/FileJSDataverseRepository.spec.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/e2e-integration/integration/files/FileJSDataverseRepository.spec.ts b/tests/e2e-integration/integration/files/FileJSDataverseRepository.spec.ts index 9a0614e91..e13c99147 100644 --- a/tests/e2e-integration/integration/files/FileJSDataverseRepository.spec.ts +++ b/tests/e2e-integration/integration/files/FileJSDataverseRepository.spec.ts @@ -675,5 +675,52 @@ describe('File JSDataverse Repository', () => { expect(totalDownloadSize).to.deep.equal(expectedTotalDownloadSize) }) }) + + it('gets the total download size of all files in a dataset when passing files criteria', async () => { + const files = [ + FileHelper.create('csv', { + description: 'Some description', + categories: ['category'], + restrict: 'true', + tabIngest: 'false' + }), + FileHelper.create('txt', { + description: 'Some description', + tabIngest: 'false' + }), + FileHelper.create('txt', { + description: 'Some description', + categories: ['category_1'] + }) + ] + const dataset = await DatasetHelper.createWithFiles(files).then((datasetResponse) => + datasetRepository.getByPersistentId(datasetResponse.persistentId) + ) + if (!dataset) throw new Error('Dataset not found') + + await TestsUtils.wait(2500) // wait for the files to be ingested + + const expectedTotalDownloadSize = await fileRepository + .getAllByDatasetPersistentId( + dataset.persistentId, + dataset.version, + new FilePaginationInfo(1, 10, 3), + new FileCriteria().withFilterByType('csv') + ) + .then((files) => { + return files.reduce((totalDownloadSize, file) => { + return totalDownloadSize + file.size.toBytes() + }, 0) + }) + await fileRepository + .getFilesTotalDownloadSizeByDatasetPersistentId( + dataset.persistentId, + dataset.version, + new FileCriteria().withFilterByType('csv') + ) + .then((totalDownloadSize) => { + expect(totalDownloadSize).to.deep.equal(expectedTotalDownloadSize) + }) + }) }) }) From 09721700ca00a670dc1574f366671d13a17ece6c Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 11:15:53 +0200 Subject: [PATCH 5/6] fix: increase e2e github action timeout --- dev-env/run-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-env/run-env.sh b/dev-env/run-env.sh index a8a2f09cd..28ebd20c6 100755 --- a/dev-env/run-env.sh +++ b/dev-env/run-env.sh @@ -6,7 +6,7 @@ export DATAVERSE_IMAGE_TAG=$1 export COMPOSE_HTTP_TIMEOUT=200 # Timeout for Dataverse bootstrap configbaker -export DATAVERSE_BOOTSTRAP_TIMEOUT="5m" +export DATAVERSE_BOOTSTRAP_TIMEOUT="10m" echo "INFO - Setting up Dataverse on image tag ${DATAVERSE_IMAGE_TAG}..." From 94a01927feceed1e2b347a56fba6b737ad4a10dd Mon Sep 17 00:00:00 2001 From: MellyGray Date: Wed, 25 Oct 2023 11:33:45 +0200 Subject: [PATCH 6/6] fix: increase e2e github action timeout run command --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d962cc4f4..5aa5600cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,7 +49,7 @@ jobs: run: ./run-env.sh "$E2E_DATAVERSE_IMAGE_TAG" - name: Wait for containers to be ready - run: timeout 360s sh -c 'while ! docker logs dev_dataverse_bootstrap 2>&1 | grep -q "your instance has been configured"; do sleep 2; done' + run: timeout 600s sh -c 'while ! docker logs dev_dataverse_bootstrap 2>&1 | grep -q "your instance has been configured"; do sleep 2; done' - name: Run e2e tests run: npm run test:e2e