From a08fbe2b345dd9f10998b758eb962f899bd6653e Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:13:02 +0100 Subject: [PATCH 001/360] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index c36c7d1e963..a97adf4149b 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -286,5 +286,8 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl git push OdumInstitute 4709-postgresql_96 ---- +Develop branch and pull request +-------------------------------------------- +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 8277ac5adac8511efa9bb92d3f51dcf99973e67b Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:15:10 +0100 Subject: [PATCH 002/360] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index a97adf4149b..11ce0c0d144 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -286,8 +286,9 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl git push OdumInstitute 4709-postgresql_96 ---- + Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From be8c167a6454bc86535ce77e66dedb1d76ca0b08 Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:53:13 +0100 Subject: [PATCH 003/360] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 11ce0c0d144..5b809782f76 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 8c5113b9c3f4e8380df45d91c84fdc3acdaf7f1a Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:04:48 +0100 Subject: [PATCH 004/360] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 5b809782f76..35e37ac0471 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 1edb8bac22046dffd5c7c21197e013bb3be80a60 Mon Sep 17 00:00:00 2001 From: Martin Amouzou <85512093+martinAmouzou@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:07:58 +0100 Subject: [PATCH 005/360] Update version-control.rst --- doc/sphinx-guides/source/developers/version-control.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 35e37ac0471..99e33164417 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -289,6 +289,6 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl Develop branch and pull request -------------------------------------------- -Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide `_. +Please do not use your forked develop branch to push a PR and follow the `1st scenario: preparing the first pull request of Version Control Guide <#summary-of-git-commands>`_. Previous: :doc:`troubleshooting` | Next: :doc:`sql-upgrade-scripts` From 376f31de215a677ad874adebe4c179be675c6ec3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 10 Apr 2024 23:57:38 +0200 Subject: [PATCH 006/360] ci(ct): do no longer run base push workflow in PRs --- .github/workflows/container_base_push.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b938851f816..20083c90f33 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -3,17 +3,11 @@ name: Base Container Image on: push: + tags: + - 'v[6-9].*' branches: - 'develop' - - 'master' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_base_push.yml' - pull_request: - branches: - - 'develop' - - 'master' + - '10478-version-base-img' paths: - 'modules/container-base/**' - 'modules/dataverse-parent/pom.xml' From b8b95a995b3cebab201577429622ba086a81634e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:05:25 +0200 Subject: [PATCH 007/360] ci(ct): add a job for base images to discover branches to work on For scheduled maintenance, we will provide updates to the last three base images as well as the develop branch. --- .github/workflows/container_base_push.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 20083c90f33..41935a025a8 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -20,6 +20,25 @@ env: PLATFORMS: linux/amd64,linux/arm64 jobs: + discover: + name: Discover Release Matrix + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled + if: ${{ github.repository_owner == 'IQSS' }} + steps: + - name: Build branch matrix options + id: matrix + run: | + # Get last three releases and include develop branch as matrix elements + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" + else + echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + fi + build: name: Build image runs-on: ubuntu-latest From 6f046208086bc7f3bb4078c92b6ae8723f78b030 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:06:24 +0200 Subject: [PATCH 008/360] ci(ct): setup base image build job as matrix of discovered branches Checkout the branch/tag we receive from the discovery job --- .github/workflows/container_base_push.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 41935a025a8..b424936383c 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -45,15 +45,18 @@ jobs: permissions: contents: read packages: read + needs: discover strategy: matrix: - jdk: [ '17' ] + branch: ${{ fromJson(needs.discover.outputs.branches) }} # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v3 From 4fc0267296944a7dec92e1a70a651649cf9941db Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:07:40 +0200 Subject: [PATCH 009/360] ci(ct): upgrade java action in base image and determine java version from Maven Using Maven properties, we find out which is the projects preferred Java version. We don't need a build matrix here. --- .github/workflows/container_base_push.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b424936383c..970e67794c2 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -57,14 +57,19 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} + - name: Determine Java version from package + run: | + echo "JAVA_VERSION=$(mvn -f modules/container-base -Pct help:evaluate -Dexpression=target.java.version -q -DforceStdout)" >> ${GITHUB_ENV} - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 with: - java-version: ${{ matrix.jdk }} - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v3 + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: | + modules/container-base/pom.xml + with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} From 8d3097181296f70e1948adf43c2823862e8cb1fd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:08:57 +0200 Subject: [PATCH 010/360] ci(ct): remove pull request safeguards from base image workflow We no longer work on PRs in this workflow, so we don't need to safeguard against not running steps in this case. --- .github/workflows/container_base_push.yml | 33 +++++++++-------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 970e67794c2..31a044ef02a 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -78,8 +78,18 @@ jobs: - name: Build base container image with local architecture run: mvn -f modules/container-base -Pct package - # Run anything below only if this is not a pull request. - # Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets. + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v2 + - name: Deploy multi-arch base container image to Docker Hub + run: mvn -f modules/container-base -Pct deploy -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub @@ -90,23 +100,6 @@ jobs: repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" readme-filepath: ./modules/container-base/README.md - - - if: ${{ github.event_name != 'pull_request' }} - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - if: ${{ github.event_name != 'pull_request' }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 - - name: Re-set image tag based on branch - if: ${{ github.ref_name == 'master' }} - run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV - - if: ${{ github.event_name != 'pull_request' }} - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }} push-app-img: name: "Rebase & Publish App Image" permissions: @@ -115,6 +108,6 @@ jobs: pull-requests: write needs: build # We do not release a new base image for pull requests, so do not trigger. - if: ${{ github.event_name != 'pull_request' }} + # if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/container_app_push.yml secrets: inherit From bb06a94e27e6edf6f9ccd11dbf6209d90b7d4dc2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:09:43 +0200 Subject: [PATCH 011/360] ci(ct): remove pull request safeguards from base image workflow We no longer work on PRs in this workflow, so we don't need to safeguard against not running steps in this case. Also we no longer use the static image names, but rely on the truth Maven has in the container-base module about the version of the image. --- .github/workflows/container_base_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 31a044ef02a..4295280b639 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -16,7 +16,6 @@ on: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC env: - IMAGE_TAG: unstable PLATFORMS: linux/amd64,linux/arm64 jobs: From 0ba86b2e4f68709d49c72a99580ec5c5b1ac4d5a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:10:23 +0200 Subject: [PATCH 012/360] ci(ct): stop on purpose to test the feature branch before we actually push to Docker Hub --- .github/workflows/container_base_push.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4295280b639..bfccefaf9eb 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -69,13 +69,11 @@ jobs: cache-dependency-path: | modules/container-base/pom.xml + - name: Stop on purpose for testing + uses: actions/github-script@v3 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Build base container image with local architecture - run: mvn -f modules/container-base -Pct package + script: | + core.setFailed('Stopped on purpose.') # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 37f4a51b2ff0341f7bb6d930614f04c2238c213b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:11:00 +0200 Subject: [PATCH 013/360] feat(ct): make the container base image follow sequential, incrementing versions --- modules/container-base/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index fc672696df4..2754b081986 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -12,6 +12,8 @@ io.gdcc container-base + + 1 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -40,7 +42,7 @@ docker-build gdcc/base:${base.image.tag} - unstable + R${project.version} eclipse-temurin:${target.java.version}-jre 1000 1000 From 20c6a58915711efd9e18d8454813623ead9dbb9b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:17:47 +0200 Subject: [PATCH 014/360] ci(ct): add missing output definitions to base image workflow matrix discovery --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index bfccefaf9eb..29abab230e3 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -27,6 +27,8 @@ jobs: packages: read # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled if: ${{ github.repository_owner == 'IQSS' }} + outputs: + branches: ${{ steps.matrix.outputs.branches }} steps: - name: Build branch matrix options id: matrix From e9236638eb2921b46318c3a4fb51e663ec021a56 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:20:41 +0200 Subject: [PATCH 015/360] test(ct): for testing, determine if matrix discovery works for schedule event --- .github/workflows/container_base_push.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 29abab230e3..ab91252efc6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -34,11 +34,11 @@ jobs: id: matrix run: | # Get last three releases and include develop branch as matrix elements - if [[ "${{ github.event_name }}" == "schedule" ]]; then + #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" - else - echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" - fi + #else + # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + #fi build: name: Build image From def5d9e2842c09c3d647619f0cea266528c71a82 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:23:56 +0200 Subject: [PATCH 016/360] fix(ct): base image build matrix output must be on a single line Fix jq output by removing newlines and superfluous whitespace. --- .github/workflows/container_base_push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ab91252efc6..705b152d1bc 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -35,7 +35,9 @@ jobs: run: | # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tee -a "$GITHUB_OUTPUT" + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ + jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tr -d "\n" | tr -s " " | \ + tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" #fi From b028c2610db5688bf443f436e1bbdb1c7c8fcb6c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:27:06 +0200 Subject: [PATCH 017/360] fix(ct): do not stop building other base images if one build fails --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 705b152d1bc..f36c5a8de2d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -50,6 +50,7 @@ jobs: packages: read needs: discover strategy: + fail-fast: false matrix: branch: ${{ fromJson(needs.discover.outputs.branches) }} # Only run in upstream repo - avoid unnecessary runs in forks From b453a12b780c4149a3ca2ffb5469d6cfb85c5290 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:45:07 +0200 Subject: [PATCH 018/360] fix(ct): avoid costly lookup of Java version to be used via Maven, grep from XML instead --- .github/workflows/container_base_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f36c5a8de2d..d6af36702f6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -61,9 +61,9 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - - name: Determine Java version from package + - name: Determine Java version from Parent POM run: | - echo "JAVA_VERSION=$(mvn -f modules/container-base -Pct help:evaluate -Dexpression=target.java.version -q -DforceStdout)" >> ${GITHUB_ENV} + echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} uses: actions/setup-java@v4 From cb2fac02fb9370f565ef613c0f2e57f5ce7957a0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 00:48:19 +0200 Subject: [PATCH 019/360] test(ct): no longer stop before Docker, but tell DMP to skip pushing --- .github/workflows/container_base_push.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index d6af36702f6..24b2640f3e3 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -74,12 +74,6 @@ jobs: cache-dependency-path: | modules/container-base/pom.xml - - name: Stop on purpose for testing - uses: actions/github-script@v3 - with: - script: | - core.setFailed('Stopped on purpose.') - # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! @@ -91,7 +85,7 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v2 - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From afa51c78327748d14a1e9bb6f1d6eac78e357f48 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:12:13 +0200 Subject: [PATCH 020/360] fix(ct): skip building and pushing the base image for v6.0 and v6.1, which is compatible with the v6.2 image --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 24b2640f3e3..cb83a2799bb 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -85,6 +85,8 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v2 - name: Deploy multi-arch base container image to Docker Hub + # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. + if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} From 8afac921361fcb76752c9425f0ce2e2a5708244e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:12:47 +0200 Subject: [PATCH 021/360] test(ct): try to limit QEMU architectures, maybe shaving off a few seconds of setup --- .github/workflows/container_base_push.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index cb83a2799bb..ed9e2a1cc1e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -83,7 +83,10 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} From 7c4242ea384e06b70664bffde4c5d1828ac70b1a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:18:27 +0200 Subject: [PATCH 022/360] feat(ct): add additional tags for base images --- .github/workflows/container_base_push.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ed9e2a1cc1e..21a15e02c1d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,11 +86,20 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: ${{ env.PLATFORMS }} + - name: Add additional tags as options + run: | + # For the development branch, update the latest tag in addition + if [[ "${{ matrix.branch }}" == "develop" ]]; then + echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" + elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then + echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" + fi - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From c08220dd4e40895973206d178f9cdeb7c7b5bf1c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:35:45 +0200 Subject: [PATCH 023/360] test(ct): add feature branch to matrix and build for everything but it. --- .github/workflows/container_base_push.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 21a15e02c1d..af22b6b1a55 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -32,11 +32,12 @@ jobs: steps: - name: Build branch matrix options id: matrix + # TODO: remove the feature branch and re-enable the if/else! run: | # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:3] | .[].tag_name, "develop" ]')" | tr -d "\n" | tr -s " " | \ + jq '[ .[0:3] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" @@ -87,19 +88,28 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - name: Add additional tags as options + # TODO: remove the feature branch and re-enable the if/else! run: | # For the development branch, update the latest tag in addition - if [[ "${{ matrix.branch }}" == "develop" ]]; then + if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # TODO: remove when feature branch is done + - name: Skip all but feature-branch + if: ${{ matrix.branch != '10478-version-base-img' }} + uses: actions/github-script@v3 + with: + script: | + core.setFailed('Stopping on purpose to avoid mayhem') + - name: Deploy multi-arch base container image to Docker Hub # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy -Ddocker.skip.push ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From c4cbc576ef3dba974e8bffa338ad7a74d3962960 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:56:10 +0200 Subject: [PATCH 024/360] chore,test(ct): update script action to v7 --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index af22b6b1a55..f7f8e889d56 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,7 +100,7 @@ jobs: # TODO: remove when feature branch is done - name: Skip all but feature-branch if: ${{ matrix.branch != '10478-version-base-img' }} - uses: actions/github-script@v3 + uses: actions/github-script@v7 with: script: | core.setFailed('Stopping on purpose to avoid mayhem') From 05e48875387f769fb9cc966868699cbd4edce8c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 01:56:44 +0200 Subject: [PATCH 025/360] fix(ct): add missing option for DMP to add additional tags to base image --- .github/workflows/container_base_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f7f8e889d56..cefa63786c9 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -92,10 +92,10 @@ jobs: run: | # For the development branch, update the latest tag in addition if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then - echo "DOCKER_TAGS=-Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then - echo "DOCKER_TAGS=-Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi # TODO: remove when feature branch is done - name: Skip all but feature-branch From be5b9c98cbcab64085ec8a5ffc5b8275a4fec335 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:09:53 +0200 Subject: [PATCH 026/360] feat(ct): trigger build of app image when all builds of base image succeed We use the branch name from the matrix as input argument. It's not being used yet. --- .github/workflows/container_app_push.yml | 5 +++++ .github/workflows/container_base_push.yml | 22 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index b3e247e376c..fba693eee05 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -5,6 +5,11 @@ on: # We are deliberately *not* running on push events here to avoid double runs. # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: + inputs: + branch: + type: string + description: "A tag or branch to checkout for building the image" + required: true pull_request: branches: - develop diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index cefa63786c9..4e80caa8586 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -107,6 +107,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub + id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} @@ -120,14 +121,27 @@ jobs: repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" readme-filepath: ./modules/container-base/README.md + + # - if: always() + # name: Save status (workaround for matrix outputs) + # run: | + # # steps.build.outcome is the status BEFORE continue-on-error + # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + push-app-img: name: "Rebase & Publish App Image" permissions: contents: read packages: write pull-requests: write - needs: build - # We do not release a new base image for pull requests, so do not trigger. - # if: ${{ github.event_name != 'pull_request' }} + needs: + - discover + - build + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} uses: ./.github/workflows/container_app_push.yml - secrets: inherit + with: + branch: ${{ matrix.branch }} + From d4a196b8a6247542051bf3d0daf24789121d6b94 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:12:00 +0200 Subject: [PATCH 027/360] test(ct): let's skip the build for anything but our feature branch --- .github/workflows/container_app_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index fba693eee05..a92ca4aecbd 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,7 +35,7 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' && inputs.branch == '10478-version-base-img' }} steps: - name: Checkout repository From 77aec0d62dcb0aad1e3e1bf7abdf740fc7bed540 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:19:35 +0200 Subject: [PATCH 028/360] style(ct): fix wording to test CI pipeline --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index f093ced37c1..0905ebb62a1 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -221,7 +221,7 @@ RUN < Date: Thu, 11 Apr 2024 14:22:08 +0200 Subject: [PATCH 029/360] test(ct): try what happens when removing the on purpose failure of base image workflow --- .github/workflows/container_base_push.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4e80caa8586..8c15c0a7ff7 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -97,13 +97,14 @@ jobs: elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # TODO: remove when feature branch is done - - name: Skip all but feature-branch - if: ${{ matrix.branch != '10478-version-base-img' }} - uses: actions/github-script@v7 - with: - script: | - core.setFailed('Stopping on purpose to avoid mayhem') + #- name: Skip all but feature-branch + # if: ${{ matrix.branch != '10478-version-base-img' }} + # uses: actions/github-script@v7 + # with: + # script: | + # core.setFailed('Stopping on purpose to avoid mayhem') - name: Deploy multi-arch base container image to Docker Hub From 2887d13ca93103420ad78973e4e975dd0da90b41 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 11 Apr 2024 14:42:40 +0200 Subject: [PATCH 030/360] fix(ct): make secrets available to app workflow again Deleted by accident when working on alternativ matrix way to build app image after base. --- .github/workflows/container_base_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 8c15c0a7ff7..1631c08f10e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -135,6 +135,7 @@ jobs: contents: read packages: write pull-requests: write + secrets: inherit needs: - discover - build From bbe527d4646b09fd41fa4e57c52e2759183d2042 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:44:06 +0200 Subject: [PATCH 031/360] chore(ct,build): upgrade to DMP plugin v0.44.0 - Enable -Ddocker.noCache for BuildX - Hopefully straightens out some bugs on Darwin with M1/M2 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 612902b47a4..d03d3e242fc 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.43.4 + 0.44.0 From c496ef31304f8ced339aace0b903a52d69991b71 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:45:35 +0200 Subject: [PATCH 032/360] feat(ct): pin down name of the builder for base image This way we have a distinct path and name to inject a BuildX builder instance configuration --- modules/container-base/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 2754b081986..72811e34e84 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -100,6 +100,7 @@ ${docker.platforms} + dataverse ${project.build.directory}/buildx-state Dockerfile From 82c8e725365c72bbc4a2a906779bc1c61a71ff89 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:46:05 +0200 Subject: [PATCH 033/360] ci(ct): make number of past release configurable via env var --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 1631c08f10e..fe3b38d0284 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -17,6 +17,7 @@ on: env: PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 jobs: discover: @@ -37,7 +38,7 @@ jobs: # Get last three releases and include develop branch as matrix elements #if [[ "${{ github.event_name }}" == "schedule" ]]; then echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:3] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ + jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" From 0c91541f5c11c3daead61f657dbdee6af4addc43 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:54:49 +0200 Subject: [PATCH 034/360] fix(ct): correction of tags specification for base image workflow Per example at https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-only-when-a-push-of-specific-tags-occurs it should be two asterisks. --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index fe3b38d0284..838321b10f6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -4,7 +4,7 @@ name: Base Container Image on: push: tags: - - 'v[6-9].*' + - 'v[6-9].**' branches: - 'develop' - '10478-version-base-img' From eef60f295ced438a7c3e91fbef61edc273ec51d9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 08:56:21 +0200 Subject: [PATCH 035/360] refactor(ct): when pushes for a tag or branch occur, build for it Before, we would have run the workflow for develop only. In case develop has merges before the run starts because of a tag being created, this may result in undesired side effects. Keep in mind that pushes of tags here will also trigger releasing a new application image! --- .github/workflows/container_base_push.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 838321b10f6..ec8881b3ef5 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -41,7 +41,8 @@ jobs: jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ tee -a "$GITHUB_OUTPUT" #else - # echo "branches=['develop']" | tee -a "$GITHUB_OUTPUT" + # # Note: github.ref_name will be the name of the branch or the tag pushed + # echo "branches=['${{ github.ref_name }}']" | tee -a "$GITHUB_OUTPUT" #fi build: From 90cb9ce129ff1b3b9de9e0585232418b820d7dbb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 09:59:01 +0200 Subject: [PATCH 036/360] feat(ct): replace QEMU with remote builder for ARM64 in base image --- .github/workflows/container_base_push.yml | 35 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ec8881b3ef5..324f8134a64 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -85,10 +85,39 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v3 + + # This is replaced by adding a remote ARM64 build host in addition to the local AMD64 + # - name: Set up QEMU for multi-arch builds + # uses: docker/setup-qemu-action@v3 + # with: + # platforms: ${{ env.PLATFORMS }} + # Setup SSH access to ARM64 builder node + - name: Setup SSH agent + uses: webfactory/ssh-agent@v0.9.0 with: - platforms: ${{ env.PLATFORMS }} + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF + { "Name": "dataverse", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "dataverse0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "dataverse1", + "Endpoint": "ssh://${{ secret.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF + - name: Add additional tags as options # TODO: remove the feature branch and re-enable the if/else! run: | From d7fbf871cffe8685e75533280e4ee2fa1f1add60 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:00:17 +0200 Subject: [PATCH 037/360] feat(ct): disable caching during base image build The ARM64 remote build host will otherwise reuse the cache with potentially stale content --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 324f8134a64..aaf20397f9d 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -142,7 +142,7 @@ jobs: id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} - run: mvn -f modules/container-base -Pct deploy ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} name: Push description to DockerHub From b6fb1224b9ca6fd0f345efc6ad73ebddea962450 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:02:31 +0200 Subject: [PATCH 038/360] style(ct): fix typo in secrets reference --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index aaf20397f9d..9243c2725e1 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -111,7 +111,7 @@ jobs: "Flags": ["--allow-insecure-entitlement=network.host"], "Files": null}, {"Name": "dataverse1", - "Endpoint": "ssh://${{ secret.BUILDER_ARM64_SSH_CONNECTION }}", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", "Platforms": [{"os": "linux", "architecture": "arm64"}], "DriverOpts": null, "Flags": ["--allow-insecure-entitlement=network.host"], From b3ac714f98971ef21cd47650e04a370226334fc8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 10:04:08 +0200 Subject: [PATCH 039/360] fix(ct): add missing pipe redirection for builder config --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 9243c2725e1..69cd31afcf4 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,7 +100,7 @@ jobs: run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF + cat > modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF { "Name": "dataverse", "Driver": "docker-container", "Dynamic": false, From 898d9053becaa64151c06ec645080f6544292776 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 11:06:32 +0200 Subject: [PATCH 040/360] refactor(ct): switch back to "maven" as builder name Using "maven" (the default name) again allows us to use the ARM64 builder for previously released images as well as develop before we merge the feature branch --- .github/workflows/container_base_push.yml | 8 ++++---- modules/container-base/pom.xml | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 69cd31afcf4..e4733cca027 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -100,17 +100,17 @@ jobs: run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat > modules/container-base/target/buildx-state/buildx/instances/dataverse << EOF - { "Name": "dataverse", + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", "Driver": "docker-container", "Dynamic": false, - "Nodes": [{"Name": "dataverse0", + "Nodes": [{"Name": "maven0", "Endpoint": "unix:///var/run/docker.sock", "Platforms": [{"os": "linux", "architecture": "amd64"}], "DriverOpts": null, "Flags": ["--allow-insecure-entitlement=network.host"], "Files": null}, - {"Name": "dataverse1", + {"Name": "maven1", "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", "Platforms": [{"os": "linux", "architecture": "arm64"}], "DriverOpts": null, diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 72811e34e84..2754b081986 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -100,7 +100,6 @@ ${docker.platforms} - dataverse ${project.build.directory}/buildx-state Dockerfile From 577804b3776847131b6e5f05727c9f0f69232d63 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 13:02:15 +0200 Subject: [PATCH 041/360] refactor(ct): use remote ARM64 builder only for pushes, but not scheduled maintenance --- .github/workflows/container_base_push.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index e4733cca027..36741ccd211 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,17 +86,21 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # This is replaced by adding a remote ARM64 build host in addition to the local AMD64 - # - name: Set up QEMU for multi-arch builds - # uses: docker/setup-qemu-action@v3 - # with: - # platforms: ${{ env.PLATFORMS }} - # Setup SSH access to ARM64 builder node + # In case of scheduled maintenance, we don't care about buildtime: use QEMU for AMD64 + ARM64 + - name: Set up QEMU for multi-arch builds + if: ${{ github.event_name == 'schedule' }} + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + # In case this is a push to develop, we care about buildtime. + # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} uses: webfactory/ssh-agent@v0.9.0 with: ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances From 25cbdd7b19478f9110ceb55436f9f84bcbb3aff5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 14:19:35 +0200 Subject: [PATCH 042/360] test(ct): use DMP v0.45-SNAPSHOT to see if pushing images that have a ARG in their ref works with it --- modules/dataverse-parent/pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index d03d3e242fc..473c143296a 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.44.0 + 0.45-SNAPSHOT @@ -215,6 +215,14 @@ never + + oss.sonatype.org + https://oss.sonatype.org/content/repositories/snapshots + + true + always + + From 2de674f6d80249ea2e446275533f4e3b6daa630f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:23:21 +0200 Subject: [PATCH 043/360] feat(build,ci): workflow pre-seeding Maven cache from develop #10428 Contains the feature branch for now, too. Will be deleted later before merging it to develop. --- .github/workflows/maven_cache_seed.yaml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/maven_cache_seed.yaml diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml new file mode 100644 index 00000000000..e82ce36f829 --- /dev/null +++ b/.github/workflows/maven_cache_seed.yaml @@ -0,0 +1,37 @@ +name: Maven Cache Seeding + +on: + push: + branches: + - develop + - 10478-version-base-img + +env: + COMMON_CACHE_NAME: dataverse-maven-cache + +jobs: + preseed: + name: Drop and Re-Seed Local Repository + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' }} + steps: + - name: Drop common cache + uses: prantlf/delete-cache-action@v2 + with: + key: ${{ env.COMMON_CACHE_NAME }} + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: "17" + distribution: temurin + - name: Re-Seed common cache + run: | + mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + - name: Save the cache + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: ${{ env.COMMON_CACHE_NAME }} + From b1943c114110405a613c4093fb2d7eee5fa904d7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:35:38 +0200 Subject: [PATCH 044/360] fix(build,ci): ignore not found when deleting common cache --- .github/workflows/maven_cache_seed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index e82ce36f829..4650cfcf930 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -16,6 +16,7 @@ jobs: if: ${{ github.event_name == 'push' }} steps: - name: Drop common cache + continue-on-error: true # we don't care if the cache is not around uses: prantlf/delete-cache-action@v2 with: key: ${{ env.COMMON_CACHE_NAME }} From 941fbc904d5700cebcff9d64b0a8a1f1c2268d36 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:51:33 +0200 Subject: [PATCH 045/360] style(ct): remove some stale comments --- .github/workflows/container_base_push.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 36741ccd211..7f2f194cbcd 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -123,7 +123,7 @@ jobs: EOF - name: Add additional tags as options - # TODO: remove the feature branch and re-enable the if/else! + # TODO: remove the feature branch run: | # For the development branch, update the latest tag in addition if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then @@ -133,15 +133,6 @@ jobs: echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi - # TODO: remove when feature branch is done - #- name: Skip all but feature-branch - # if: ${{ matrix.branch != '10478-version-base-img' }} - # uses: actions/github-script@v7 - # with: - # script: | - # core.setFailed('Stopping on purpose to avoid mayhem') - - - name: Deploy multi-arch base container image to Docker Hub id: build # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. From b64b6aff9ce38009fdf90c451493b257690472f7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:52:28 +0200 Subject: [PATCH 046/360] feat(ct): make use of common cache in app container builds #10428 --- .github/workflows/container_app_push.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index a92ca4aecbd..347fcf06b86 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -42,11 +42,18 @@ jobs: uses: actions/checkout@v3 - name: Set up JDK + id: setup-java uses: actions/setup-java@v3 with: java-version: "17" distribution: temurin cache: maven + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -111,11 +118,19 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - name: Set up JDK + id: setup-java + uses: actions/setup-java@v3 with: java-version: "17" distribution: temurin + cache: maven + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' From 4f48123f73674eb56f3f687cc2e7df55908e13db Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 15:52:49 +0200 Subject: [PATCH 047/360] feat(ci): remove feature branch from maven seed workflow --- .github/workflows/maven_cache_seed.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 4650cfcf930..d002a446b99 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,7 +4,6 @@ on: push: branches: - develop - - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From 721814a62672a8483d1fd0708071444bc239e806 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:01:47 +0200 Subject: [PATCH 048/360] style(ct): rename base image revision from RX to revX It's just much more readable --- modules/container-base/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 2754b081986..e5de8841530 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -42,7 +42,7 @@ docker-build gdcc/base:${base.image.tag} - R${project.version} + rev${project.version} eclipse-temurin:${target.java.version}-jre 1000 1000 From 3c0650f6710bcda94c0e83278654a454466b5a41 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:02:12 +0200 Subject: [PATCH 049/360] feat(ct): make dataverse app image use base rev1 image --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f9d06b8744..fa267b6fbee 100644 --- a/pom.xml +++ b/pom.xml @@ -956,7 +956,8 @@ unstable false gdcc/base:${base.image.tag} - unstable + + rev1 gdcc/configbaker:${conf.image.tag} ${app.image.tag} From 06a21612347e0a64cfe0adfa05d1823c4927f5b2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:15:00 +0200 Subject: [PATCH 050/360] refactor(ci): delete common cache after we downloaded the old one #10428 --- .github/workflows/maven_cache_seed.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index d002a446b99..5a38abed767 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -14,11 +14,6 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' }} steps: - - name: Drop common cache - continue-on-error: true # we don't care if the cache is not around - uses: prantlf/delete-cache-action@v2 - with: - key: ${{ env.COMMON_CACHE_NAME }} - name: Checkout repository uses: actions/checkout@v4 - name: Set up JDK @@ -26,9 +21,16 @@ jobs: with: java-version: "17" distribution: temurin - - name: Re-Seed common cache + - name: Seed common cache run: | mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + # This non-obvious order is due to the fact that the download via Maven will take a very long time. + # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. + - name: Drop common cache + continue-on-error: true # we don't care if the cache is not around + uses: prantlf/delete-cache-action@v2 + with: + key: ${{ env.COMMON_CACHE_NAME }} - name: Save the cache uses: actions/cache/save@v4 with: From 818bfd4ec32fa0d67f39611f6f3235af730fbfb9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:15:20 +0200 Subject: [PATCH 051/360] test(ci): readd feature branch to test cache handling logic --- .github/workflows/maven_cache_seed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 5a38abed767..f4b13725e70 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,6 +4,7 @@ on: push: branches: - develop + - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From a28997c815685b0527e3250c7526d9a03521ddae Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:16:33 +0200 Subject: [PATCH 052/360] style(ct): add common cache download to base image Without accurate measuring it seems like we use ~1 minute to download Maven artifacts for this workflow. Lets try to cut that down some by reusing the common cache. --- .github/workflows/container_base_push.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 7f2f194cbcd..667808d4d08 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -69,6 +69,7 @@ jobs: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java uses: actions/setup-java@v4 with: java-version: ${{ env.JAVA_VERSION }} @@ -76,6 +77,12 @@ jobs: cache: 'maven' cache-dependency-path: | modules/container-base/pom.xml + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! From 085e41e9c1bd7c550e76ad47b5450ac5fe197450 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:19:59 +0200 Subject: [PATCH 053/360] style(ct): remove stale comment about platforms empty in base image --- modules/container-base/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index e5de8841530..60be420bd9a 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -97,7 +97,6 @@ - ${docker.platforms} ${project.build.directory}/buildx-state From 2fe0ca39ac5c9cf044ad833b7e66de449d61efa3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 16:52:12 +0200 Subject: [PATCH 054/360] fix(build,ci): downgrade to cache action v2 Per https://github.com/actions/cache/issues/1361 restores in v3 and v4 were not working, maybe saving is botched, too? --- .github/workflows/maven_cache_seed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index f4b13725e70..148808a4b04 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -33,7 +33,7 @@ jobs: with: key: ${{ env.COMMON_CACHE_NAME }} - name: Save the cache - uses: actions/cache/save@v4 + uses: actions/cache@v2 with: path: ~/.m2/repository key: ${{ env.COMMON_CACHE_NAME }} From afad97e4160d60e31b01611e8a1e28ad0192631f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:09:50 +0200 Subject: [PATCH 055/360] fix(build,ci,ct): fix missing path for cache restore --- .github/workflows/container_app_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 347fcf06b86..3344a8e2d0d 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -52,6 +52,7 @@ jobs: if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} uses: actions/cache/restore@v4 with: + path: ~/.m2/repository key: dataverse-maven-cache @@ -130,6 +131,7 @@ jobs: if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} uses: actions/cache/restore@v4 with: + path: ~/.m2/repository key: dataverse-maven-cache # Depending on context, we push to different targets. Login accordingly. From 1c1c30e8b1d11f830354535f5164a92112eb3ca3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:11:51 +0200 Subject: [PATCH 056/360] test(ci): fix a comment to trigger base image workflow --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 0905ebb62a1..a8bd3a32ca3 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -203,7 +203,7 @@ RUN < Date: Fri, 12 Apr 2024 17:17:49 +0200 Subject: [PATCH 057/360] style(ci): remove feature branch from maven cache seeding again --- .github/workflows/maven_cache_seed.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 148808a4b04..85f1381c789 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,7 +4,6 @@ on: push: branches: - develop - - 10478-version-base-img env: COMMON_CACHE_NAME: dataverse-maven-cache From 43402435ae4d619b872ac9bd14704f54306fc49e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:20:08 +0200 Subject: [PATCH 058/360] doc(ci): add note about cache availability to seeding workflow --- .github/workflows/maven_cache_seed.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index 85f1381c789..e11d8c91c0e 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -31,6 +31,8 @@ jobs: uses: prantlf/delete-cache-action@v2 with: key: ${{ env.COMMON_CACHE_NAME }} + # NOTE: It is vital here to remember that only caches with the scope of the default branch are + # available to other branches. We use the v2 action here to save it anyway. - name: Save the cache uses: actions/cache@v2 with: From 899e8aa6548d8b6076c266807cc265d4c0bb049b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 12 Apr 2024 17:31:16 +0200 Subject: [PATCH 059/360] feat(ci): make sure to rejuvenate the common cache every 7 days --- .github/workflows/maven_cache_seed.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml index e11d8c91c0e..d31559138b8 100644 --- a/.github/workflows/maven_cache_seed.yaml +++ b/.github/workflows/maven_cache_seed.yaml @@ -4,6 +4,10 @@ on: push: branches: - develop + # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # all caches are depleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. + schedule: + - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC env: COMMON_CACHE_NAME: dataverse-maven-cache @@ -12,7 +16,6 @@ jobs: preseed: name: Drop and Re-Seed Local Repository runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' }} steps: - name: Checkout repository uses: actions/checkout@v4 From f538896971022cc4fa011a38aca41936125a30dc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:14:56 +0200 Subject: [PATCH 060/360] chore(ct): replace wait-for with wait4x Aligning configbaker and base image with same tool. wait4x has much more features to wait for different services. --- doc/sphinx-guides/source/container/base-image.rst | 2 +- modules/container-base/src/main/docker/Dockerfile | 12 ++++++------ .../docker/scripts/init_3_wait_dataverse_db_host.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index c41250d48c5..29c357b91f6 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -46,7 +46,7 @@ The base image provides: - CLI tools necessary to run Dataverse (i. e. ``curl`` or ``jq`` - see also :doc:`../installation/prerequisites` in Installation Guide) - Linux tools for analysis, monitoring and so on - `Jattach `__ (attach to running JVM) -- `wait-for `__ (tool to "wait for" a service to be available) +- `wait4x `__ (tool to "wait for" a service to be available) - `dumb-init `__ (see :ref:`below ` for details) This image is created as a "multi-arch image", see :ref:`below `. diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index a8bd3a32ca3..e3fa6e477a9 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -92,8 +92,7 @@ EOF ARG JATTACH_VERSION="v2.1" ARG JATTACH_CHECKSUM="07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e" -ARG WAIT_FOR_VERSION="v2.2.3" -ARG WAIT_FOR_CHECKSUM="70271181be69cd2c7265b2746f97fccfd7e8aa1059894138a775369c23589ff4" +ARG WAIT4X_VERSION="v2.14.0" ARG PKGS="jq imagemagick curl unzip wget acl dirmngr gpg lsof procps netcat dumb-init" # Installing the packages in an extra container layer for better caching @@ -108,10 +107,11 @@ RUN < Date: Mon, 15 Apr 2024 14:17:55 +0200 Subject: [PATCH 061/360] build(ct): make target architecture available in base image build As per https://docs.docker.com/reference/dockerfile/#automatic-platform-args-in-the-global-scope BuildKit / buildx will expose the target architecture. It requires adding an ARG in the Dockerfile to inject the data. --- modules/container-base/src/main/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index e3fa6e477a9..256d9159d2d 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -69,6 +69,9 @@ ENV PATH="${PATH}:${PAYARA_DIR}/bin:${SCRIPT_DIR}" \ ### PART 1: SYSTEM ### ARG UID=1000 ARG GID=1000 +# Auto-populated by BuildKit / buildx +#ARG TARGETARCH="amd64" +ARG TARGETARCH USER root WORKDIR / SHELL ["/bin/bash", "-euo", "pipefail", "-c"] From e86133468a0086f0701efc69ad1549e21102f284 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 15 Apr 2024 14:20:18 +0200 Subject: [PATCH 062/360] chore(ct): upgrade base image with jattach v2.2 jattach binary is now available for ARM64 and AMD64, but requires special handling with download URLs and checksums. --- .../container-base/src/main/docker/Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 256d9159d2d..fbd43604eac 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -93,8 +93,9 @@ RUN < Date: Tue, 16 Apr 2024 17:42:18 +0200 Subject: [PATCH 063/360] chore: remove obsolete Maven Cache seed workflow --- .github/workflows/maven_cache_seed.yaml | 44 ------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/maven_cache_seed.yaml diff --git a/.github/workflows/maven_cache_seed.yaml b/.github/workflows/maven_cache_seed.yaml deleted file mode 100644 index d31559138b8..00000000000 --- a/.github/workflows/maven_cache_seed.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Maven Cache Seeding - -on: - push: - branches: - - develop - # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy - # all caches are depleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. - schedule: - - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC - -env: - COMMON_CACHE_NAME: dataverse-maven-cache - -jobs: - preseed: - name: Drop and Re-Seed Local Repository - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up JDK - uses: actions/setup-java@v4 - with: - java-version: "17" - distribution: temurin - - name: Seed common cache - run: | - mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins - # This non-obvious order is due to the fact that the download via Maven will take a very long time. - # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. - - name: Drop common cache - continue-on-error: true # we don't care if the cache is not around - uses: prantlf/delete-cache-action@v2 - with: - key: ${{ env.COMMON_CACHE_NAME }} - # NOTE: It is vital here to remember that only caches with the scope of the default branch are - # available to other branches. We use the v2 action here to save it anyway. - - name: Save the cache - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ env.COMMON_CACHE_NAME }} - From 54fe365964f8c726ec2a3e42bee5b28b41f952d0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 16 Apr 2024 17:43:42 +0200 Subject: [PATCH 064/360] style(ci): remove superfluous empty line --- .github/workflows/container_base_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 667808d4d08..660f9bdd861 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -86,7 +86,6 @@ jobs: # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! - - name: Log in to the Container registry uses: docker/login-action@v3 with: From 4878cfe47a284f029b2d98adb64d02dafdb540b6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:31:06 -0400 Subject: [PATCH 065/360] separate metadata parsing/params from XML generation code --- .../pidproviders/doi/AbstractDOIProvider.java | 25 ++-- .../pidproviders/doi/DoiMetadata.java | 138 ++++++++++++++++++ .../datacite/DOIDataCiteRegisterService.java | 57 ++++---- 3 files changed, 180 insertions(+), 40 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/DoiMetadata.java diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java index 43e34e74c59..02a7dedce47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java @@ -91,31 +91,30 @@ public String getMetadataFromDvObject(String identifier, Map met } else { dataset = (Dataset) dvObject.getOwner(); } - - XmlMetadataTemplate metadataTemplate = new XmlMetadataTemplate(); - metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); - metadataTemplate.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); - metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); + DoiMetadata doiMetadata = new DoiMetadata(); + doiMetadata.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + doiMetadata.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); + doiMetadata.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); if (dvObject.isInstanceofDataset()) { - metadataTemplate.setDescription(dataset.getLatestVersion().getDescriptionPlainText()); + doiMetadata.setDescription(dataset.getLatestVersion().getDescriptionPlainText()); } if (dvObject.isInstanceofDataFile()) { DataFile df = (DataFile) dvObject; String fileDescription = df.getDescription(); - metadataTemplate.setDescription(fileDescription == null ? "" : fileDescription); + doiMetadata.setDescription(fileDescription == null ? "" : fileDescription); } - metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); - metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); - metadataTemplate.setTitle(dvObject.getCurrentName()); + doiMetadata.setContacts(dataset.getLatestVersion().getDatasetContacts()); + doiMetadata.setProducers(dataset.getLatestVersion().getDatasetProducers()); + doiMetadata.setTitle(dvObject.getCurrentName()); String producerString = pidProviderService.getProducer(); if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { producerString = UNAVAILABLE; } - metadataTemplate.setPublisher(producerString); - metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); + doiMetadata.setPublisher(producerString); + doiMetadata.setPublisherYear(metadata.get("datacite.publicationyear")); - String xmlMetadata = metadataTemplate.generateXML(dvObject); + String xmlMetadata = new XmlMetadataTemplate(doiMetadata).generateXML(dvObject); logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); return xmlMetadata; } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/DoiMetadata.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/DoiMetadata.java new file mode 100644 index 00000000000..ffd24747bc2 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/DoiMetadata.java @@ -0,0 +1,138 @@ +package edu.harvard.iq.dataverse.pidproviders.doi; + +import java.util.ArrayList; +import java.util.List; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import edu.harvard.iq.dataverse.DatasetAuthor; + + +//Parses some specific parts of a DataCite XML metadata file +public class DoiMetadata { + + private String identifier; + private List creators; + private String title; + private String publisher; + private String publisherYear; + private List datafileIdentifiers; + private List authors; + private String description; + private List contacts; + private List producers; + + + public DoiMetadata() { + } + + public void parseDataCiteXML(String xmlMetaData) { + Document doc = Jsoup.parseBodyFragment(xmlMetaData); + Elements identifierElements = doc.select("identifier"); + if (identifierElements.size() > 0) { + identifier = identifierElements.get(0).html(); + } + Elements creatorElements = doc.select("creatorName"); + creators = new ArrayList<>(); + for (Element creatorElement : creatorElements) { + creators.add(creatorElement.html()); + } + Elements titleElements = doc.select("title"); + if (titleElements.size() > 0) { + title = titleElements.get(0).html(); + } + Elements publisherElements = doc.select("publisher"); + if (publisherElements.size() > 0) { + publisher = publisherElements.get(0).html(); + } + Elements publisherYearElements = doc.select("publicationYear"); + if (publisherYearElements.size() > 0) { + publisherYear = publisherYearElements.get(0).html(); + } + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public List getCreators() { + return creators; + } + + public void setCreators(List creators) { + this.creators = creators; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public String getPublisherYear() { + return publisherYear; + } + + public void setPublisherYear(String publisherYear) { + this.publisherYear = publisherYear; + } + + + public List getProducers() { + return producers; + } + + public void setProducers(List producers) { + this.producers = producers; + } + + public List getContacts() { + return contacts; + } + + public void setContacts(List contacts) { + this.contacts = contacts; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getAuthors() { + return authors; + } + + public void setAuthors(List authors) { + this.authors = authors; + } + + + public List getDatafileIdentifiers() { + return datafileIdentifiers; + } + + public void setDatafileIdentifiers(List datafileIdentifiers) { + this.datafileIdentifiers = datafileIdentifiers; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java index 0e322eace05..bc69275ac1d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java @@ -21,6 +21,7 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.DoiMetadata; import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; /** @@ -90,28 +91,28 @@ public static String getMetadataFromDvObject(String identifier, Map from HTML, it leaves '&' (at least so we need to xml escape as well String description = StringEscapeUtils.escapeXml10(dataset.getLatestVersion().getDescriptionPlainText()); if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { description = AbstractPidProvider.UNAVAILABLE; } - metadataTemplate.setDescription(description); + doiMetadata.setDescription(description); } if (dvObject.isInstanceofDataFile()) { DataFile df = (DataFile) dvObject; //Note: File metadata is not escaped like dataset metadata is, so adding an xml escape here. //This could/should be removed if the datafile methods add escaping String fileDescription = StringEscapeUtils.escapeXml10(df.getDescription()); - metadataTemplate.setDescription(fileDescription == null ? AbstractPidProvider.UNAVAILABLE : fileDescription); + doiMetadata.setDescription(fileDescription == null ? AbstractPidProvider.UNAVAILABLE : fileDescription); } - metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); - metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); + doiMetadata.setContacts(dataset.getLatestVersion().getDatasetContacts()); + doiMetadata.setProducers(dataset.getLatestVersion().getDatasetProducers()); String title = dvObject.getCurrentName(); if(dvObject.isInstanceofDataFile()) { //Note file title is not currently escaped the way the dataset title is, so adding it here. @@ -122,40 +123,41 @@ public static String getMetadataFromDvObject(String identifier, Map metadata, DvObject dvObject) { - XmlMetadataTemplate metadataTemplate = new XmlMetadataTemplate(); - metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); - metadataTemplate.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); + DoiMetadata doiMetadata = new DoiMetadata(); + + doiMetadata.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + doiMetadata.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); - metadataTemplate.setDescription(AbstractPidProvider.UNAVAILABLE); + doiMetadata.setDescription(AbstractPidProvider.UNAVAILABLE); String title =metadata.get("datacite.title"); System.out.print("Map metadata title: "+ metadata.get("datacite.title")); - metadataTemplate.setAuthors(null); + doiMetadata.setAuthors(null); - metadataTemplate.setTitle(title); + doiMetadata.setTitle(title); String producerString = AbstractPidProvider.UNAVAILABLE; - metadataTemplate.setPublisher(producerString); - metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); + doiMetadata.setPublisher(producerString); + doiMetadata.setPublisherYear(metadata.get("datacite.publicationyear")); - String xmlMetadata = metadataTemplate.generateXML(dvObject); + String xmlMetadata = new XmlMetadataTemplate(doiMetadata).generateXML(dvObject); logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); return xmlMetadata; } @@ -209,11 +211,12 @@ Map getMetadata(String identifier) throws IOException { Map metadata = new HashMap<>(); try { String xmlMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1)); - XmlMetadataTemplate template = new XmlMetadataTemplate(xmlMetadata); - metadata.put("datacite.creator", String.join("; ", template.getCreators())); - metadata.put("datacite.title", template.getTitle()); - metadata.put("datacite.publisher", template.getPublisher()); - metadata.put("datacite.publicationyear", template.getPublisherYear()); + DoiMetadata doiMetadata = new DoiMetadata(); + doiMetadata.parseDataCiteXML(xmlMetadata); + metadata.put("datacite.creator", String.join("; ", doiMetadata.getCreators())); + metadata.put("datacite.title", doiMetadata.getTitle()); + metadata.put("datacite.publisher", doiMetadata.getPublisher()); + metadata.put("datacite.publicationyear", doiMetadata.getPublisherYear()); } catch (RuntimeException e) { logger.log(Level.INFO, identifier, e); } From 68792c2f92c90f716f39caaa5f76b652592186c0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:31:37 -0400 Subject: [PATCH 066/360] extract some common xml writing util code --- .../dataverse/export/ddi/DdiExportUtil.java | 486 ++++++------------ .../iq/dataverse/util/xml/XmlWriterUtil.java | 174 +++++++ 2 files changed, 340 insertions(+), 320 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9a689f7a4ed..0c861cb6c09 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -24,6 +24,8 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; +import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -111,9 +113,9 @@ private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) thr xmlw.writeDefaultNamespace("ddi:codebook:2_5"); xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION); - writeAttribute(xmlw, "version", DDIExporter.DEFAULT_XML_VERSION); + xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION); if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { - writeAttribute(xmlw, "xml:lang", datasetDto.getMetadataLanguage()); + xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles()); @@ -133,9 +135,9 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDe xmlw.writeDefaultNamespace("ddi:codebook:2_5"); xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION); - writeAttribute(xmlw, "version", DDIExporter.DEFAULT_XML_VERSION); + xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION); if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) { - writeAttribute(xmlw, "xml:lang", datasetDto.getMetadataLanguage()); + xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage()); } createStdyDscr(xmlw, datasetDto); createFileDscr(xmlw, fileDetails); @@ -186,15 +188,15 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); - writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); - writeFullElement(xmlw, "subTitl", dto2Primitive(version, DatasetFieldConstant.subTitle)); + XmlWriterUtil.writeFullElement(xmlw, "titl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); + XmlWriterUtil.writeFullElement(xmlw, "subTitl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.subTitle)); FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation" ); if (altField != null) { writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage()); } xmlw.writeStartElement("IDNo"); - writeAttribute(xmlw, "agency", persistentAgency); + XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); xmlw.writeCharacters(pid); @@ -218,23 +220,23 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) boolean excludeRepository = settingsService.isTrueForKey(SettingsServiceBean.Key.ExportInstallationAsDistributorOnlyWhenNotSet, false); if (!StringUtils.isEmpty(datasetDto.getPublisher()) && !(excludeRepository && distributorSet)) { xmlw.writeStartElement("distrbtr"); - writeAttribute(xmlw, "source", "archive"); + xmlw.writeAttribute("source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); xmlw.writeEndElement(); //distrbtr } writeDistributorsElement(xmlw, version, datasetDto.getMetadataLanguage()); writeContactsElement(xmlw, version); /* per SCHEMA, depositr comes before depDate! - L.A. */ - writeFullElement(xmlw, "depositr", dto2Primitive(version, DatasetFieldConstant.depositor)); + XmlWriterUtil.writeFullElement(xmlw, "depositr", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.depositor)); /* ... and depDate comes before distDate - L.A. */ - writeFullElement(xmlw, "depDate", dto2Primitive(version, DatasetFieldConstant.dateOfDeposit)); - writeFullElement(xmlw, "distDate", dto2Primitive(version, DatasetFieldConstant.distributionDate)); + XmlWriterUtil.writeFullElement(xmlw, "depDate", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.dateOfDeposit)); + XmlWriterUtil.writeFullElement(xmlw, "distDate", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.distributionDate)); xmlw.writeEndElement(); // diststmt writeSeriesElement(xmlw, version); xmlw.writeStartElement("holdings"); - writeAttribute(xmlw, "URI", pidUri); + XmlWriterUtil.writeAttribute(xmlw, "URI", pidUri); xmlw.writeEndElement(); //holdings xmlw.writeEndElement(); // citation @@ -247,7 +249,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeSubjectElement(xmlw, version, datasetDto.getMetadataLanguage()); //Subject and Keywords writeAbstractElement(xmlw, version, datasetDto.getMetadataLanguage()); // Description writeSummaryDescriptionElement(xmlw, version, datasetDto.getMetadataLanguage()); - writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.notesText)); + XmlWriterUtil.writeFullElement(xmlw, "notes", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.notesText)); //////// xmlw.writeEndElement(); // stdyInfo @@ -255,7 +257,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeDataAccess(xmlw , version); writeOtherStudyMaterial(xmlw , version); - writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); + XmlWriterUtil.writeFullElement(xmlw, "notes", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); xmlw.writeEndElement(); // stdyDscr @@ -274,10 +276,10 @@ private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersio return; } xmlw.writeStartElement("othrStdyMat"); - writeFullElementList(xmlw, "relMat", relMaterials); - writeFullElementList(xmlw, "relStdy", relDatasets); + XmlWriterUtil.writeFullElementList(xmlw, "relMat", relMaterials); + XmlWriterUtil.writeFullElementList(xmlw, "relStdy", relDatasets); writeRelPublElement(xmlw, version); - writeFullElementList(xmlw, "othRefs", relReferences); + XmlWriterUtil.writeFullElementList(xmlw, "othRefs", relReferences); xmlw.writeEndElement(); //othrStdyMat } @@ -292,29 +294,29 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver xmlw.writeStartElement("dataAccs"); xmlw.writeStartElement("setAvail"); - writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); - writeFullElement(xmlw, "origArch", version.getOriginalArchive()); - writeFullElement(xmlw, "avlStatus", version.getAvailabilityStatus()); - writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); - writeFullElement(xmlw, "complete", version.getStudyCompletion()); + XmlWriterUtil.writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); + XmlWriterUtil.writeFullElement(xmlw, "origArch", version.getOriginalArchive()); + XmlWriterUtil.writeFullElement(xmlw, "avlStatus", version.getAvailabilityStatus()); + XmlWriterUtil.writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); + XmlWriterUtil.writeFullElement(xmlw, "complete", version.getStudyCompletion()); xmlw.writeEndElement(); //setAvail xmlw.writeStartElement("useStmt"); - writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); - writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); - writeFullElement(xmlw, "restrctn", version.getRestrictions()); - writeFullElement(xmlw, "contact", version.getContactForAccess()); - writeFullElement(xmlw, "citReq", version.getCitationRequirements()); - writeFullElement(xmlw, "deposReq", version.getDepositorRequirements()); - writeFullElement(xmlw, "conditions", version.getConditions()); - writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); + XmlWriterUtil.writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); + XmlWriterUtil.writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); + XmlWriterUtil.writeFullElement(xmlw, "restrctn", version.getRestrictions()); + XmlWriterUtil.writeFullElement(xmlw, "contact", version.getContactForAccess()); + XmlWriterUtil.writeFullElement(xmlw, "citReq", version.getCitationRequirements()); + XmlWriterUtil.writeFullElement(xmlw, "deposReq", version.getDepositorRequirements()); + XmlWriterUtil.writeFullElement(xmlw, "conditions", version.getConditions()); + XmlWriterUtil.writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); xmlw.writeEndElement(); //useStmt /* any s: */ if (version.getTermsOfAccess() != null && !version.getTermsOfAccess().trim().equals("")) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_ACCESS); - writeAttribute(xmlw, "level", LEVEL_DV); + xmlw.writeAttribute("type", NOTE_TYPE_TERMS_OF_ACCESS); + xmlw.writeAttribute("level", LEVEL_DV); xmlw.writeCharacters(version.getTermsOfAccess()); xmlw.writeEndElement(); //notes } @@ -341,9 +343,9 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeStartElement("docDscr"); xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); - writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); + XmlWriterUtil.writeFullElement(xmlw, "titl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); xmlw.writeStartElement("IDNo"); - writeAttribute(xmlw, "agency", persistentAgency); + XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt @@ -351,11 +353,11 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase //The doc is always published by the Dataverse Repository if (!StringUtils.isEmpty(datasetDto.getPublisher())) { xmlw.writeStartElement("distrbtr"); - writeAttribute(xmlw, "source", "archive"); + xmlw.writeAttribute("source", "archive"); xmlw.writeCharacters(datasetDto.getPublisher()); xmlw.writeEndElement(); // distrbtr } - writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); + XmlWriterUtil.writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); xmlw.writeEndElement(); // diststmt writeVersionStatement(xmlw, version); @@ -369,10 +371,10 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ xmlw.writeStartElement("verStmt"); - writeAttribute(xmlw,"source","archive"); + XmlWriterUtil.writeAttribute(xmlw,"source","archive"); xmlw.writeStartElement("version"); - writeAttribute(xmlw,"date", datasetVersionDTO.getReleaseTime().substring(0, 10)); - writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); + XmlWriterUtil.writeAttribute(xmlw,"date", datasetVersionDTO.getReleaseTime().substring(0, 10)); + XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); xmlw.writeCharacters(datasetVersionDTO.getVersionNumber().toString()); xmlw.writeEndElement(); // version xmlw.writeEndElement(); // verStmt @@ -523,14 +525,14 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset * "" entries, then all the "" ones: */ for (String nationEntry : nationList) { - writeFullElement(xmlw, "nation", nationEntry); + XmlWriterUtil.writeFullElement(xmlw, "nation", nationEntry); } for (String geogCoverEntry : geogCoverList) { - writeFullElement(xmlw, "geogCover", geogCoverEntry); + XmlWriterUtil.writeFullElement(xmlw, "geogCover", geogCoverEntry); } } - writeFullElementList(xmlw, "geogUnit", dto2PrimitiveList(datasetVersionDTO, DatasetFieldConstant.geographicUnit)); + XmlWriterUtil.writeFullElementList(xmlw, "geogUnit", dto2PrimitiveList(datasetVersionDTO, DatasetFieldConstant.geographicUnit)); /* Only 1 geoBndBox is allowed in the DDI. So, I'm just going to arbitrarily use the first one, and ignore the rest! -L.A. */ @@ -563,16 +565,16 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset */ if (geoBndBoxMap.get("westBL") != null) { - writeFullElement(xmlw, "westBL", geoBndBoxMap.get("westBL")); + XmlWriterUtil.writeFullElement(xmlw, "westBL", geoBndBoxMap.get("westBL")); } if (geoBndBoxMap.get("eastBL") != null) { - writeFullElement(xmlw, "eastBL", geoBndBoxMap.get("eastBL")); + XmlWriterUtil.writeFullElement(xmlw, "eastBL", geoBndBoxMap.get("eastBL")); } if (geoBndBoxMap.get("southBL") != null) { - writeFullElement(xmlw, "southBL", geoBndBoxMap.get("southBL")); + XmlWriterUtil.writeFullElement(xmlw, "southBL", geoBndBoxMap.get("southBL")); } if (geoBndBoxMap.get("northBL") != null) { - writeFullElement(xmlw, "northBL", geoBndBoxMap.get("northBL")); + XmlWriterUtil.writeFullElement(xmlw, "northBL", geoBndBoxMap.get("northBL")); } xmlw.writeEndElement(); @@ -580,7 +582,7 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset /* analyUnit: */ if (unitOfAnalysisDTO != null) { - writeI18NElementList(xmlw, "anlyUnit", unitOfAnalysisDTO.getMultipleVocab(), "unitOfAnalysis", unitOfAnalysisDTO.getTypeClass(), "socialscience", lang); + XmlWriterUtil.writeI18NElementList(xmlw, "anlyUnit", unitOfAnalysisDTO.getMultipleVocab(), "unitOfAnalysis", unitOfAnalysisDTO.getTypeClass(), "socialscience", lang); } @@ -600,16 +602,16 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset private static void writeMultipleElement(XMLStreamWriter xmlw, String element, FieldDTO fieldDTO, String lang) throws XMLStreamException { for (String value : fieldDTO.getMultiplePrimitive()) { //Write multiple lang vals for controlled vocab, otherwise don't include any lang tag - writeFullElement(xmlw, element, value, fieldDTO.isControlledVocabularyField() ? lang : null); + XmlWriterUtil.writeFullElement(xmlw, element, value, fieldDTO.isControlledVocabularyField() ? lang : null); } } private static void writeDateElement(XMLStreamWriter xmlw, String element, String cycle, String event, String dateIn) throws XMLStreamException { xmlw.writeStartElement(element); - writeAttribute(xmlw, "cycle", cycle); - writeAttribute(xmlw, "event", event); - writeAttribute(xmlw, "date", dateIn); + XmlWriterUtil.writeAttribute(xmlw, "cycle", cycle); + XmlWriterUtil.writeAttribute(xmlw, "event", event); + XmlWriterUtil.writeAttribute(xmlw, "date", dateIn); xmlw.writeCharacters(dateIn); xmlw.writeEndElement(); @@ -641,15 +643,15 @@ private static void writeDateElement(XMLStreamWriter xmlw, String element, Strin private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO version, String lang) throws XMLStreamException{ xmlw.writeStartElement("method"); xmlw.writeStartElement("dataColl"); - writeI18NElement(xmlw, "timeMeth", version, DatasetFieldConstant.timeMethod,lang); - writeI18NElement(xmlw, "dataCollector", version, DatasetFieldConstant.dataCollector, lang); - writeI18NElement(xmlw, "collectorTraining", version, DatasetFieldConstant.collectorTraining, lang); - writeI18NElement(xmlw, "frequenc", version, DatasetFieldConstant.frequencyOfDataCollection, lang); - writeI18NElement(xmlw, "sampProc", version, DatasetFieldConstant.samplingProcedure, lang); + XmlWriterUtil.writeI18NElement(xmlw, "timeMeth", version, DatasetFieldConstant.timeMethod,lang); + XmlWriterUtil.writeI18NElement(xmlw, "dataCollector", version, DatasetFieldConstant.dataCollector, lang); + XmlWriterUtil.writeI18NElement(xmlw, "collectorTraining", version, DatasetFieldConstant.collectorTraining, lang); + XmlWriterUtil.writeI18NElement(xmlw, "frequenc", version, DatasetFieldConstant.frequencyOfDataCollection, lang); + XmlWriterUtil.writeI18NElement(xmlw, "sampProc", version, DatasetFieldConstant.samplingProcedure, lang); writeTargetSampleElement(xmlw, version); - writeI18NElement(xmlw, "deviat", version, DatasetFieldConstant.deviationsFromSampleDesign, lang); + XmlWriterUtil.writeI18NElement(xmlw, "deviat", version, DatasetFieldConstant.deviationsFromSampleDesign, lang); /* comes before : */ FieldDTO collModeFieldDTO = dto2FieldDTO(version, DatasetFieldConstant.collectionMode, "socialscience"); @@ -658,37 +660,37 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO // Below is a backward compatibility check allowing export to work in // an instance where the metadata block has not been updated yet. if (collModeFieldDTO.getMultiple()) { - writeI18NElementList(xmlw, "collMode", collModeFieldDTO.getMultipleVocab(), DatasetFieldConstant.collectionMode, collModeFieldDTO.getTypeClass(), "socialscience", lang); + XmlWriterUtil.writeI18NElementList(xmlw, "collMode", collModeFieldDTO.getMultipleVocab(), DatasetFieldConstant.collectionMode, collModeFieldDTO.getTypeClass(), "socialscience", lang); } else { - writeI18NElement(xmlw, "collMode", version, DatasetFieldConstant.collectionMode, lang); + XmlWriterUtil.writeI18NElement(xmlw, "collMode", version, DatasetFieldConstant.collectionMode, lang); } } /* and so does : */ - writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); + XmlWriterUtil.writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); xmlw.writeStartElement("sources"); - writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); - writeI18NElement(xmlw, "srcOrig", version, DatasetFieldConstant.originOfSources, lang); - writeI18NElement(xmlw, "srcChar", version, DatasetFieldConstant.characteristicOfSources, lang); - writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang); + XmlWriterUtil.writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); + XmlWriterUtil.writeI18NElement(xmlw, "srcOrig", version, DatasetFieldConstant.originOfSources, lang); + XmlWriterUtil.writeI18NElement(xmlw, "srcChar", version, DatasetFieldConstant.characteristicOfSources, lang); + XmlWriterUtil.writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang); xmlw.writeEndElement(); //sources - writeI18NElement(xmlw, "collSitu", version, DatasetFieldConstant.dataCollectionSituation, lang); - writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang); + XmlWriterUtil.writeI18NElement(xmlw, "collSitu", version, DatasetFieldConstant.dataCollectionSituation, lang); + XmlWriterUtil.writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang); /* "" has the uppercase C: */ - writeI18NElement(xmlw, "ConOps", version, DatasetFieldConstant.controlOperations, lang); - writeI18NElement(xmlw, "weight", version, DatasetFieldConstant.weighting, lang); - writeI18NElement(xmlw, "cleanOps", version, DatasetFieldConstant.cleaningOperations, lang); + XmlWriterUtil.writeI18NElement(xmlw, "ConOps", version, DatasetFieldConstant.controlOperations, lang); + XmlWriterUtil.writeI18NElement(xmlw, "weight", version, DatasetFieldConstant.weighting, lang); + XmlWriterUtil.writeI18NElement(xmlw, "cleanOps", version, DatasetFieldConstant.cleaningOperations, lang); xmlw.writeEndElement(); //dataColl /* before : */ writeNotesElement(xmlw, version); xmlw.writeStartElement("anlyInfo"); - //writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); - writeI18NElement(xmlw, "respRate", version, DatasetFieldConstant.responseRate, lang); - writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang); - writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); + //XmlWriterUtil.writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); + XmlWriterUtil.writeI18NElement(xmlw, "respRate", version, DatasetFieldConstant.responseRate, lang); + XmlWriterUtil.writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang); + XmlWriterUtil.writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); xmlw.writeEndElement(); //anlyInfo xmlw.writeEndElement();//method @@ -705,7 +707,7 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO if (CITATION_BLOCK_NAME.equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.subject.equals(fieldDTO.getTypeName())) { - writeI18NElementList(xmlw, "keyword", fieldDTO.getMultipleVocab(), "subject", + XmlWriterUtil.writeI18NElementList(xmlw, "keyword", fieldDTO.getMultipleVocab(), "subject", fieldDTO.getTypeClass(), "citation", lang); } @@ -732,14 +734,10 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!keywordValue.isEmpty()) { xmlw.writeStartElement("keyword"); - if (!keywordVocab.isEmpty()) { - writeAttribute(xmlw, "vocab", keywordVocab); - } - if (!keywordURI.isEmpty()) { - writeAttribute(xmlw, "vocabURI", keywordURI); - } + XmlWriterUtil.writeAttribute(xmlw, "vocab", keywordVocab); + XmlWriterUtil.writeAttribute(xmlw, "vocabURI", keywordURI); if (lang != null && isCVV) { - writeAttribute(xmlw, "xml:lang", defaultLocale.getLanguage()); + XmlWriterUtil.writeAttribute(xmlw, "xml:lang", defaultLocale.getLanguage()); xmlw.writeCharacters(ControlledVocabularyValue.getLocaleStrValue(keywordValue, DatasetFieldConstant.keywordValue, CITATION_BLOCK_NAME, defaultLocale, true)); @@ -753,13 +751,9 @@ DatasetFieldConstant.keywordValue, CITATION_BLOCK_NAME, new Locale(lang), false); if (translatedValue != null) { xmlw.writeStartElement("keyword"); - if (!keywordVocab.isEmpty()) { - writeAttribute(xmlw, "vocab", keywordVocab); - } - if (!keywordURI.isEmpty()) { - writeAttribute(xmlw, "vocabURI", keywordURI); - } - writeAttribute(xmlw, "xml:lang", lang); + XmlWriterUtil.writeAttribute(xmlw, "vocab", keywordVocab); + XmlWriterUtil.writeAttribute(xmlw, "vocabURI", keywordURI); + XmlWriterUtil.writeAttribute(xmlw, "xml:lang", lang); xmlw.writeCharacters(translatedValue); xmlw.writeEndElement(); // Keyword } @@ -792,14 +786,10 @@ DatasetFieldConstant.keywordValue, CITATION_BLOCK_NAME, new Locale(lang), } if (!topicClassificationValue.isEmpty()) { xmlw.writeStartElement("topcClas"); - if (!topicClassificationVocab.isEmpty()) { - writeAttribute(xmlw, "vocab", topicClassificationVocab); - } - if (!topicClassificationURI.isEmpty()) { - writeAttribute(xmlw, "vocabURI", topicClassificationURI); - } + XmlWriterUtil.writeAttribute(xmlw, "vocab", topicClassificationVocab); + XmlWriterUtil.writeAttribute(xmlw, "vocabURI", topicClassificationURI); if (lang != null && isCVV) { - writeAttribute(xmlw, "xml:lang", defaultLocale.getLanguage()); + XmlWriterUtil.writeAttribute(xmlw, "xml:lang", defaultLocale.getLanguage()); xmlw.writeCharacters(ControlledVocabularyValue.getLocaleStrValue( topicClassificationValue, DatasetFieldConstant.topicClassValue, CITATION_BLOCK_NAME, defaultLocale, true)); @@ -813,13 +803,9 @@ DatasetFieldConstant.keywordValue, CITATION_BLOCK_NAME, new Locale(lang), CITATION_BLOCK_NAME, new Locale(lang), false); if (translatedValue != null) { xmlw.writeStartElement("topcClas"); - if (!topicClassificationVocab.isEmpty()) { - writeAttribute(xmlw, "vocab", topicClassificationVocab); - } - if (!topicClassificationURI.isEmpty()) { - writeAttribute(xmlw, "vocabURI", topicClassificationURI); - } - writeAttribute(xmlw, "xml:lang", lang); + XmlWriterUtil.writeAttribute(xmlw, "vocab", topicClassificationVocab); + XmlWriterUtil.writeAttribute(xmlw, "vocabURI", topicClassificationURI); + XmlWriterUtil.writeAttribute(xmlw, "xml:lang", lang); xmlw.writeCharacters(translatedValue); xmlw.writeEndElement(); // topcClas } @@ -857,7 +843,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!authorName.isEmpty()){ xmlw.writeStartElement("AuthEnty"); if(!authorAffiliation.isEmpty()){ - writeAttribute(xmlw,"affiliation",authorAffiliation); + XmlWriterUtil.writeAttribute(xmlw,"affiliation",authorAffiliation); } xmlw.writeCharacters(authorName); xmlw.writeEndElement(); //AuthEnty @@ -880,7 +866,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!contributorName.isEmpty()){ xmlw.writeStartElement("othId"); if(!contributorType.isEmpty()){ - writeAttribute(xmlw,"role", contributorType); + XmlWriterUtil.writeAttribute(xmlw,"role", contributorType); } xmlw.writeCharacters(contributorName); xmlw.writeEndElement(); //othId @@ -922,10 +908,10 @@ private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!datasetContactName.isEmpty()){ xmlw.writeStartElement("contact"); if(!datasetContactAffiliation.isEmpty()){ - writeAttribute(xmlw,"affiliation",datasetContactAffiliation); + XmlWriterUtil.writeAttribute(xmlw,"affiliation",datasetContactAffiliation); } if(!datasetContactEmail.isEmpty()){ - writeAttribute(xmlw,"email",datasetContactEmail); + XmlWriterUtil.writeAttribute(xmlw,"email",datasetContactEmail); } xmlw.writeCharacters(datasetContactName); xmlw.writeEndElement(); //AuthEnty @@ -969,14 +955,10 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT } if (!producerName.isEmpty()) { xmlw.writeStartElement("producer"); - if (!producerAffiliation.isEmpty()) { - writeAttribute(xmlw, "affiliation", producerAffiliation); - } - if (!producerAbbreviation.isEmpty()) { - writeAttribute(xmlw, "abbr", producerAbbreviation); - } + XmlWriterUtil.writeAttribute(xmlw, "affiliation", producerAffiliation); + XmlWriterUtil.writeAttribute(xmlw, "abbr", producerAbbreviation); /*if (!producerLogo.isEmpty()) { - writeAttribute(xmlw, "role", producerLogo); + XmlWriterUtil.writeAttribute(xmlw, "role", producerLogo); }*/ xmlw.writeCharacters(producerName); xmlw.writeEndElement(); //AuthEnty @@ -987,7 +969,7 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT } } } - writeFullElement(xmlw, "prodDate", dto2Primitive(version, DatasetFieldConstant.productionDate)); + XmlWriterUtil.writeFullElement(xmlw, "prodDate", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.productionDate)); // productionPlace was made multiple as of 5.14: // (a quick backward compatibility check was added to dto2PrimitiveList(), // see the method for details) @@ -1033,17 +1015,11 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio if (!distributorName.isEmpty()) { xmlw.writeStartElement("distrbtr"); if(DvObjectContainer.isMetadataLanguageSet(lang)) { - writeAttribute(xmlw, "xml:lang", lang); - } - if (!distributorAffiliation.isEmpty()) { - writeAttribute(xmlw, "affiliation", distributorAffiliation); - } - if (!distributorAbbreviation.isEmpty()) { - writeAttribute(xmlw, "abbr", distributorAbbreviation); - } - if (!distributorURL.isEmpty()) { - writeAttribute(xmlw, "URI", distributorURL); + xmlw.writeAttribute("xml:lang", lang); } + XmlWriterUtil.writeAttribute(xmlw, "affiliation", distributorAffiliation); + XmlWriterUtil.writeAttribute(xmlw, "abbr", distributorAbbreviation); + XmlWriterUtil.writeAttribute(xmlw, "URI", distributorURL); xmlw.writeCharacters(distributorName); xmlw.writeEndElement(); //AuthEnty } @@ -1102,7 +1078,7 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO (In other words - titlStmt is mandatory! -L.A.) */ xmlw.writeStartElement("titlStmt"); - writeFullElement(xmlw, "titl", citation); + XmlWriterUtil.writeFullElement(xmlw, "titl", citation); if (IDNo != null && !IDNo.trim().equals("")) { xmlw.writeStartElement("IDNo"); @@ -1115,7 +1091,7 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO xmlw.writeEndElement(); // titlStmt - writeFullElement(xmlw,"biblCit",citation); + XmlWriterUtil.writeFullElement(xmlw,"biblCit",citation); xmlw.writeEndElement(); //citation if (url != null && !url.trim().equals("") ) { xmlw.writeStartElement("ExtLink"); @@ -1164,10 +1140,10 @@ private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!descriptionText.isEmpty()){ xmlw.writeStartElement("abstract"); if(!descriptionDate.isEmpty()){ - writeAttribute(xmlw,"date",descriptionDate); + XmlWriterUtil.writeAttribute(xmlw,"date",descriptionDate); } if(DvObjectContainer.isMetadataLanguageSet(lang)) { - writeAttribute(xmlw, "xml:lang", lang); + xmlw.writeAttribute("xml:lang", lang); } xmlw.writeCharacters(descriptionText); xmlw.writeEndElement(); //abstract @@ -1201,7 +1177,7 @@ private static void writeGrantElement(XMLStreamWriter xmlw, DatasetVersionDTO da if (!grantNumber.isEmpty()){ xmlw.writeStartElement("grantNo"); if(!grantAgency.isEmpty()){ - writeAttribute(xmlw,"agency",grantAgency); + XmlWriterUtil.writeAttribute(xmlw,"agency",grantAgency); } xmlw.writeCharacters(grantNumber); xmlw.writeEndElement(); //grantno @@ -1235,7 +1211,7 @@ private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!otherId.isEmpty()){ xmlw.writeStartElement("IDNo"); if(!otherIdAgency.isEmpty()){ - writeAttribute(xmlw,"agency",otherIdAgency); + XmlWriterUtil.writeAttribute(xmlw,"agency",otherIdAgency); } xmlw.writeCharacters(otherId); xmlw.writeEndElement(); //IDNo @@ -1269,7 +1245,7 @@ private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO if (!softwareName.isEmpty()){ xmlw.writeStartElement("software"); if(!softwareVersion.isEmpty()){ - writeAttribute(xmlw,"version",softwareVersion); + XmlWriterUtil.writeAttribute(xmlw,"version",softwareVersion); } xmlw.writeCharacters(softwareName); xmlw.writeEndElement(); //software @@ -1384,10 +1360,10 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da if (!notesText.isEmpty()) { xmlw.writeStartElement("notes"); if(!notesType.isEmpty()){ - writeAttribute(xmlw,"type",notesType); + XmlWriterUtil.writeAttribute(xmlw,"type",notesType); } if(!notesSubject.isEmpty()){ - writeAttribute(xmlw,"subject",notesSubject); + XmlWriterUtil.writeAttribute(xmlw,"subject",notesSubject); } xmlw.writeCharacters(notesText); xmlw.writeEndElement(); @@ -1412,14 +1388,14 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos // and observations, etc.) if (fileDTo.getDataFile().getDataTables() == null || fileDTo.getDataFile().getDataTables().isEmpty()) { xmlw.writeStartElement("otherMat"); - writeAttribute(xmlw, "ID", "f" + fileDTo.getDataFile().getId()); + XmlWriterUtil.writeAttribute(xmlw, "ID", "f" + fileDTo.getDataFile().getId()); String pidURL = fileDTo.getDataFile().getPidURL(); if (pidURL != null && !pidURL.isEmpty()){ - writeAttribute(xmlw, "URI", pidURL); + xmlw.writeAttribute("URI", pidURL); } else { - writeAttribute(xmlw, "URI", dataverseUrl + "/api/access/datafile/" + fileDTo.getDataFile().getId()); + xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileDTo.getDataFile().getId()); } - writeAttribute(xmlw, "level", "datafile"); + xmlw.writeAttribute("level", "datafile"); xmlw.writeStartElement("labl"); xmlw.writeCharacters(fileDTo.getDataFile().getFilename()); xmlw.writeEndElement(); // labl @@ -1430,9 +1406,9 @@ private static void createOtherMats(XMLStreamWriter xmlw, List fileDtos String contentType = fileDTo.getDataFile().getContentType(); if (!StringUtilisEmpty(contentType)) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "level", LEVEL_FILE); - writeAttribute(xmlw, "type", NOTE_TYPE_CONTENTTYPE); - writeAttribute(xmlw, "subject", NOTE_SUBJECT_CONTENTTYPE); + xmlw.writeAttribute("level", LEVEL_FILE); + xmlw.writeAttribute("type", NOTE_TYPE_CONTENTTYPE); + xmlw.writeAttribute("subject", NOTE_SUBJECT_CONTENTTYPE); xmlw.writeCharacters(contentType); xmlw.writeEndElement(); // notes } @@ -1460,14 +1436,14 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonA // and observations, etc.) if (!fileJson.containsKey("dataTables")) { xmlw.writeStartElement("otherMat"); - writeAttribute(xmlw, "ID", "f" + fileJson.getJsonNumber(("id").toString())); + xmlw.writeAttribute("ID", "f" + fileJson.getJsonNumber(("id").toString())); if (fileJson.containsKey("pidUrl")){ - writeAttribute(xmlw, "URI", fileJson.getString("pidUrl")); + XmlWriterUtil.writeAttribute(xmlw, "URI", fileJson.getString("pidUrl")); } else { - writeAttribute(xmlw, "URI", dataverseUrl + "/api/access/datafile/" + fileJson.getJsonNumber("id").toString()); + xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileJson.getJsonNumber("id").toString()); } - writeAttribute(xmlw, "level", "datafile"); + xmlw.writeAttribute("level", "datafile"); xmlw.writeStartElement("labl"); xmlw.writeCharacters(fileJson.getString("filename")); xmlw.writeEndElement(); // labl @@ -1482,9 +1458,9 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonA // specially formatted notes section: if (fileJson.containsKey("contentType")) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "level", LEVEL_FILE); - writeAttribute(xmlw, "type", NOTE_TYPE_CONTENTTYPE); - writeAttribute(xmlw, "subject", NOTE_SUBJECT_CONTENTTYPE); + xmlw.writeAttribute("level", LEVEL_FILE); + xmlw.writeAttribute("type", NOTE_TYPE_CONTENTTYPE); + xmlw.writeAttribute("subject", NOTE_SUBJECT_CONTENTTYPE); xmlw.writeCharacters(fileJson.getString("contentType")); xmlw.writeEndElement(); // notes } @@ -1502,33 +1478,7 @@ private static void writeFileDescription(XMLStreamWriter xmlw, FileDTO fileDTo) xmlw.writeEndElement(); // txt } - private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { - for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { - MetadataBlockDTO value = entry.getValue(); - for (FieldDTO fieldDTO : value.getFields()) { - if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) { - return fieldDTO.getSinglePrimitive(); - } - } - } - return null; - } - - private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName, Locale locale) { - for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { - MetadataBlockDTO value = entry.getValue(); - for (FieldDTO fieldDTO : value.getFields()) { - if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) { - String rawVal = fieldDTO.getSinglePrimitive(); - if (fieldDTO.isControlledVocabularyField()) { - return ControlledVocabularyValue.getLocaleStrValue(rawVal, datasetFieldTypeName, value.getName(), - locale, false); - } - } - } - } - return null; - } + private static List dto2PrimitiveList(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { @@ -1562,104 +1512,6 @@ private static FieldDTO dto2FieldDTO(DatasetVersionDTO datasetVersionDTO, String return null; } - private static void writeFullElementList(XMLStreamWriter xmlw, String name, List values) throws XMLStreamException { - //For the simplest Elements we can - if (values != null && !values.isEmpty()) { - for (String value : values) { - xmlw.writeStartElement(name); - xmlw.writeCharacters(value); - xmlw.writeEndElement(); // labl - } - } - } - - private static void writeI18NElementList(XMLStreamWriter xmlw, String name, List values, - String fieldTypeName, String fieldTypeClass, String metadataBlockName, String lang) - throws XMLStreamException { - - if (values != null && !values.isEmpty()) { - Locale defaultLocale = Locale.getDefault(); - for (String value : values) { - if (fieldTypeClass.equals("controlledVocabulary")) { - String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, defaultLocale, false); - if (localeVal != null) { - - value = localeVal; - writeFullElement(xmlw, name, value, defaultLocale.getLanguage()); - } else { - writeFullElement(xmlw, name, value); - } - } else { - writeFullElement(xmlw, name, value); - } - } - if (lang != null && !defaultLocale.getLanguage().equals(lang)) { - // Get values in dataset metadata language - // Loop before testing fieldTypeClass to be ready for external CVV - for (String value : values) { - if (fieldTypeClass.equals("controlledVocabulary")) { - String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, new Locale(lang), false); - if (localeVal != null) { - writeFullElement(xmlw, name, localeVal, lang); - } - } - } - } - } - } - - private static void writeI18NElement(XMLStreamWriter xmlw, String name, DatasetVersionDTO version, - String fieldTypeName, String lang) throws XMLStreamException { - // Get the default value - String val = dto2Primitive(version, fieldTypeName); - Locale defaultLocale = Locale.getDefault(); - // Get the language-specific value for the default language - // A null value is returned if this is not a CVV field - String localeVal = dto2Primitive(version, fieldTypeName, defaultLocale); - String requestedLocaleVal = null; - if (lang != null && localeVal != null && !defaultLocale.getLanguage().equals(lang)) { - // Also get the value in the requested locale/lang if that's not the default - // lang. - requestedLocaleVal = dto2Primitive(version, fieldTypeName, new Locale(lang)); - } - // FWIW locale-specific vals will only be non-null for CVV values (at present) - if (localeVal == null && requestedLocaleVal == null) { - // Not CVV/no translations so print without lang tag - writeFullElement(xmlw, name, val); - } else { - // Print in either/both languages if we have values - if (localeVal != null) { - // Print the value for the default locale with it's own lang tag - writeFullElement(xmlw, name, localeVal, defaultLocale.getLanguage()); - } - // Also print in the request lang (i.e. the metadata language for the dataset) if a value exists, print it with a lang tag - if (requestedLocaleVal != null) { - writeFullElement(xmlw, name, requestedLocaleVal, lang); - } - } - } - - private static void writeFullElement(XMLStreamWriter xmlw, String name, String value) throws XMLStreamException { - writeFullElement(xmlw, name, value, null); - } - - private static void writeFullElement (XMLStreamWriter xmlw, String name, String value, String lang) throws XMLStreamException { - //For the simplest Elements we can - if (!StringUtilisEmpty(value)) { - xmlw.writeStartElement(name); - if(DvObjectContainer.isMetadataLanguageSet(lang)) { - writeAttribute(xmlw, "xml:lang", lang); - } - xmlw.writeCharacters(value); - xmlw.writeEndElement(); // labl - } - } - - private static void writeAttribute(XMLStreamWriter xmlw, String name, String value) throws XMLStreamException { - if (!StringUtilisEmpty(value)) { - xmlw.writeAttribute(name, value); - } - } private static boolean StringUtilisEmpty(String str) { if (str == null || str.trim().equals("")) { @@ -1747,14 +1599,14 @@ public static void createDataDscr(XMLStreamWriter xmlw, JsonArray fileDetails) t } private static void createVarGroupDDI(XMLStreamWriter xmlw, JsonObject varGrp) throws XMLStreamException { xmlw.writeStartElement("varGrp"); - writeAttribute(xmlw, "ID", "VG" + varGrp.getJsonNumber("id").toString()); + xmlw.writeAttribute("ID", "VG" + varGrp.getJsonNumber("id").toString()); String vars = ""; JsonArray varsInGroup = varGrp.getJsonArray("dataVariableIds"); for (int j=0;j sumStat : dvar.getJsonObject("summaryStatistics").entrySet()) { xmlw.writeStartElement("sumStat"); - writeAttribute(xmlw, "type", sumStat.getKey()); + XmlWriterUtil.writeAttribute(xmlw, "type", sumStat.getKey()); xmlw.writeCharacters(((JsonString)sumStat.getValue()).getString()); xmlw.writeEndElement(); // sumStat } @@ -1917,7 +1769,7 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f JsonObject varCat = varCats.getJsonObject(i); xmlw.writeStartElement("catgry"); if (varCat.getBoolean("isMissing")) { - writeAttribute(xmlw, "missing", "Y"); + xmlw.writeAttribute("missing", "Y"); } // catValu @@ -1928,7 +1780,7 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f // label if (varCat.containsKey("label")) { xmlw.writeStartElement("labl"); - writeAttribute(xmlw, "level", "category"); + xmlw.writeAttribute("level", "category"); xmlw.writeCharacters(varCat.getString("label")); xmlw.writeEndElement(); // labl } @@ -1936,7 +1788,7 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f // catStat if (varCat.containsKey("frequency")) { xmlw.writeStartElement("catStat"); - writeAttribute(xmlw, "type", "freq"); + xmlw.writeAttribute("type", "freq"); Double freq = varCat.getJsonNumber("frequency").doubleValue(); // if frequency is actually a long value, we want to write "100" instead of // "100.0" @@ -1955,8 +1807,8 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f JsonObject cm = catMetas.getJsonObject(j); if (cm.getString("categoryValue").equals(varCat.getString("value"))) { xmlw.writeStartElement("catStat"); - writeAttribute(xmlw, "wgtd", "wgtd"); - writeAttribute(xmlw, "type", "freq"); + xmlw.writeAttribute("wgtd", "wgtd"); + xmlw.writeAttribute("type", "freq"); xmlw.writeCharacters(cm.getJsonNumber("wFreq").toString()); xmlw.writeEndElement(); // catStat break; @@ -1972,24 +1824,24 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f // varFormat xmlw.writeEmptyElement("varFormat"); if(dvar.containsKey("variableFormatType")) { - writeAttribute(xmlw, "type", dvar.getString("variableFormatType").toLowerCase()); + XmlWriterUtil.writeAttribute(xmlw, "type", dvar.getString("variableFormatType").toLowerCase()); } else { throw new XMLStreamException("Illegal Variable Format Type!"); } if(dvar.containsKey("format")) { - writeAttribute(xmlw, "formatname", dvar.getString("format")); + XmlWriterUtil.writeAttribute(xmlw, "formatname", dvar.getString("format")); } //experiment writeAttribute(xmlw, "schema", dv.getFormatSchema()); if(dvar.containsKey("formatCategory")) { - writeAttribute(xmlw, "category", dvar.getString("formatCategory")); + XmlWriterUtil.writeAttribute(xmlw, "category", dvar.getString("formatCategory")); } // notes if (dvar.containsKey("UNF") && !dvar.getString("UNF").isBlank()) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "subject", "Universal Numeric Fingerprint"); - writeAttribute(xmlw, "level", "variable"); - writeAttribute(xmlw, "type", "Dataverse:UNF"); + xmlw.writeAttribute("subject", "Universal Numeric Fingerprint"); + xmlw.writeAttribute("level", "variable"); + xmlw.writeAttribute("type", "Dataverse:UNF"); xmlw.writeCharacters(dvar.getString("UNF")); xmlw.writeEndElement(); //notes } @@ -2020,8 +1872,8 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) } xmlw.writeStartElement("fileDscr"); String fileId = fileJson.getJsonNumber("id").toString(); - writeAttribute(xmlw, "ID", "f" + fileId); - writeAttribute(xmlw, "URI", dataverseUrl + "/api/access/datafile/" + fileId); + xmlw.writeAttribute("ID", "f" + fileId); + xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileId); xmlw.writeStartElement("fileTxt"); xmlw.writeStartElement("fileName"); @@ -2064,9 +1916,9 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) // (Universal Numeric Fingerprint) signature: if ((dt!=null) && (dt.containsKey("UNF") && !dt.getString("UNF").isBlank())) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "level", LEVEL_FILE); - writeAttribute(xmlw, "type", NOTE_TYPE_UNF); - writeAttribute(xmlw, "subject", NOTE_SUBJECT_UNF); + xmlw.writeAttribute("level", LEVEL_FILE); + xmlw.writeAttribute("type", NOTE_TYPE_UNF); + xmlw.writeAttribute("subject", NOTE_SUBJECT_UNF); xmlw.writeCharacters(dt.getString("UNF")); xmlw.writeEndElement(); // notes } @@ -2075,9 +1927,9 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) JsonArray tags = fileJson.getJsonArray("tabularTags"); for (int j = 0; j < tags.size(); j++) { xmlw.writeStartElement("notes"); - writeAttribute(xmlw, "level", LEVEL_FILE); - writeAttribute(xmlw, "type", NOTE_TYPE_TAG); - writeAttribute(xmlw, "subject", NOTE_SUBJECT_TAG); + xmlw.writeAttribute("level", LEVEL_FILE); + xmlw.writeAttribute("type", NOTE_TYPE_TAG); + xmlw.writeAttribute("subject", NOTE_SUBJECT_TAG); xmlw.writeCharacters(tags.getString(j)); xmlw.writeEndElement(); // notes } @@ -2091,13 +1943,7 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails) - private static boolean checkParentElement(XMLStreamWriter xmlw, String elementName, boolean elementAdded) throws XMLStreamException { - if (!elementAdded) { - xmlw.writeStartElement(elementName); - } - return true; - } public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java new file mode 100644 index 00000000000..e932307d3d0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java @@ -0,0 +1,174 @@ +package edu.harvard.iq.dataverse.util.xml; + +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; + +import org.apache.commons.lang3.StringUtils; + +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; +import edu.harvard.iq.dataverse.api.dto.FieldDTO; +import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; + +public class XmlWriterUtil { + + public static void writeFullElementList(XMLStreamWriter xmlw, String name, List values) throws XMLStreamException { + // For the simplest Elements we can + if (values != null && !values.isEmpty()) { + for (String value : values) { + xmlw.writeStartElement(name); + xmlw.writeCharacters(value); + xmlw.writeEndElement(); // labl + } + } + } + + public static void writeI18NElementList(XMLStreamWriter xmlw, String name, List values, + String fieldTypeName, String fieldTypeClass, String metadataBlockName, String lang) + throws XMLStreamException { + + if (values != null && !values.isEmpty()) { + Locale defaultLocale = Locale.getDefault(); + for (String value : values) { + if (fieldTypeClass.equals("controlledVocabulary")) { + String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, defaultLocale, false); + if (localeVal != null) { + + value = localeVal; + writeFullElement(xmlw, name, value, defaultLocale.getLanguage()); + } else { + writeFullElement(xmlw, name, value); + } + } else { + writeFullElement(xmlw, name, value); + } + } + if (lang != null && !defaultLocale.getLanguage().equals(lang)) { + // Get values in dataset metadata language + // Loop before testing fieldTypeClass to be ready for external CVV + for (String value : values) { + if (fieldTypeClass.equals("controlledVocabulary")) { + String localeVal = ControlledVocabularyValue.getLocaleStrValue(value, fieldTypeName, metadataBlockName, new Locale(lang), false); + if (localeVal != null) { + writeFullElement(xmlw, name, localeVal, lang); + } + } + } + } + } + } + + public static void writeI18NElement(XMLStreamWriter xmlw, String name, DatasetVersionDTO version, + String fieldTypeName, String lang) throws XMLStreamException { + // Get the default value + String val = dto2Primitive(version, fieldTypeName); + Locale defaultLocale = Locale.getDefault(); + // Get the language-specific value for the default language + // A null value is returned if this is not a CVV field + String localeVal = dto2Primitive(version, fieldTypeName, defaultLocale); + String requestedLocaleVal = null; + if (lang != null && localeVal != null && !defaultLocale.getLanguage().equals(lang)) { + // Also get the value in the requested locale/lang if that's not the default + // lang. + requestedLocaleVal = dto2Primitive(version, fieldTypeName, new Locale(lang)); + } + // FWIW locale-specific vals will only be non-null for CVV values (at present) + if (localeVal == null && requestedLocaleVal == null) { + // Not CVV/no translations so print without lang tag + writeFullElement(xmlw, name, val); + } else { + // Print in either/both languages if we have values + if (localeVal != null) { + // Print the value for the default locale with it's own lang tag + writeFullElement(xmlw, name, localeVal, defaultLocale.getLanguage()); + } + // Also print in the request lang (i.e. the metadata language for the dataset) + // if a value exists, print it with a lang tag + if (requestedLocaleVal != null) { + writeFullElement(xmlw, name, requestedLocaleVal, lang); + } + } + } + + public static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { + for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { + MetadataBlockDTO value = entry.getValue(); + for (FieldDTO fieldDTO : value.getFields()) { + if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) { + return fieldDTO.getSinglePrimitive(); + } + } + } + return null; + } + + public static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName, Locale locale) { + for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { + MetadataBlockDTO value = entry.getValue(); + for (FieldDTO fieldDTO : value.getFields()) { + if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) { + String rawVal = fieldDTO.getSinglePrimitive(); + if (fieldDTO.isControlledVocabularyField()) { + return ControlledVocabularyValue.getLocaleStrValue(rawVal, datasetFieldTypeName, value.getName(), + locale, false); + } + } + } + } + return null; + } + + public static void writeFullElement(XMLStreamWriter xmlw, String name, String value) throws XMLStreamException { + writeFullElement(xmlw, name, value, null); + } + + public static void writeFullElement(XMLStreamWriter xmlw, String name, String value, String lang) throws XMLStreamException { + // For the simplest Elements we can + if (!StringUtils.isEmpty(value)) { + xmlw.writeStartElement(name); + if (DvObjectContainer.isMetadataLanguageSet(lang)) { + writeAttribute(xmlw, "xml:lang", lang); + } + xmlw.writeCharacters(value); + xmlw.writeEndElement(); // labl + } + } + + public static void writeAttribute(XMLStreamWriter xmlw, String name, String value) throws XMLStreamException { + if (!StringUtils.isEmpty(value)) { + xmlw.writeAttribute(name, value); + } + } + + public static boolean checkParentElement(XMLStreamWriter xmlw, String elementName, boolean elementAdded) throws XMLStreamException { + if (!elementAdded) { + xmlw.writeStartElement(elementName); + } + + return true; + } + + public static void writeFullElementWithAttributes(XMLStreamWriter xmlw, String name, Map attributeMap, String value) throws XMLStreamException { + if (!StringUtils.isEmpty(value)) { + xmlw.writeStartElement(name); + for (String key : attributeMap.keySet()) { + writeAttribute(xmlw, key, attributeMap.get(key)); + } + xmlw.writeCharacters(value); + xmlw.writeEndElement(); // labl + } + } + + public static boolean writeOpenTagIfNeeded(XMLStreamWriter xmlw, String tag, boolean element_check) throws XMLStreamException { + // check if the current tag isn't opened + if (!element_check) { + xmlw.writeStartElement(tag); // + } + return true; + } +} From 1a46155a5ed37545455a194650301cbee5691358 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:32:21 -0400 Subject: [PATCH 067/360] note duplicate method --- .../iq/dataverse/export/openaire/OpenAireExportUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index 49ceabc5900..820ced3d6c4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -1428,6 +1428,8 @@ public static void writeFundingReferencesElement(XMLStreamWriter xmlw, DatasetVe writeEndTag(xmlw, fundingReference_check); } + + //Duplicates XmlWriterUtil.dto2Primitive private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { // give the single value of the given metadata for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { From ace656ce890d6bd4ecb1b7000995e0934a2c214e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:33:13 -0400 Subject: [PATCH 068/360] remove xml template doc, refactor to generate xml, adding OA fields --- .../pidproviders/doi/XmlMetadataTemplate.java | 819 +++++++++++++----- .../doi/datacite_metadata_template.xml | 2 +- 2 files changed, 617 insertions(+), 204 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 30e4dfd79cc..8a5fe9f9d32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1,208 +1,599 @@ package edu.harvard.iq.dataverse.pidproviders.doi; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.logging.Level; +import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamWriter; + +import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.ocpsoft.common.util.Strings; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetAuthor; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.api.dto.DatasetDTO; +import edu.harvard.iq.dataverse.api.dto.FieldDTO; +import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; +import edu.harvard.iq.dataverse.export.DDIExporter; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; +import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; +import edu.harvard.iq.dataverse.util.StringUtil; +import edu.harvard.iq.dataverse.util.xml.XmlPrinter; +import edu.harvard.iq.dataverse.util.xml.XmlWriterUtil; +import jakarta.json.JsonObject; public class XmlMetadataTemplate { - private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.DataCiteMetadataTemplate"); - private static String template; - - static { - try (InputStream in = XmlMetadataTemplate.class.getResourceAsStream("datacite_metadata_template.xml")) { - template = new String(in.readAllBytes(), StandardCharsets.UTF_8); - } catch (Exception e) { - logger.log(Level.SEVERE, "datacite metadata template load error"); - logger.log(Level.SEVERE, "String " + e.toString()); - logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); - logger.log(Level.SEVERE, "cause " + e.getCause()); - logger.log(Level.SEVERE, "message " + e.getMessage()); - } - } + private static final Logger logger = Logger.getLogger(XmlMetadataTemplate.class.getName()); - private String xmlMetadata; - private String identifier; - private List datafileIdentifiers; - private List creators; - private String title; - private String publisher; - private String publisherYear; - private List authors; - private String description; - private List contacts; - private List producers; - - public List getProducers() { - return producers; - } + public static final String XML_NAMESPACE = "http://datacite.org/schema/kernel-4"; + public static final String XML_SCHEMA_LOCATION = "http://datacite.org/schema/kernel-4 http://schema.datacite.org/meta/kernel-4.5/metadata.xsd"; + public static final String XML_XSI = "http://www.w3.org/2001/XMLSchema-instance"; + public static final String XML_SCHEMA_VERSION = "4.5"; - public void setProducers(List producers) { - this.producers = producers; - } + private DoiMetadata doiMetadata; - public List getContacts() { - return contacts; + public XmlMetadataTemplate() { } - public void setContacts(List contacts) { - this.contacts = contacts; + public XmlMetadataTemplate(DoiMetadata doiMetadata) { + this.doiMetadata = doiMetadata; } - public String getDescription() { - return description; + public String generateXML(DvObject dvObject) { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + generateXML(dvObject, outputStream); + + String xml = outputStream.toString(); + return XmlPrinter.prettyPrintXml(xml); + } catch (XMLStreamException | IOException e) { + logger.severe("Unable to generate DataCite XML for DOI: " + dvObject.getGlobalId().asString() + " : " + e.getMessage()); + e.printStackTrace(); + } + return null; } - public void setDescription(String description) { - this.description = description; - } + private void generateXML(DvObject dvObject, OutputStream outputStream) throws XMLStreamException { + // Could/should use dataset metadata language for metadata from DvObject itself? + String language = null; // machine locale? e.g. for Publisher which is global + String metadataLanguage = null; // when set, otherwise = language? + XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); + xmlw.writeStartElement("resource"); + xmlw.writeDefaultNamespace(XML_NAMESPACE); + xmlw.writeAttribute("xmlns:xsi", XML_XSI); + xmlw.writeAttribute("xsi:schemaLocation", XML_SCHEMA_LOCATION); + + writeIdentifier(xmlw, dvObject); + writeCreators(xmlw, doiMetadata.getAuthors()); + writeTitles(xmlw, dvObject, language); + writePublisher(xmlw, dvObject); + writePublicationYear(xmlw, dvObject); + writeSubjects(xmlw, dvObject); + writeContributors(xmlw, dvObject); + writeDates(xmlw, dvObject); + writeLanguage(xmlw, dvObject); + writeResourceType(xmlw, dvObject); + writeAlternateIdentifiers(xmlw, dvObject); + writeRelatedIdentifiers(xmlw, dvObject); + writeSize(xmlw, dvObject); + writeFormats(xmlw, dvObject); + writeVersion(xmlw, dvObject); + writeAccessRights(xmlw, dvObject); + writeDescriptions(xmlw, dvObject); + writeGeoLocations(xmlw, dvObject); + writeFundingReferences(xmlw, dvObject); - public List getAuthors() { - return authors; - } + StringBuilder contributorsElement = new StringBuilder(); + if (doiMetadata.getContacts() != null) { + for (String[] contact : doiMetadata.getContacts()) { + if (!contact[0].isEmpty()) { + contributorsElement.append("" + + StringEscapeUtils.escapeXml10(contact[0]) + ""); + if (!contact[1].isEmpty()) { + contributorsElement.append("" + StringEscapeUtils.escapeXml10(contact[1]) + ""); + } + contributorsElement.append(""); + } + } + } - public void setAuthors(List authors) { - this.authors = authors; - } + if (doiMetadata.getProducers() != null) { + for (String[] producer : doiMetadata.getProducers()) { + contributorsElement.append("" + StringEscapeUtils.escapeXml10(producer[0]) + + ""); + if (!producer[1].isEmpty()) { + contributorsElement.append("" + StringEscapeUtils.escapeXml10(producer[1]) + ""); + } + contributorsElement.append(""); + } + } - public XmlMetadataTemplate() { - } + String relIdentifiers = generateRelatedIdentifiers(dvObject); - public List getDatafileIdentifiers() { - return datafileIdentifiers; } - public void setDatafileIdentifiers(List datafileIdentifiers) { - this.datafileIdentifiers = datafileIdentifiers; - } - public XmlMetadataTemplate(String xmlMetaData) { - this.xmlMetadata = xmlMetaData; - Document doc = Jsoup.parseBodyFragment(xmlMetaData); - Elements identifierElements = doc.select("identifier"); - if (identifierElements.size() > 0) { - identifier = identifierElements.get(0).html(); - } - Elements creatorElements = doc.select("creatorName"); - creators = new ArrayList<>(); - for (Element creatorElement : creatorElements) { - creators.add(creatorElement.html()); + /** + * 3, Title(s) (with optional type sub-properties) (M) + * + * @param xmlw + * The Stream writer + * @param dvObject + * The dataset/file + * @param language + * the metadata language + * @return + * @throws XMLStreamException + */ + private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String language) throws XMLStreamException { + String title = doiMetadata.getTitle(); + String subTitle = null; + List altTitles = null; + // Only Datasets can have a subtitle or alternative titles + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersion(); + Optional subTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.subTitle)).findFirst(); + if (subTitleField.isPresent()) { + subTitle = subTitleField.get().getValue(); + } + Optional altTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.alternativeTitle)).findFirst(); + if (altTitleField.isPresent()) { + altTitles = altTitleField.get().getValues(); + } } - Elements titleElements = doc.select("title"); - if (titleElements.size() > 0) { - title = titleElements.get(0).html(); + + if (StringUtils.isNotBlank(title) || StringUtils.isNotBlank(subTitle) || (altTitles != null && !String.join("", altTitles).isBlank())) { + xmlw.writeStartElement("titles"); + XmlWriterUtil.writeFullElement(xmlw, "title", title, language); + + Map attributes = new HashMap(); + attributes.put("titleType", "Subtitle"); + + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, title); + + attributes.clear(); + attributes.put("titleType", "AlternativeTitle"); + + for (String altTitle : altTitles) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, altTitle); + } + + xmlw.writeEndElement(); } - Elements publisherElements = doc.select("publisher"); - if (publisherElements.size() > 0) { - publisher = publisherElements.get(0).html(); + } + + /** + * 1, Identifier (with mandatory type sub-property) (M) Note DataCite expects + * identifierType="DOI" but OpenAire allows several others (see + * https://guidelines.readthedocs.io/en/latest/data/field_identifier.html#d-identifiertype) + * Dataverse is currently only capable of creating DOI, Handle, or URL types + * from the OpenAire list (the last from PermaLinks) ToDo - If we add,e.g., an + * ARK or PURL provider, this code has to change or we'll need to refactor so + * that the identifiertype and id value can be sent via the JSON/ORE + * + * @param xmlw + * The Steam writer + * @param dvObject + * The dataset or file with the PID + * @throws XMLStreamException + */ + private void writeIdentifier(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + GlobalId pid = dvObject.getGlobalId(); + // identifier with identifierType attribute + Map identifier_map = new HashMap(); + String identifierType = null; + String identifier = null; + switch (pid.getProtocol()) { + case AbstractDOIProvider.DOI_PROTOCOL: + identifierType = AbstractDOIProvider.DOI_PROTOCOL.toUpperCase(); + identifier = pid.asRawIdentifier(); + break; + case HandlePidProvider.HDL_PROTOCOL: + identifierType = "Handle"; + identifier = pid.asRawIdentifier(); + break; + case PermaLinkPidProvider.PERMA_PROTOCOL: + identifierType = "URL"; + identifier = pid.asURL(); + break; } - Elements publisherYearElements = doc.select("publicationYear"); - if (publisherYearElements.size() > 0) { - publisherYear = publisherYearElements.get(0).html(); + Map attributeMap = new HashMap(); + attributeMap.put("identifierType", identifierType); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "identifier", attributeMap, identifier); + } + + /** + * 2, Creator (with optional given name, family name, name identifier and + * affiliation sub-properties) (M) + * + * @param xmlw + * The stream writer + * @param authorList + * - the list of authors + * @throws XMLStreamException + */ + public void writeCreators(XMLStreamWriter xmlw, List authorList) throws XMLStreamException { + // creators -> creator -> creatorName with nameType attribute, givenName, + // familyName, nameIdentifier + // write all creators + xmlw.writeStartElement("creators"); // + + if (authorList != null && !authorList.isEmpty()) { + for (DatasetAuthor author : authorList) { + String creatorName = StringEscapeUtils.escapeXml10(author.getName().getDisplayValue()); + String affiliation = null; + if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { + affiliation = StringEscapeUtils.escapeXml10(author.getAffiliation().getDisplayValue()); + } + String nameIdentifier = null; + String nameIdentifierScheme = null; + if (StringUtils.isNotBlank(author.getIdValue()) && StringUtils.isNotBlank(author.getIdType())) { + nameIdentifier = author.getIdValue(); + if(nameIdentifier != null) { + // Normalizes to the URL form of the identifier, returns null if the identifier + // is not valid given the type + nameIdentifier = author.getIdentifierAsUrl(); + } + nameIdentifierScheme = author.getIdType(); + } + + if (StringUtils.isNotBlank(creatorName)) { + xmlw.writeStartElement("creator"); // + JsonObject creatorObj = PersonOrOrgUtil.getPersonOrOrganization(creatorName, false, + StringUtils.containsIgnoreCase(nameIdentifierScheme, "orcid")); + + writeEntityElements(xmlw, "creator", null, creatorObj, affiliation, nameIdentifier, nameIdentifierScheme); + xmlw.writeEndElement(); // + } + + else { + // Authors unavailable + XmlWriterUtil.writeFullElement(xmlw, "creator", "creatorName", AbstractPidProvider.UNAVAILABLE); + } + } } + xmlw.writeEndElement(); // } - public String generateXML(DvObject dvObject) { + private void writePublisher(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + // publisher should already be non null - :unav if it wasn't available + XmlWriterUtil.writeFullElement(xmlw, "publisher", doiMetadata.getPublisher()); + } + + private void writePublicationYear(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { // Can't use "UNKNOWN" here because DataCite will respond with "[facet // 'pattern'] the value 'unknown' is not accepted by the pattern '[\d]{4}'" - String publisherYearFinal = "9999"; + String pubYear = "9999"; // FIXME: Investigate why this.publisherYear is sometimes null now that pull // request #4606 has been merged. - if (this.publisherYear != null) { + if (doiMetadata.getPublisherYear() != null) { // Added to prevent a NullPointerException when trying to destroy datasets when // using DataCite rather than EZID. - publisherYearFinal = this.publisherYear; + pubYear = doiMetadata.getPublisherYear(); } - xmlMetadata = template.replace("${identifier}", getIdentifier().trim()).replace("${title}", this.title) - .replace("${publisher}", this.publisher).replace("${publisherYear}", publisherYearFinal) - .replace("${description}", this.description); - - StringBuilder creatorsElement = new StringBuilder(); - if (authors != null && !authors.isEmpty()) { - for (DatasetAuthor author : authors) { - creatorsElement.append(""); - creatorsElement.append(author.getName().getDisplayValue()); - creatorsElement.append(""); - - if (author.getIdType() != null && author.getIdValue() != null && !author.getIdType().isEmpty() - && !author.getIdValue().isEmpty() && author.getAffiliation() != null - && !author.getAffiliation().getDisplayValue().isEmpty()) { - - if (author.getIdType().equals("ORCID")) { - creatorsElement.append( - "" - + author.getIdValue() + ""); - } - if (author.getIdType().equals("ISNI")) { - creatorsElement.append( - "" - + author.getIdValue() + ""); - } - if (author.getIdType().equals("LCNA")) { - creatorsElement.append( - "" - + author.getIdValue() + ""); - } - } - if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - creatorsElement - .append("" + author.getAffiliation().getDisplayValue() + ""); + XmlWriterUtil.writeFullElement(xmlw, "publicationYear", String.valueOf(pubYear)); + } + + /** + * 6, Subject (with scheme sub-property) R + * + * @param xmlw + * The Steam writer + * @param dvObject + * The Dataset/DataFile + * @throws XMLStreamException + */ + private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + // subjects -> subject with subjectScheme and schemeURI attributes when + // available + boolean subjectsCreated = false; + List subjects = null; + List compoundKeywords = null; + List compoundTopics = null; + // Dataset Subject= Dataverse subject, keyword, and/or topic classification + // fields + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + dv.getDatasetSubjects(); + for (DatasetField dsf : dv.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.keyword)) { + compoundKeywords = dsf.getDatasetFieldCompoundValues(); + } else if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.topicClassification)) { + compoundTopics = dsf.getDatasetFieldCompoundValues(); } - creatorsElement.append(""); } - } else { - creatorsElement.append("").append(AbstractPidProvider.UNAVAILABLE) - .append(""); + } else if (dvObject instanceof DataFile df) { + subjects = df.getTagLabels(); + } + for (String subject : subjects) { + if (StringUtils.isNotBlank(subject)) { + subjectsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "subjects", subjectsCreated); + XmlWriterUtil.writeFullElement(xmlw, "subject", StringEscapeUtils.escapeXml10(subject)); + } } + for (DatasetFieldCompoundValue keywordFieldValue : compoundKeywords) { + String keyword = null; + String scheme = null; + String schemeUri = null; + + for (DatasetField subField : keywordFieldValue.getChildDatasetFields()) { + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.keyword: + keyword = subField.getValue(); + break; + case DatasetFieldConstant.keywordVocab: + scheme = subField.getValue(); + break; + case DatasetFieldConstant.keywordVocabURI: + schemeUri = subField.getValue(); + break; + } + } + if (StringUtils.isNotBlank(keyword)) { + Map attributesMap = new HashMap(); + if (StringUtils.isNotBlank(scheme)) { + attributesMap.put("subjectScheme", scheme); + } + if (StringUtils.isNotBlank(schemeUri)) { + attributesMap.put("schemeURI", schemeUri); + } + subjectsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "subjects", subjectsCreated); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "subject", attributesMap, StringEscapeUtils.escapeXml10(keyword)); + } + } + for (DatasetFieldCompoundValue topicFieldValue : compoundTopics) { + String topic = null; + String scheme = null; + String schemeUri = null; + + for (DatasetField subField : topicFieldValue.getChildDatasetFields()) { + + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.topicClassValue: + topic = subField.getValue(); + break; + case DatasetFieldConstant.topicClassVocab: + scheme = subField.getValue(); + break; + case DatasetFieldConstant.topicClassVocabURI: + schemeUri = subField.getValue(); + break; + } + } + if (StringUtils.isNotBlank(topic)) { + Map attributesMap = new HashMap(); + if (StringUtils.isNotBlank(scheme)) { + attributesMap.put("subjectScheme", scheme); + } + if (StringUtils.isNotBlank(schemeUri)) { + attributesMap.put("schemeURI", schemeUri); + } + subjectsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "subjects", subjectsCreated); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "subject", attributesMap, StringEscapeUtils.escapeXml10(topic)); + } + } + if (subjectsCreated) { + xmlw.writeEndElement(); + } + } - xmlMetadata = xmlMetadata.replace("${creators}", creatorsElement.toString()); + /** + * 7, Contributor (with optional given name, family name, name identifier + * and affiliation sub-properties) + * + * @see #writeContributorElement(javax.xml.stream.XMLStreamWriter, + * java.lang.String, java.lang.String, java.lang.String) + * + * @param xmlw The stream writer + * @param dvObject The Dataset/DataFile + * @throws XMLStreamException + */ + private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + boolean contributorsCreated = false; + List compoundProducers = null; + List compoundDistributors = null; + List compoundContacts = null; + List compoundContributors = null; + // Dataset Subject= Dataverse subject, keyword, and/or topic classification + // fields + //ToDo Include for files? + /*if(dvObject instanceof DataFile df) { + dvObject = df.getOwner(); + }*/ + + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + for (DatasetField dsf : dv.getDatasetFields()) { + switch (dsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.producer: + compoundProducers = dsf.getDatasetFieldCompoundValues(); + break; + case DatasetFieldConstant.distributor: + compoundDistributors = dsf.getDatasetFieldCompoundValues(); + break; + case DatasetFieldConstant.contact: + compoundContacts = dsf.getDatasetFieldCompoundValues(); + break; + case DatasetFieldConstant.contributor: + compoundContributors = dsf.getDatasetFieldCompoundValues(); + } + } + } + + + for (DatasetFieldCompoundValue producerFieldValue : compoundProducers) { + String producer = null; + String affiliation = null; + + for (DatasetField subField : producerFieldValue.getChildDatasetFields()) { + + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.producerName: + producer = subField.getValue(); + break; + case DatasetFieldConstant.producerAffiliation: + affiliation = subField.getValue(); + break; + } + } + if (StringUtils.isNotBlank(producer)) { + contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); + JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(producer, false, false); + writeEntityElements(xmlw, "contributor", "Producer", entityObject, affiliation, null, null); + } - StringBuilder contributorsElement = new StringBuilder(); - if (this.getContacts() != null) { - for (String[] contact : this.getContacts()) { - if (!contact[0].isEmpty()) { - contributorsElement.append("" - + contact[0] + ""); - if (!contact[1].isEmpty()) { - contributorsElement.append("" + contact[1] + ""); - } - contributorsElement.append(""); + } + + for (DatasetFieldCompoundValue distributorFieldValue : compoundDistributors) { + String distributor = null; + String affiliation = null; + + for (DatasetField subField : distributorFieldValue.getChildDatasetFields()) { + + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.distributorName: + distributor = subField.getValue(); + break; + case DatasetFieldConstant.distributorAffiliation: + affiliation = subField.getValue(); + break; } } + if (StringUtils.isNotBlank(distributor)) { + contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); + JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(distributor, false, false); + writeEntityElements(xmlw, "contributor", "Distributor", entityObject, affiliation, null, null); + } + } + for (DatasetFieldCompoundValue contactFieldValue : compoundContacts) { + String contact = null; + String affiliation = null; + + for (DatasetField subField : contactFieldValue.getChildDatasetFields()) { + + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.datasetContactName: + contact = subField.getValue(); + break; + case DatasetFieldConstant.datasetContactAffiliation: + affiliation = subField.getValue(); + break; + } + } + if (StringUtils.isNotBlank(contact)) { + contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); + JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(contact, false, false); + writeEntityElements(xmlw, "contributor", "ContactPerson", entityObject, affiliation, null, null); + } - if (this.getProducers() != null) { - for (String[] producer : this.getProducers()) { - contributorsElement.append("" + producer[0] - + ""); - if (!producer[1].isEmpty()) { - contributorsElement.append("" + producer[1] + ""); + } + for (DatasetFieldCompoundValue contributorFieldValue : compoundContributors) { + String contributor = null; + String contributorType = null; + + for (DatasetField subField : contributorFieldValue.getChildDatasetFields()) { + + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.contributorName: + contributor = subField.getValue(); + break; + case DatasetFieldConstant.contributorType: + contributorType = subField.getValue().replace(" ", ""); + break; } - contributorsElement.append(""); } + // QDR - doesn't have Funder in the contributor type list. + // Using a string isn't i18n + if (StringUtils.isNotBlank(contributor) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { + contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); + JsonObject entityObject = PersonOrOrgUtil.getPersonOrOrganization(contributor, false, false); + writeEntityElements(xmlw, "contributor", contributorType, entityObject, null, null, null); + } + + } + + if (contributorsCreated) { + xmlw.writeEndElement(); } + } - String relIdentifiers = generateRelatedIdentifiers(dvObject); + private void writeEntityElements(XMLStreamWriter xmlw, String elementName, String type, JsonObject entityObject, String affiliation, String nameIdentifier, String nameIdentifierScheme) throws XMLStreamException { + xmlw.writeStartElement(elementName); + Map attributeMap = new HashMap(); + if (StringUtils.isNotBlank(type)) { + attributeMap.put("contributorType", type); + } + // person name=, + if (entityObject.getBoolean("isPerson")) { + attributeMap.put("nameType", "Personal"); + } else { + attributeMap.put("nameType", "Organizational"); + } + XmlWriterUtil.writeFullElementWithAttributes(xmlw, elementName + "Name", attributeMap, + StringEscapeUtils.escapeXml10(entityObject.getString("fullName"))); + if (entityObject.containsKey("givenName")) { + XmlWriterUtil.writeFullElement(xmlw, "givenName", StringEscapeUtils.escapeXml10(entityObject.getString("givenName"))); + } + if (entityObject.containsKey("familyName")) { + XmlWriterUtil.writeFullElement(xmlw, "familyName", StringEscapeUtils.escapeXml10(entityObject.getString("familyName"))); + } - xmlMetadata = xmlMetadata.replace("${relatedIdentifiers}", relIdentifiers); + if (nameIdentifier != null) { + attributeMap.clear(); + URL url; + try { + url = new URL(nameIdentifier); + String protocol = url.getProtocol(); + String authority = url.getAuthority(); + String site = String.format("%s://%s", protocol, authority); + attributeMap.put("schemeURI", site); + attributeMap.put("nameIdentifierScheme", nameIdentifierScheme); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "nameIdentifier", attributeMap, nameIdentifier); + } catch (MalformedURLException e) { + logger.warning("DatasetAuthor.getIdentifierAsUrl returned a Malformed URL: " + nameIdentifier); + } + } + + if (StringUtils.isNotBlank(affiliation)) { + attributeMap.clear(); + if (affiliation.startsWith("https://ror.org/")) { - xmlMetadata = xmlMetadata.replace("{$contributors}", contributorsElement.toString()); - return xmlMetadata; + attributeMap.put("schemeURI", "https://ror.org"); + attributeMap.put("affiliationIdentifierScheme", "ROR"); + } + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(affiliation)); + } + xmlw.writeEndElement(); } private String generateRelatedIdentifiers(DvObject dvObject) { @@ -210,9 +601,67 @@ private String generateRelatedIdentifiers(DvObject dvObject) { StringBuilder sb = new StringBuilder(); if (dvObject.isInstanceofDataset()) { Dataset dataset = (Dataset) dvObject; + + List relatedPublications = dataset.getLatestVersionForCopy().getRelatedPublications(); + if (!relatedPublications.isEmpty()) { + for (DatasetRelPublication relatedPub : relatedPublications) { + String pubIdType = relatedPub.getIdType(); + String identifier = relatedPub.getIdNumber(); + /* + * Note - with identifier and url fields, it's not clear that there's a single + * way those two fields are used for all identifier types In QDR, at this time, + * doi and isbn types always have the raw number in the identifier field, + * whereas there are examples where URLs are in the identifier or url fields. + * The code here addresses those practices and is not generic. + */ + if (pubIdType != null) { + switch (pubIdType) { + case "doi": + if (identifier != null && identifier.length() != 0) { + appendIdentifier(sb, "DOI", "IsSupplementTo", "doi:" + identifier); + } + break; + case "isbn": + if (identifier != null && identifier.length() != 0) { + appendIdentifier(sb, "ISBN", "IsSupplementTo", "ISBN:" + identifier); + } + break; + case "url": + if (identifier != null && identifier.length() != 0) { + appendIdentifier(sb, "URL", "IsSupplementTo", identifier); + } else { + String pubUrl = relatedPub.getUrl(); + if (pubUrl != null && pubUrl.length() > 0) { + appendIdentifier(sb, "URL", "IsSupplementTo", pubUrl); + } + } + break; + default: + if (identifier != null && identifier.length() != 0) { + if (pubIdType.equalsIgnoreCase("arXiv")) { + pubIdType = "arXiv"; + } else if (pubIdType.equalsIgnoreCase("handle")) { + // Initial cap required for handle + pubIdType = "Handle"; + } else if (!pubIdType.equals("bibcode")) { + pubIdType = pubIdType.toUpperCase(); + } + // For all others, do a generic attempt to match the identifier type to the + // datacite schema and send the raw identifier as the value + appendIdentifier(sb, pubIdType, "IsSupplementTo", identifier); + } + break; + } + + } else { + logger.info(relatedPub.getIdNumber() + relatedPub.getUrl() + relatedPub.getTitle()); + } + } + } + if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - datafileIdentifiers = new ArrayList<>(); + List datafileIdentifiers = new ArrayList<>(); for (DataFile dataFile : dataset.getFiles()) { if (dataFile.getGlobalId() != null) { if (sb.toString().isEmpty()) { @@ -229,14 +678,23 @@ private String generateRelatedIdentifiers(DvObject dvObject) { } } else if (dvObject.isInstanceofDataFile()) { DataFile df = (DataFile) dvObject; - sb.append(""); - sb.append("" - + df.getOwner().getGlobalId() + ""); - sb.append(""); + appendIdentifier(sb, "DOI", "IsPartOf", df.getOwner().getGlobalId().asString()); + if (sb.length() != 0) { + // Should always be true + sb.append(""); + } } return sb.toString(); } + + private void appendIdentifier(StringBuilder sb, String idType, String relationType, String identifier) { + if (sb.toString().isEmpty()) { + sb.append(""); + } + sb.append("" + identifier + ""); + } + public void generateFileIdentifiers(DvObject dvObject) { if (dvObject.isInstanceofDataset()) { @@ -244,71 +702,26 @@ public void generateFileIdentifiers(DvObject dvObject) { if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - datafileIdentifiers = new ArrayList<>(); + List datafileIdentifiers = new ArrayList<>(); for (DataFile dataFile : dataset.getFiles()) { datafileIdentifiers.add(dataFile.getIdentifier()); - int x = xmlMetadata.indexOf("") - 1; - xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", dataFile.getIdentifier()); - xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" - + template.substring(x, template.length() - 1); + // int x = xmlMetadata.indexOf("") - 1; + // xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", + // dataFile.getIdentifier()); + // xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" + // + template.substring(x, template.length() - 1); } } else { - xmlMetadata = xmlMetadata.replace( - "${relatedIdentifier}", - ""); + // xmlMetadata = xmlMetadata.replace( + // "${relatedIdentifier}", + // ""); } } } - public static String getTemplate() { - return template; - } - - public static void setTemplate(String template) { - XmlMetadataTemplate.template = template; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public List getCreators() { - return creators; - } - - public void setCreators(List creators) { - this.creators = creators; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getPublisher() { - return publisher; - } - - public void setPublisher(String publisher) { - this.publisher = publisher; - } - - public String getPublisherYear() { - return publisherYear; - } - - public void setPublisherYear(String publisherYear) { - this.publisherYear = publisherYear; - } - } \ No newline at end of file diff --git a/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml b/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml index abe7ce79972..8348691d4c7 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml +++ b/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml @@ -1,5 +1,5 @@ - ${identifier} From dba03e2bb1597d4e01317139d950e305d0d9dec5 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:33:40 -0400 Subject: [PATCH 069/360] refactor source of XML info --- .../iq/dataverse/export/DataCiteExporter.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DataCiteExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/DataCiteExporter.java index 8caf32b2df0..c21d6b5cd1a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DataCiteExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DataCiteExporter.java @@ -7,6 +7,7 @@ import io.gdcc.spi.export.ExportException; import io.gdcc.spi.export.Exporter; import io.gdcc.spi.export.XMLExporter; +import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; import java.io.OutputStream; @@ -20,11 +21,7 @@ */ @AutoService(Exporter.class) public class DataCiteExporter implements XMLExporter { - - private static String DEFAULT_XML_NAMESPACE = "http://datacite.org/schema/kernel-3"; - private static String DEFAULT_XML_SCHEMALOCATION = "http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd"; - private static String DEFAULT_XML_VERSION = "3.0"; - + public static final String NAME = "Datacite"; @Override @@ -60,17 +57,17 @@ public Boolean isAvailableToUsers() { @Override public String getXMLNameSpace() { - return DataCiteExporter.DEFAULT_XML_NAMESPACE; + return XmlMetadataTemplate.XML_NAMESPACE; } @Override public String getXMLSchemaLocation() { - return DataCiteExporter.DEFAULT_XML_SCHEMALOCATION; + return XmlMetadataTemplate.XML_SCHEMA_LOCATION; } @Override public String getXMLSchemaVersion() { - return DataCiteExporter.DEFAULT_XML_VERSION; + return XmlMetadataTemplate.XML_SCHEMA_VERSION; } } From af3e24b0b7bc1bff2c378f2a682455fe6aef0ee2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 10:34:09 -0400 Subject: [PATCH 070/360] add code to get raw alphanumeric pid value --- src/main/java/edu/harvard/iq/dataverse/GlobalId.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java index a542cb52ac0..1c8783c5bd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java @@ -100,6 +100,13 @@ public String asURL() { } return null; } + + public String asRawIdentifier() { + if (protocol == null || authority == null || identifier == null) { + return ""; + } + return authority + separator + identifier; + } From fa23884647c893285e456d749a741d6d36ac90eb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 13:58:32 -0400 Subject: [PATCH 071/360] remove duplicate method --- .../edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java index e932307d3d0..8ec426ead1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlWriterUtil.java @@ -145,13 +145,6 @@ public static void writeAttribute(XMLStreamWriter xmlw, String name, String valu } } - public static boolean checkParentElement(XMLStreamWriter xmlw, String elementName, boolean elementAdded) throws XMLStreamException { - if (!elementAdded) { - xmlw.writeStartElement(elementName); - } - - return true; - } public static void writeFullElementWithAttributes(XMLStreamWriter xmlw, String name, Map attributeMap, String value) throws XMLStreamException { if (!StringUtils.isEmpty(value)) { From 0d22d6c580df4aa689b019dfdc88321a59e02e4d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 3 May 2024 13:59:22 -0400 Subject: [PATCH 072/360] dates, resourceType, alternate Ids --- .../pidproviders/doi/XmlMetadataTemplate.java | 224 +++++++++++++++++- 1 file changed, 215 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8a5fe9f9d32..92bf7afd273 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -6,12 +6,14 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -26,6 +28,8 @@ import org.jsoup.select.Elements; import org.ocpsoft.common.util.Strings; +import edu.harvard.iq.dataverse.AlternativePersistentIdentifier; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetAuthor; @@ -33,9 +37,11 @@ import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; @@ -207,8 +213,6 @@ private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String languag */ private void writeIdentifier(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { GlobalId pid = dvObject.getGlobalId(); - // identifier with identifierType attribute - Map identifier_map = new HashMap(); String identifierType = null; String identifier = null; switch (pid.getProtocol()) { @@ -315,9 +319,9 @@ private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLSt // subjects -> subject with subjectScheme and schemeURI attributes when // available boolean subjectsCreated = false; - List subjects = null; - List compoundKeywords = null; - List compoundTopics = null; + List subjects = new ArrayList(); + List compoundKeywords = new ArrayList(); + List compoundTopics = new ArrayList(); // Dataset Subject= Dataverse subject, keyword, and/or topic classification // fields if (dvObject instanceof Dataset d) { @@ -419,10 +423,10 @@ private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLSt */ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { boolean contributorsCreated = false; - List compoundProducers = null; - List compoundDistributors = null; - List compoundContacts = null; - List compoundContributors = null; + List compoundProducers = new ArrayList(); + List compoundDistributors = new ArrayList(); + List compoundContacts = new ArrayList(); + List compoundContributors = new ArrayList(); // Dataset Subject= Dataverse subject, keyword, and/or topic classification // fields //ToDo Include for files? @@ -596,6 +600,208 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin xmlw.writeEndElement(); } + /** + * 8, Date (with type sub-property) (R) + * + * @param xmlw The Steam writer + * @param dvObject The dataset/datafile + * @throws XMLStreamException + */ + private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + boolean datesWritten = false; + String dateOfDistribution = null; + String dateOfProduction = null; + String dateOfDeposit = null; + Date releaseDate = null; + List datesOfCollection = new ArrayList(); + + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + releaseDate = dv.getReleaseTime(); + for (DatasetField dsf : dv.getDatasetFields()) { + switch (dsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.distributionDate: + dateOfDistribution = dsf.getValue(); + break; + case DatasetFieldConstant.productionDate: + dateOfProduction = dsf.getValue(); + break; + case DatasetFieldConstant.dateOfDeposit: + dateOfDeposit = dsf.getValue(); + break; + case DatasetFieldConstant.dateOfCollection: + datesOfCollection = dsf.getDatasetFieldCompoundValues(); + } + } + } + Map attributes = new HashMap(); + if (StringUtils.isNotBlank(dateOfDistribution)) { + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + attributes.put("dateType", "Issued"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, dateOfDistribution); + } + // dates -> date with dateType attribute + + if (StringUtils.isNotBlank(dateOfProduction)) { + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + attributes.put("dateType", "Created"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, dateOfProduction); + } + if (StringUtils.isNotBlank(dateOfDeposit)) { + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + attributes.put("dateType", "Submitted"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, dateOfDeposit); + } + + if (releaseDate != null) { + String date = Util.getDateTimeFormat().format(releaseDate); + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + + attributes.put("dateType", "Available"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, date); + } + if (datesOfCollection != null) { + for (DatasetFieldCompoundValue collectionDateFieldValue : datesOfCollection) { + String startDate = null; + String endDate = null; + + for (DatasetField subField : collectionDateFieldValue.getChildDatasetFields()) { + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.dateOfCollectionStart: + startDate = subField.getValue(); + break; + case DatasetFieldConstant.dateOfCollectionEnd: + endDate = subField.getValue(); + break; + } + } + if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + attributes.put("dateType", "Collected"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, (startDate + "/" + endDate).trim()); + } + } + } + if (datesWritten) { + xmlw.writeEndElement(); + } + } + + + // 9, Language (MA), language + private void writeLanguage(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + //Currently not supported. Spec indicates one 'primary' language. Could send the first entry in DatasetFieldConstant.language or send iff there is only one entry, and/or default to the machine's default lang? + return; + } + + // 10, ResourceType (with mandatory general type + // description sub- property) (M) + private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + List kindOfDataValues = new ArrayList(); + Map attributes = new HashMap(); + + attributes.put("resourceTypeGeneral", "Dataset"); + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + for (DatasetField dsf : dv.getDatasetFields()) { + switch (dsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.kindOfData: + kindOfDataValues = dsf.getControlledVocabularyValues(); + break; + } + + if (kindOfDataValues.isEmpty()) { + // Write an attribute only element if there are no kindOfData values. + xmlw.writeStartElement("resourceType"); + xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); + xmlw.writeEndElement(); + } else { + for (ControlledVocabularyValue kindOfDataValue : kindOfDataValues) { + String resourceType = kindOfDataValue.getStrValue(); + if (StringUtils.isNotBlank(resourceType)) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, resourceType); + } + } + } + } + } + } + + /** + * 11 AlternateIdentifier (with type sub-property) (O) + * + * @param xmlw The Steam writer + * @param dvObject The dataset/datafile + * @throws XMLStreamException + */ + private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + List otherIdentifiers = new ArrayList(); + Set altPids = dvObject.getAlternativePersistentIndentifiers(); + + boolean alternatesWritten = false; + + Map attributes = new HashMap(); + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + for (DatasetField dsf : dv.getDatasetFields()) { + if (DatasetFieldConstant.otherId.equals(dsf.getDatasetFieldType().getName())) { + otherIdentifiers = dsf.getDatasetFieldCompoundValues(); + break; + } + } + } + if (!altPids.isEmpty()) { + alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); + } + for (AlternativePersistentIdentifier altPid : altPids) { + String identifierType = null; + String identifier = null; + switch (altPid.getProtocol()) { + case AbstractDOIProvider.DOI_PROTOCOL: + identifierType = AbstractDOIProvider.DOI_PROTOCOL.toUpperCase(); + identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); + break; + case HandlePidProvider.HDL_PROTOCOL: + identifierType = "Handle"; + identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); + break; + default: + // The AlternativePersistentIdentifier class isn't really ready for anything but + // doi or handle pids, but will add this as a default. + identifierType = ":unav"; + identifier = altPid.getAuthority() + altPid.getIdentifier(); + break; + } + attributes.put("alternativeIdentifierType", identifierType); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); + + } + for (DatasetFieldCompoundValue otherIdentifier : otherIdentifiers) { + String identifierType = null; + String identifier = null; + for (DatasetField subField : otherIdentifier.getChildDatasetFields()) { + identifierType = ":unav"; + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.otherIdAgency: + identifierType = subField.getValue(); + break; + case DatasetFieldConstant.otherIdValue: + identifier = subField.getValue(); + break; + } + } + attributes.put("alternativeIdentifierType", identifierType); + if (!StringUtils.isBlank(identifier)) { + alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); + + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); + } + } + if (alternatesWritten) { + xmlw.writeEndElement(); + } + } + private String generateRelatedIdentifiers(DvObject dvObject) { StringBuilder sb = new StringBuilder(); From d69bf414f3cc3b5fc8e0214b0c5c4fc6f7ec155e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 8 May 2024 13:19:56 -0400 Subject: [PATCH 073/360] more methods --- .../pidproviders/doi/XmlMetadataTemplate.java | 313 +++++++++++++----- 1 file changed, 223 insertions(+), 90 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 92bf7afd273..eb2465257a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -47,6 +47,7 @@ import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.export.DDIExporter; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; @@ -146,7 +147,6 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM } - /** * 3, Title(s) (with optional type sub-properties) (M) * @@ -802,132 +802,265 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) } } - private String generateRelatedIdentifiers(DvObject dvObject) { + /** + * 12, RelatedIdentifier (with type and relation type sub-properties) (R) + * + * @param xmlw The Steam writer + * @param dvObject the dataset/datafile + * @throws XMLStreamException + */ + private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + + boolean relatedIdentifiersWritten = false; + + Map attributes = new HashMap(); - StringBuilder sb = new StringBuilder(); - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; + if (dvObject instanceof Dataset dataset) { List relatedPublications = dataset.getLatestVersionForCopy().getRelatedPublications(); if (!relatedPublications.isEmpty()) { for (DatasetRelPublication relatedPub : relatedPublications) { + attributes.clear(); + String pubIdType = relatedPub.getIdType(); String identifier = relatedPub.getIdNumber(); + String url = relatedPub.getUrl(); /* * Note - with identifier and url fields, it's not clear that there's a single - * way those two fields are used for all identifier types In QDR, at this time, - * doi and isbn types always have the raw number in the identifier field, - * whereas there are examples where URLs are in the identifier or url fields. - * The code here addresses those practices and is not generic. + * way those two fields are used for all identifier types. The code here is + * ~best effort to interpret those fields. */ - if (pubIdType != null) { - switch (pubIdType) { - case "doi": - if (identifier != null && identifier.length() != 0) { - appendIdentifier(sb, "DOI", "IsSupplementTo", "doi:" + identifier); - } - break; - case "isbn": - if (identifier != null && identifier.length() != 0) { - appendIdentifier(sb, "ISBN", "IsSupplementTo", "ISBN:" + identifier); - } - break; - case "url": - if (identifier != null && identifier.length() != 0) { - appendIdentifier(sb, "URL", "IsSupplementTo", identifier); - } else { - String pubUrl = relatedPub.getUrl(); - if (pubUrl != null && pubUrl.length() > 0) { - appendIdentifier(sb, "URL", "IsSupplementTo", pubUrl); - } - } - break; - default: - if (identifier != null && identifier.length() != 0) { - if (pubIdType.equalsIgnoreCase("arXiv")) { - pubIdType = "arXiv"; - } else if (pubIdType.equalsIgnoreCase("handle")) { - // Initial cap required for handle - pubIdType = "Handle"; - } else if (!pubIdType.equals("bibcode")) { - pubIdType = pubIdType.toUpperCase(); - } - // For all others, do a generic attempt to match the identifier type to the - // datacite schema and send the raw identifier as the value - appendIdentifier(sb, pubIdType, "IsSupplementTo", identifier); - } - break; + pubIdType = getCanonicalPublicationType(pubIdType); + + // Prefer url if set, otherwise check identifier + String relatedIdentifier = url; + if (StringUtils.isBlank(relatedIdentifier)) { + relatedIdentifier = identifier; + } + // For types where we understand the protocol, get the canonical form + switch (pubIdType) { + case "DOI": + if (!relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http")) { + relatedIdentifier = "doi:" + relatedIdentifier; + } + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + } catch (IllegalArgumentException e) { + relatedIdentifier = null; } + break; + case "Handle": + if (!relatedIdentifier.startsWith("hdl:") || relatedIdentifier.startsWith("http")) { + relatedIdentifier = "hdl:" + relatedIdentifier; + } + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + } catch (IllegalArgumentException e) { + relatedIdentifier = null; + } + break; + case "URL": + break; + default: + + // For non-URL types, if a URL is given, split the string to get a schemeUri + try { + URL relatedUrl = new URL(relatedIdentifier); + String protocol = relatedUrl.getProtocol(); + String authority = relatedUrl.getAuthority(); + String site = String.format("%s://%s", protocol, authority); + relatedIdentifier = relatedIdentifier.substring(site.length()); + attributes.put("schemeURI", site); + } catch (MalformedURLException e) { + // Just an identifier + } + } - } else { - logger.info(relatedPub.getIdNumber() + relatedPub.getUrl() + relatedPub.getTitle()); + if (StringUtils.isNotBlank(relatedIdentifier)) { + // Still have a valid entry + attributes.put("relatedIdentifierType", pubIdType); + attributes.put("relationType", "IsSupplementTo"); + relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, relatedIdentifier); } } } - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - - List datafileIdentifiers = new ArrayList<>(); + attributes.clear(); + attributes.put("relationType", "HasPart"); for (DataFile dataFile : dataset.getFiles()) { - if (dataFile.getGlobalId() != null) { - if (sb.toString().isEmpty()) { - sb.append(""); + GlobalId pid = dataFile.getGlobalId(); + if (pid != null) { + String pubIdType = getCanonicalPublicationType(pid.getProtocol()); + if (pubIdType != null) { + attributes.put("relatedIdentifierType", pubIdType); + relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, pid.asRawIdentifier()); } - sb.append("" - + dataFile.getGlobalId() + ""); } } - - if (!sb.toString().isEmpty()) { - sb.append(""); - } } - } else if (dvObject.isInstanceofDataFile()) { - DataFile df = (DataFile) dvObject; - appendIdentifier(sb, "DOI", "IsPartOf", df.getOwner().getGlobalId().asString()); - if (sb.length() != 0) { - // Should always be true - sb.append(""); + } else if (dvObject instanceof DataFile df) { + GlobalId pid = df.getOwner().getGlobalId(); + if (pid != null) { + String pubIdType = getCanonicalPublicationType(pid.getProtocol()); + if (pubIdType != null) { + + attributes.clear(); + attributes.put("relationType", "IsPartOf"); + relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, pid.asRawIdentifier()); + } } } - return sb.toString(); + if (relatedIdentifiersWritten) { + xmlw.writeEndElement(); + } } - private void appendIdentifier(StringBuilder sb, String idType, String relationType, String identifier) { - if (sb.toString().isEmpty()) { - sb.append(""); + static HashMap relatedIdentifierTypeMap = new HashMap(); + + private static String getCanonicalPublicationType(String pubIdType) { + if (relatedIdentifierTypeMap.isEmpty()) { + relatedIdentifierTypeMap.put("ARK".toLowerCase(), "ARK"); + relatedIdentifierTypeMap.put("arXiv", "arXiv"); + relatedIdentifierTypeMap.put("bibcode".toLowerCase(), "bibcode"); + relatedIdentifierTypeMap.put("DOI".toLowerCase(), "DOI"); + relatedIdentifierTypeMap.put("EAN13".toLowerCase(), "EAN13"); + relatedIdentifierTypeMap.put("EISSN".toLowerCase(), "EISSN"); + relatedIdentifierTypeMap.put("Handle".toLowerCase(), "Handle"); + relatedIdentifierTypeMap.put("IGSN".toLowerCase(), "IGSN"); + relatedIdentifierTypeMap.put("ISBN".toLowerCase(), "ISBN"); + relatedIdentifierTypeMap.put("ISSN".toLowerCase(), "ISSN"); + relatedIdentifierTypeMap.put("ISTC".toLowerCase(), "ISTC"); + relatedIdentifierTypeMap.put("LISSN".toLowerCase(), "LISSN"); + relatedIdentifierTypeMap.put("LSID".toLowerCase(), "LSID"); + relatedIdentifierTypeMap.put("PISSN".toLowerCase(), "PISSN"); + relatedIdentifierTypeMap.put("PMID".toLowerCase(), "PMID"); + relatedIdentifierTypeMap.put("PURL".toLowerCase(), "PURL"); + relatedIdentifierTypeMap.put("UPC".toLowerCase(), "UPC"); + relatedIdentifierTypeMap.put("URL".toLowerCase(), "URL"); + relatedIdentifierTypeMap.put("URN".toLowerCase(), "URN"); + relatedIdentifierTypeMap.put("WOS".toLowerCase(), "WOS"); + // Add entry for Handle protocol so this can be used with GlobalId/getProtocol() + relatedIdentifierTypeMap.put("hdl".toLowerCase(), "Handle"); } - sb.append("" + identifier + ""); + return relatedIdentifierTypeMap.get(pubIdType); } - public void generateFileIdentifiers(DvObject dvObject) { + private void writeSize(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + // sizes -> size + boolean sizesWritten = false; + List dataFiles = new ArrayList(); - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; + if (dvObject instanceof Dataset dataset) { + dataFiles = dataset.getFiles(); + } else if (dvObject instanceof DataFile df) { + dataFiles.add(df); + } + if (dataFiles != null && !dataFiles.isEmpty()) { + for (DataFile dataFile : dataFiles) { + Long size = dataFile.getFilesize(); + if (size != -1) { + sizesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "sizes", sizesWritten); + XmlWriterUtil.writeFullElement(xmlw, "size", size.toString()); + } + } + } + if (sizesWritten) { + xmlw.writeEndElement(); + } - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { + } - List datafileIdentifiers = new ArrayList<>(); - for (DataFile dataFile : dataset.getFiles()) { - datafileIdentifiers.add(dataFile.getIdentifier()); - // int x = xmlMetadata.indexOf("") - 1; - // xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", - // dataFile.getIdentifier()); - // xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" - // + template.substring(x, template.length() - 1); + private void writeFormats(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + boolean formatsWritten = false; + List dataFiles = new ArrayList(); + + if (dvObject instanceof Dataset dataset) { + dataFiles = dataset.getFiles(); + } else if (dvObject instanceof DataFile df) { + dataFiles.add(df); + } + if (dataFiles != null && !dataFiles.isEmpty()) { + for (DataFile dataFile : dataFiles) { + String format = dataFile.getContentType(); + if (StringUtils.isNotBlank(format)) { + formatsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "formats", formatsWritten); + XmlWriterUtil.writeFullElement(xmlw, "format", format); } + /* Should original formats be sent? What about original sizes above? + if(dataFile.isTabularData()) { + String originalFormat = dataFile.getOriginalFileFormat(); + if(StringUtils.isNotBlank(originalFormat)) { + XmlWriterUtil.writeFullElement(xmlw, "format", format); + } + }*/ + } + } + if (formatsWritten) { + xmlw.writeEndElement(); + } + + } - } else { - // xmlMetadata = xmlMetadata.replace( - // "${relatedIdentifier}", - // ""); + private void writeVersion(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + Dataset d = null; + if(dvObject instanceof Dataset) { + d = (Dataset) dvObject; + } else if (dvObject instanceof DataFile) { + d = ((DataFile) dvObject).getOwner(); + } + if(d !=null) { + DatasetVersion dv = d.getLatestVersionForCopy(); + String version = dv.getFriendlyVersionNumber(); + if (StringUtils.isNotBlank(version)) { + XmlWriterUtil.writeFullElement(xmlw, "version", version); } } + } + private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) { + // rightsList -> rights with rightsURI attribute + xmlw.writeStartElement("rightsList"); // + + // set terms from the info:eu-repo-Access-Terms vocabulary + writeRightsHeader(xmlw, language); + boolean restrict = false; + boolean closed = false; + + if (datasetVersionDTO.isFileAccessRequest()) { + restrict = true; + } + if (datasetVersionDTO.getFiles() != null) { + for (int i = 0; i < datasetVersionDTO.getFiles().size(); i++) { + if (datasetVersionDTO.getFiles().get(i).isRestricted()) { + closed = true; + break; + } + } + } + + if (restrict && closed) { + xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/restrictedAccess"); + } else if (!restrict && closed) { + xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/closedAccess"); + } else { + xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/openAccess"); + } + xmlw.writeEndElement(); // + + writeRightsHeader(xmlw, language); + if (datasetVersionDTO.getLicense() != null) { + xmlw.writeAttribute("rightsURI", datasetVersionDTO.getLicense().getUri()); + xmlw.writeCharacters(datasetVersionDTO.getLicense().getName()); + } + xmlw.writeEndElement(); // + xmlw.writeEndElement(); // + } } \ No newline at end of file From 04b367f641fe8e8da77c8eceafd7a012985f9a1f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 15 May 2024 10:41:07 -0400 Subject: [PATCH 074/360] only one field to look for --- src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 943693355a3..d723cf3d528 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1344,6 +1344,7 @@ public List getGeographicCoverage() { } } + break; } return geoCoverages; } From 003431dde79bc7b80077c1aa6d0998329e85f4e3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 15 May 2024 10:41:30 -0400 Subject: [PATCH 075/360] use common util method --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 0c861cb6c09..c0e3057696a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1715,12 +1715,12 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f // range.getBeginValueType().getName().equals(DB_VAR_RANGE_TYPE_POINT)) { if (range.getBoolean("hasBeginValueType") && range.getBoolean("isBeginValueTypePoint")) { if (range.containsKey("beginValue")) { - invalrngAdded = XmlWriterUtil.checkParentElement(xmlw, "invalrng", invalrngAdded); + invalrngAdded = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "invalrng", invalrngAdded); xmlw.writeEmptyElement("item"); XmlWriterUtil.writeAttribute(xmlw, "VALUE", range.getString("beginValue")); } } else { - invalrngAdded = XmlWriterUtil.checkParentElement(xmlw, "invalrng", invalrngAdded); + invalrngAdded = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "invalrng", invalrngAdded); xmlw.writeEmptyElement("range"); if (range.getBoolean("hasBeginValueType") && range.containsKey("beginValue")) { if (range.getBoolean("isBeginValueTypeMin")) { From fea2f5e01d9a9e3d37f1714e3aaba8dc32f84ab0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 15 May 2024 10:42:29 -0400 Subject: [PATCH 076/360] access rights descriptions, geolocations, funding refs --- .../pidproviders/doi/XmlMetadataTemplate.java | 297 +++++++++++++++--- 1 file changed, 249 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index eb2465257a1..be55b7a4837 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -41,16 +41,20 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; +import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.export.DDIExporter; +import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.xml.XmlPrinter; @@ -117,34 +121,6 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM writeDescriptions(xmlw, dvObject); writeGeoLocations(xmlw, dvObject); writeFundingReferences(xmlw, dvObject); - - StringBuilder contributorsElement = new StringBuilder(); - if (doiMetadata.getContacts() != null) { - for (String[] contact : doiMetadata.getContacts()) { - if (!contact[0].isEmpty()) { - contributorsElement.append("" - + StringEscapeUtils.escapeXml10(contact[0]) + ""); - if (!contact[1].isEmpty()) { - contributorsElement.append("" + StringEscapeUtils.escapeXml10(contact[1]) + ""); - } - contributorsElement.append(""); - } - } - } - - if (doiMetadata.getProducers() != null) { - for (String[] producer : doiMetadata.getProducers()) { - contributorsElement.append("" + StringEscapeUtils.escapeXml10(producer[0]) - + ""); - if (!producer[1].isEmpty()) { - contributorsElement.append("" + StringEscapeUtils.escapeXml10(producer[1]) + ""); - } - contributorsElement.append(""); - } - } - - String relIdentifiers = generateRelatedIdentifiers(dvObject); - } /** @@ -1025,42 +1001,267 @@ private void writeVersion(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStr } - private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) { + private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { // rightsList -> rights with rightsURI attribute xmlw.writeStartElement("rightsList"); // // set terms from the info:eu-repo-Access-Terms vocabulary - writeRightsHeader(xmlw, language); - boolean restrict = false; + xmlw.writeStartElement("rights"); // + DatasetVersion dv = null; boolean closed = false; + if (dvObject instanceof Dataset d) { + dv = d.getLatestVersionForCopy(); + closed = dv.isHasRestrictedFile(); + } else if (dvObject instanceof DataFile df) { + dv = df.getOwner().getLatestVersionForCopy(); - if (datasetVersionDTO.isFileAccessRequest()) { - restrict = true; - } - if (datasetVersionDTO.getFiles() != null) { - for (int i = 0; i < datasetVersionDTO.getFiles().size(); i++) { - if (datasetVersionDTO.getFiles().get(i).isRestricted()) { - closed = true; - break; - } - } + closed = df.isRestricted(); } + TermsOfUseAndAccess terms = dv.getTermsOfUseAndAccess(); + boolean requestsAllowed = terms.isFileAccessRequest(); + License license = terms.getLicense(); - if (restrict && closed) { + if (requestsAllowed && closed) { xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/restrictedAccess"); - } else if (!restrict && closed) { + } else if (!requestsAllowed && closed) { xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/closedAccess"); } else { xmlw.writeAttribute("rightsURI", "info:eu-repo/semantics/openAccess"); } xmlw.writeEndElement(); // + xmlw.writeStartElement("rights"); // - writeRightsHeader(xmlw, language); - if (datasetVersionDTO.getLicense() != null) { - xmlw.writeAttribute("rightsURI", datasetVersionDTO.getLicense().getUri()); - xmlw.writeCharacters(datasetVersionDTO.getLicense().getName()); + if (license != null) { + xmlw.writeAttribute("rightsURI", license.getUri().toString()); + xmlw.writeCharacters(license.getName()); + } else { + xmlw.writeAttribute("rightsURI", DatasetUtil.getLicenseURI(dv)); + xmlw.writeCharacters(BundleUtil.getStringFromBundle("license.custom.description")); + ; } xmlw.writeEndElement(); // - xmlw.writeEndElement(); // + xmlw.writeEndElement(); // + } + + private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + // descriptions -> description with descriptionType attribute + boolean descriptionsWritten = false; + List descriptions = null; + DatasetVersion dv = null; + + if (dvObject instanceof Dataset d) { + dv = d.getLatestVersionForCopy(); + dv.getDescriptions(); + } else if (dvObject instanceof DataFile df) { + String description = df.getDescription(); + if (description != null) { + descriptions = new ArrayList(); + descriptions.add(description); + } + } + Map attributes = new HashMap(); + attributes.put("descriptionType", "Abstract"); + for (String description : descriptions) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, description); + ; + } + + if (dv != null) { + List dsfs = dv.getDatasetFields(); + + for (DatasetField dsf : dsfs) { + + switch (dsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.software: + attributes.clear(); + attributes.put("descriptionType", "TechnicalInfo"); + List dsfcvs = dsf.getDatasetFieldCompoundValues(); + for (DatasetFieldCompoundValue dsfcv : dsfcvs) { + + String softwareName = null; + String softwareVersion = null; + List childDsfs = dsfcv.getChildDatasetFields(); + for (DatasetField childDsf : childDsfs) { + if (DatasetFieldConstant.softwareName.equals(childDsf.getDatasetFieldType().getName())) { + softwareName = childDsf.getValue(); + } else if (DatasetFieldConstant.softwareVersion.equals(childDsf.getDatasetFieldType().getName())) { + softwareVersion = childDsf.getValue(); + } + } + if (StringUtils.isNotBlank(softwareName)) { + if (StringUtils.isNotBlank(softwareVersion)) { + } + softwareName = softwareName + ", " + softwareVersion; + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, softwareName); + } + } + break; + case DatasetFieldConstant.originOfSources: + case DatasetFieldConstant.characteristicOfSources: + case DatasetFieldConstant.accessToSources: + attributes.clear(); + attributes.put("descriptionType", "Methods"); + String method = dsf.getValue(); + if (StringUtils.isNotBlank(method)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, method); + + } + break; + case DatasetFieldConstant.series: + attributes.clear(); + attributes.put("descriptionType", "SeriesInformation"); + dsfcvs = dsf.getDatasetFieldCompoundValues(); + for (DatasetFieldCompoundValue dsfcv : dsfcvs) { + List childDsfs = dsfcv.getChildDatasetFields(); + for (DatasetField childDsf : childDsfs) { + + if (DatasetFieldConstant.seriesInformation.equals(childDsf.getDatasetFieldType().getName())) { + String seriesInformation = childDsf.getValue(); + if (StringUtils.isNotBlank(seriesInformation)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, seriesInformation); + } + break; + } + } + } + break; + case DatasetFieldConstant.notesText: + attributes.clear(); + attributes.put("descriptionType", "Other"); + String notesText = dsf.getValue(); + if (StringUtils.isNotBlank(notesText)) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, notesText); + } + break; + + } + } + + } + + if (descriptionsWritten) { + xmlw.writeEndElement(); // + } + } + + private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + if (dvObject instanceof Dataset d) { + boolean geoLocationsWritten = false; + DatasetVersion dv = d.getLatestVersionForCopy(); + + List places = dv.getGeographicCoverage(); + if (places != null && !places.isEmpty()) { + // geoLocationPlace + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); + + for (String[] place : places) { + ArrayList placeList = new ArrayList(); + for (String placePart : place) { + placeList.add(placePart); + } + XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", Strings.join(placeList, ", ")); + } + } + boolean boundingBoxFound = false; + boolean productionPlaceFound = false; + for (DatasetField dsf : dv.getDatasetFields()) { + switch (dsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.geographicBoundingBox: + boundingBoxFound = true; + for (DatasetFieldCompoundValue dsfcv : dsf.getDatasetFieldCompoundValues()) { + List childDsfs = dsfcv.getChildDatasetFields(); + String nLatitude = null; + String sLatitude = null; + String eLongitude = null; + String wLongitude = null; + for (DatasetField childDsf : childDsfs) { + switch (childDsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.northLatitude: + nLatitude = childDsf.getValue(); + break; + case DatasetFieldConstant.southLatitude: + sLatitude = childDsf.getValue(); + break; + case DatasetFieldConstant.eastLongitude: + eLongitude = childDsf.getValue(); + break; + case DatasetFieldConstant.westLongitude: + wLongitude = childDsf.getValue(); + + } + } + if (StringUtils.isNoneBlank(wLongitude, eLongitude, nLatitude, sLatitude)) { + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); + if (wLongitude.equals(eLongitude) && nLatitude.equals(sLatitude)) { + // A point + xmlw.writeStartElement("geoLocationPoint"); + XmlWriterUtil.writeFullElement(xmlw, "pointLongitude", eLongitude); + XmlWriterUtil.writeFullElement(xmlw, "pointLatitude", sLatitude); + xmlw.writeEndElement(); + } else { + // A box + xmlw.writeStartElement("geoLocationBox"); + XmlWriterUtil.writeFullElement(xmlw, "westBoundLongitude", wLongitude); + XmlWriterUtil.writeFullElement(xmlw, "eastBoundLongitude", eLongitude); + XmlWriterUtil.writeFullElement(xmlw, "southBoundLatitude", sLatitude); + XmlWriterUtil.writeFullElement(xmlw, "northBoundLatitude", nLatitude); + xmlw.writeEndElement(); + + } + } + } + case DatasetFieldConstant.productionPlace: + productionPlaceFound = true; + // geoLocationPlace + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); + List prodPlaces = dsf.getValues(); + for (String prodPlace : prodPlaces) { + XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", prodPlace); + } + break; + } + if (boundingBoxFound && productionPlaceFound) { + break; + } + } + if (geoLocationsWritten) { + xmlw.writeEndElement(); // + } + } + + } + + + private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + // fundingReferences -> fundingReference -> funderName, awardNumber + boolean fundingReferenceWritten = false; + DatasetVersion dv = null; + if (dvObject instanceof Dataset d) { + dv = d.getLatestVersionForCopy(); + } else if (dvObject instanceof DataFile df) { + dv = df.getOwner().getLatestVersionForCopy(); + } + if (dv != null) { + List funders = dv.getFunders(); + if (!funders.isEmpty()) { + + for (String funder : funders) { + if (!StringUtils.isBlank(funder)) { + fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); + xmlw.writeStartElement("fundingReference"); // + XmlWriterUtil.writeFullElement(xmlw, "funderName", funder); + xmlw.writeEndElement(); // + } + } + if (fundingReferenceWritten) { + xmlw.writeEndElement(); // + } + } + } } } \ No newline at end of file From 3c52b6a2031a55c8840948681930a1824b02820b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 17 May 2024 15:36:18 -0400 Subject: [PATCH 077/360] altTitles npe --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index be55b7a4837..a2c744be2ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -138,7 +138,7 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String language) throws XMLStreamException { String title = doiMetadata.getTitle(); String subTitle = null; - List altTitles = null; + List altTitles = new ArrayList<>(); // Only Datasets can have a subtitle or alternative titles if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersion(); From bab2a0d270b766916dfd9578fd25ab05332f0958 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sat, 18 May 2024 12:25:07 -0400 Subject: [PATCH 078/360] fixes and test --- .../pidproviders/doi/XmlMetadataTemplate.java | 64 ++++++----- .../doi/datacite/XmlMetadataTemplateTest.java | 108 ++++++++++++++++++ 2 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index a2c744be2ed..e9b7b0faa26 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -21,6 +21,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; + import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -84,6 +85,7 @@ public String generateXML(DvObject dvObject) { generateXML(dvObject, outputStream); String xml = outputStream.toString(); + logger.info(xml); return XmlPrinter.prettyPrintXml(xml); } catch (XMLStreamException | IOException e) { logger.severe("Unable to generate DataCite XML for DOI: " + dvObject.getGlobalId().asString() + " : " + e.getMessage()); @@ -98,10 +100,11 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM String metadataLanguage = null; // when set, otherwise = language? XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); xmlw.writeStartElement("resource"); + xmlw.writeDefaultNamespace(XML_NAMESPACE); xmlw.writeAttribute("xmlns:xsi", XML_XSI); xmlw.writeAttribute("xsi:schemaLocation", XML_SCHEMA_LOCATION); - + writeIdentifier(xmlw, dvObject); writeCreators(xmlw, doiMetadata.getAuthors()); writeTitles(xmlw, dvObject, language); @@ -121,6 +124,8 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM writeDescriptions(xmlw, dvObject); writeGeoLocations(xmlw, dvObject); writeFundingReferences(xmlw, dvObject); + xmlw.writeEndElement(); + xmlw.flush(); } /** @@ -726,32 +731,34 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) } } } - if (!altPids.isEmpty()) { + + if (altPids != null && !altPids.isEmpty()) { alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); - } - for (AlternativePersistentIdentifier altPid : altPids) { - String identifierType = null; - String identifier = null; - switch (altPid.getProtocol()) { - case AbstractDOIProvider.DOI_PROTOCOL: - identifierType = AbstractDOIProvider.DOI_PROTOCOL.toUpperCase(); - identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); - break; - case HandlePidProvider.HDL_PROTOCOL: - identifierType = "Handle"; - identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); - break; - default: - // The AlternativePersistentIdentifier class isn't really ready for anything but - // doi or handle pids, but will add this as a default. - identifierType = ":unav"; - identifier = altPid.getAuthority() + altPid.getIdentifier(); - break; - } - attributes.put("alternativeIdentifierType", identifierType); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); + for (AlternativePersistentIdentifier altPid : altPids) { + String identifierType = null; + String identifier = null; + switch (altPid.getProtocol()) { + case AbstractDOIProvider.DOI_PROTOCOL: + identifierType = AbstractDOIProvider.DOI_PROTOCOL.toUpperCase(); + identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); + break; + case HandlePidProvider.HDL_PROTOCOL: + identifierType = "Handle"; + identifier = altPid.getAuthority() + "/" + altPid.getIdentifier(); + break; + default: + // The AlternativePersistentIdentifier class isn't really ready for anything but + // doi or handle pids, but will add this as a default. + identifierType = ":unav"; + identifier = altPid.getAuthority() + altPid.getIdentifier(); + break; + } + attributes.put("alternativeIdentifierType", identifierType); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); + } } + for (DatasetFieldCompoundValue otherIdentifier : otherIdentifiers) { String identifierType = null; String identifier = null; @@ -1061,10 +1068,11 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws X } Map attributes = new HashMap(); attributes.put("descriptionType", "Abstract"); - for (String description : descriptions) { - descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, description); - ; + if (descriptions != null) { + for (String description : descriptions) { + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, description); + } } if (dv != null) { diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java new file mode 100644 index 00000000000..e576398a474 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -0,0 +1,108 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.DoiMetadata; +import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; + +@ExtendWith(MockitoExtension.class) +@LocalJvmSettings +@JvmSetting(key = JvmSettings.SITE_URL, value = "https://example.com") + +public class XmlMetadataTemplateTest { + + static DataverseServiceBean dataverseSvc; + static SettingsServiceBean settingsSvc; + static PidProviderFactoryBean pidService; + static final String DEFAULT_NAME = "LibraScholar"; + + @BeforeAll + public static void setupMocks() { + dataverseSvc = Mockito.mock(DataverseServiceBean.class); + settingsSvc = Mockito.mock(SettingsServiceBean.class); + BrandingUtil.injectServices(dataverseSvc, settingsSvc); + + // initial values (needed here for other tests where this method is reused!) + Mockito.when(settingsSvc.getValueForKey(SettingsServiceBean.Key.InstallationName)).thenReturn(DEFAULT_NAME); + Mockito.when(dataverseSvc.getRootDataverseName()).thenReturn(DEFAULT_NAME); + + pidService = Mockito.mock(PidProviderFactoryBean.class); + Mockito.when(pidService.isGlobalIdLocallyUnique(any(GlobalId.class))).thenReturn(true); + Mockito.when(pidService.getProducer()).thenReturn("RootDataverse"); + + } + + /** + */ + @Test + public void testDataCiteXMLCreation() throws IOException { + DoiMetadata doiMetadata = new DoiMetadata(); + doiMetadata.setTitle("A Title"); + List creators = new ArrayList(); + creators.add("Alice"); + creators.add("Bob"); + doiMetadata.setCreators(creators); + doiMetadata.setPublisher("Dataverse"); + XmlMetadataTemplate template = new XmlMetadataTemplate(doiMetadata); + + Dataset d = new Dataset(); + GlobalId doi = new GlobalId("doi", "10.5072", "FK2/ABCDEF", null, null, null); + d.setGlobalId(doi); + DatasetVersion dv = new DatasetVersion(); + TermsOfUseAndAccess toa = new TermsOfUseAndAccess(); + toa.setTermsOfUse("Some terms"); + dv.setTermsOfUseAndAccess(toa); + dv.setDataset(d); + DatasetFieldType primitiveDSFType = new DatasetFieldType(DatasetFieldConstant.title, + DatasetFieldType.FieldType.TEXT, false); + DatasetField testDatasetField = new DatasetField(); + + dv.setVersionState(VersionState.DRAFT); + + testDatasetField.setDatasetVersion(dv); + testDatasetField.setDatasetFieldType(primitiveDSFType); + testDatasetField.setSingleValue("First Title"); + List fields = new ArrayList<>(); + fields.add(testDatasetField); + dv.setDatasetFields(fields); + ArrayList dsvs = new ArrayList<>(); + dsvs.add(0, dv); + d.setVersions(dsvs); + + String xml = template.generateXML(d); + System.out.println("Output is " + xml); + + } + +} From 3cca63d2f6ff4052852876d9ccfe52424d2da615 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 20 May 2024 14:55:46 -0400 Subject: [PATCH 079/360] fix for empty rel pub entry --- .../pidproviders/doi/XmlMetadataTemplate.java | 179 +++++++++--------- 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index e9b7b0faa26..8725feca546 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -21,7 +21,6 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; - import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -97,14 +96,14 @@ public String generateXML(DvObject dvObject) { private void generateXML(DvObject dvObject, OutputStream outputStream) throws XMLStreamException { // Could/should use dataset metadata language for metadata from DvObject itself? String language = null; // machine locale? e.g. for Publisher which is global - String metadataLanguage = null; // when set, otherwise = language? + String metadataLanguage = null; // when set, otherwise = language? XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); xmlw.writeStartElement("resource"); - + xmlw.writeDefaultNamespace(XML_NAMESPACE); xmlw.writeAttribute("xmlns:xsi", XML_XSI); xmlw.writeAttribute("xsi:schemaLocation", XML_SCHEMA_LOCATION); - + writeIdentifier(xmlw, dvObject); writeCreators(xmlw, doiMetadata.getAuthors()); writeTitles(xmlw, dvObject, language); @@ -242,7 +241,7 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList) String nameIdentifierScheme = null; if (StringUtils.isNotBlank(author.getIdValue()) && StringUtils.isNotBlank(author.getIdType())) { nameIdentifier = author.getIdValue(); - if(nameIdentifier != null) { + if (nameIdentifier != null) { // Normalizes to the URL form of the identifier, returns null if the identifier // is not valid given the type nameIdentifier = author.getIdentifierAsUrl(); @@ -392,14 +391,16 @@ private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLSt } /** - * 7, Contributor (with optional given name, family name, name identifier - * and affiliation sub-properties) + * 7, Contributor (with optional given name, family name, name identifier and + * affiliation sub-properties) * * @see #writeContributorElement(javax.xml.stream.XMLStreamWriter, - * java.lang.String, java.lang.String, java.lang.String) + * java.lang.String, java.lang.String, java.lang.String) * - * @param xmlw The stream writer - * @param dvObject The Dataset/DataFile + * @param xmlw + * The stream writer + * @param dvObject + * The Dataset/DataFile * @throws XMLStreamException */ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { @@ -410,11 +411,11 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X List compoundContributors = new ArrayList(); // Dataset Subject= Dataverse subject, keyword, and/or topic classification // fields - //ToDo Include for files? - /*if(dvObject instanceof DataFile df) { - dvObject = df.getOwner(); - }*/ - + // ToDo Include for files? + /* + * if(dvObject instanceof DataFile df) { dvObject = df.getOwner(); } + */ + if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersionForCopy(); for (DatasetField dsf : dv.getDatasetFields()) { @@ -433,8 +434,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } } } - - + for (DatasetFieldCompoundValue producerFieldValue : compoundProducers) { String producer = null; String affiliation = null; @@ -457,7 +457,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } } - + for (DatasetFieldCompoundValue distributorFieldValue : compoundDistributors) { String distributor = null; String affiliation = null; @@ -517,7 +517,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X break; } } - // QDR - doesn't have Funder in the contributor type list. + // QDR - doesn't have Funder in the contributor type list. // Using a string isn't i18n if (StringUtils.isNotBlank(contributor) && !StringUtils.equalsIgnoreCase("Funder", contributorType)) { contributorsCreated = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "contributors", contributorsCreated); @@ -526,7 +526,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X } } - + if (contributorsCreated) { xmlw.writeEndElement(); } @@ -568,7 +568,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin logger.warning("DatasetAuthor.getIdentifierAsUrl returned a Malformed URL: " + nameIdentifier); } } - + if (StringUtils.isNotBlank(affiliation)) { attributeMap.clear(); if (affiliation.startsWith("https://ror.org/")) { @@ -584,8 +584,10 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin /** * 8, Date (with type sub-property) (R) * - * @param xmlw The Steam writer - * @param dvObject The dataset/datafile + * @param xmlw + * The Steam writer + * @param dvObject + * The dataset/datafile * @throws XMLStreamException */ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { @@ -668,15 +670,16 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } } - // 9, Language (MA), language private void writeLanguage(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { - //Currently not supported. Spec indicates one 'primary' language. Could send the first entry in DatasetFieldConstant.language or send iff there is only one entry, and/or default to the machine's default lang? + // Currently not supported. Spec indicates one 'primary' language. Could send + // the first entry in DatasetFieldConstant.language or send iff there is only + // one entry, and/or default to the machine's default lang? return; } - - // 10, ResourceType (with mandatory general type - // description sub- property) (M) + + // 10, ResourceType (with mandatory general type + // description sub- property) (M) private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { List kindOfDataValues = new ArrayList(); Map attributes = new HashMap(); @@ -711,8 +714,10 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X /** * 11 AlternateIdentifier (with type sub-property) (O) * - * @param xmlw The Steam writer - * @param dvObject The dataset/datafile + * @param xmlw + * The Steam writer + * @param dvObject + * The dataset/datafile * @throws XMLStreamException */ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { @@ -731,7 +736,7 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) } } } - + if (altPids != null && !altPids.isEmpty()) { alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); for (AlternativePersistentIdentifier altPid : altPids) { @@ -788,8 +793,10 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) /** * 12, RelatedIdentifier (with type and relation type sub-properties) (R) * - * @param xmlw The Steam writer - * @param dvObject the dataset/datafile + * @param xmlw + * The Steam writer + * @param dvObject + * the dataset/datafile * @throws XMLStreamException */ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { @@ -813,6 +820,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th * way those two fields are used for all identifier types. The code here is * ~best effort to interpret those fields. */ + pubIdType = getCanonicalPublicationType(pubIdType); // Prefer url if set, otherwise check identifier @@ -821,49 +829,52 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th relatedIdentifier = identifier; } // For types where we understand the protocol, get the canonical form - switch (pubIdType) { - case "DOI": - if (!relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http")) { - relatedIdentifier = "doi:" + relatedIdentifier; - } - try { - GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); - relatedIdentifier = pid.asRawIdentifier(); - } catch (IllegalArgumentException e) { - relatedIdentifier = null; - } - break; - case "Handle": - if (!relatedIdentifier.startsWith("hdl:") || relatedIdentifier.startsWith("http")) { - relatedIdentifier = "hdl:" + relatedIdentifier; - } - try { - GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); - relatedIdentifier = pid.asRawIdentifier(); - } catch (IllegalArgumentException e) { - relatedIdentifier = null; - } - break; - case "URL": - break; - default: - - // For non-URL types, if a URL is given, split the string to get a schemeUri - try { - URL relatedUrl = new URL(relatedIdentifier); - String protocol = relatedUrl.getProtocol(); - String authority = relatedUrl.getAuthority(); - String site = String.format("%s://%s", protocol, authority); - relatedIdentifier = relatedIdentifier.substring(site.length()); - attributes.put("schemeURI", site); - } catch (MalformedURLException e) { - // Just an identifier + if (pubIdType != null) { + switch (pubIdType) { + case "DOI": + if (!relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http")) { + relatedIdentifier = "doi:" + relatedIdentifier; + } + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + } catch (IllegalArgumentException e) { + relatedIdentifier = null; + } + break; + case "Handle": + if (!relatedIdentifier.startsWith("hdl:") || relatedIdentifier.startsWith("http")) { + relatedIdentifier = "hdl:" + relatedIdentifier; + } + try { + GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); + relatedIdentifier = pid.asRawIdentifier(); + } catch (IllegalArgumentException e) { + relatedIdentifier = null; + } + break; + case "URL": + break; + default: + + // For non-URL types, if a URL is given, split the string to get a schemeUri + try { + URL relatedUrl = new URL(relatedIdentifier); + String protocol = relatedUrl.getProtocol(); + String authority = relatedUrl.getAuthority(); + String site = String.format("%s://%s", protocol, authority); + relatedIdentifier = relatedIdentifier.substring(site.length()); + attributes.put("schemeURI", site); + } catch (MalformedURLException e) { + // Just an identifier + } } } - if (StringUtils.isNotBlank(relatedIdentifier)) { // Still have a valid entry - attributes.put("relatedIdentifierType", pubIdType); + if (pubIdType != null) { + attributes.put("relatedIdentifierType", pubIdType); + } attributes.put("relationType", "IsSupplementTo"); relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, relatedIdentifier); @@ -905,7 +916,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th static HashMap relatedIdentifierTypeMap = new HashMap(); - + private static String getCanonicalPublicationType(String pubIdType) { if (relatedIdentifierTypeMap.isEmpty()) { relatedIdentifierTypeMap.put("ARK".toLowerCase(), "ARK"); @@ -976,13 +987,12 @@ private void writeFormats(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStr formatsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "formats", formatsWritten); XmlWriterUtil.writeFullElement(xmlw, "format", format); } - /* Should original formats be sent? What about original sizes above? - if(dataFile.isTabularData()) { - String originalFormat = dataFile.getOriginalFileFormat(); - if(StringUtils.isNotBlank(originalFormat)) { - XmlWriterUtil.writeFullElement(xmlw, "format", format); - } - }*/ + /* + * Should original formats be sent? What about original sizes above? + * if(dataFile.isTabularData()) { String originalFormat = + * dataFile.getOriginalFileFormat(); if(StringUtils.isNotBlank(originalFormat)) + * { XmlWriterUtil.writeFullElement(xmlw, "format", format); } } + */ } } if (formatsWritten) { @@ -993,19 +1003,19 @@ private void writeFormats(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStr private void writeVersion(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { Dataset d = null; - if(dvObject instanceof Dataset) { + if (dvObject instanceof Dataset) { d = (Dataset) dvObject; } else if (dvObject instanceof DataFile) { d = ((DataFile) dvObject).getOwner(); } - if(d !=null) { + if (d != null) { DatasetVersion dv = d.getLatestVersionForCopy(); - String version = dv.getFriendlyVersionNumber(); + String version = dv.getFriendlyVersionNumber(); if (StringUtils.isNotBlank(version)) { XmlWriterUtil.writeFullElement(xmlw, "version", version); } } - + } private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { @@ -1244,7 +1254,6 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X } - private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { // fundingReferences -> fundingReference -> funderName, awardNumber boolean fundingReferenceWritten = false; From 30c80a9a5a27c51d8ca8130375d358aadb447a4f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 20 May 2024 15:32:13 -0400 Subject: [PATCH 080/360] bugs: remove bad nesting, dupe values --- .../pidproviders/doi/XmlMetadataTemplate.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8725feca546..74da57094c4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -158,20 +158,22 @@ private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String languag if (StringUtils.isNotBlank(title) || StringUtils.isNotBlank(subTitle) || (altTitles != null && !String.join("", altTitles).isBlank())) { xmlw.writeStartElement("titles"); - XmlWriterUtil.writeFullElement(xmlw, "title", title, language); - + if (StringUtils.isNotBlank(title)) { + XmlWriterUtil.writeFullElement(xmlw, "title", title, language); + } Map attributes = new HashMap(); - attributes.put("titleType", "Subtitle"); - - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, title); - - attributes.clear(); - attributes.put("titleType", "AlternativeTitle"); - for (String altTitle : altTitles) { - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, altTitle); + if (StringUtils.isNotBlank(subTitle)) { + attributes.put("titleType", "Subtitle"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, subTitle); + } + if ((altTitles != null && !String.join("", altTitles).isBlank())) { + attributes.clear(); + attributes.put("titleType", "AlternativeTitle"); + for (String altTitle : altTitles) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "title", attributes, altTitle); + } } - xmlw.writeEndElement(); } } @@ -250,12 +252,9 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList) } if (StringUtils.isNotBlank(creatorName)) { - xmlw.writeStartElement("creator"); // JsonObject creatorObj = PersonOrOrgUtil.getPersonOrOrganization(creatorName, false, StringUtils.containsIgnoreCase(nameIdentifierScheme, "orcid")); - writeEntityElements(xmlw, "creator", null, creatorObj, affiliation, nameIdentifier, nameIdentifierScheme); - xmlw.writeEndElement(); // } else { @@ -693,21 +692,21 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X kindOfDataValues = dsf.getControlledVocabularyValues(); break; } - - if (kindOfDataValues.isEmpty()) { - // Write an attribute only element if there are no kindOfData values. - xmlw.writeStartElement("resourceType"); - xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); - xmlw.writeEndElement(); - } else { - for (ControlledVocabularyValue kindOfDataValue : kindOfDataValues) { - String resourceType = kindOfDataValue.getStrValue(); - if (StringUtils.isNotBlank(resourceType)) { - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, resourceType); - } + } + if (kindOfDataValues.isEmpty()) { + // Write an attribute only element if there are no kindOfData values. + xmlw.writeStartElement("resourceType"); + xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); + xmlw.writeEndElement(); + } else { + for (ControlledVocabularyValue kindOfDataValue : kindOfDataValues) { + String resourceType = kindOfDataValue.getStrValue(); + if (StringUtils.isNotBlank(resourceType)) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, resourceType); } } } + } } From a2acdebbac758317b5d2d07fc1af01859f8bfa85 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 20 May 2024 16:54:02 -0400 Subject: [PATCH 081/360] add XML Validation to test --- .../iq/dataverse/util/xml/XmlValidator.java | 5 +++ .../doi/datacite/XmlMetadataTemplateTest.java | 45 +++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlValidator.java b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlValidator.java index 586ca50b6fd..cec64ab95b7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/xml/XmlValidator.java @@ -24,7 +24,12 @@ public class XmlValidator { private static final Logger logger = Logger.getLogger(XmlValidator.class.getCanonicalName()); public static boolean validateXmlSchema(String fileToValidate, URL schemaToValidateAgainst) throws MalformedURLException, SAXException, IOException { + Source xmlFile = new StreamSource(new File(fileToValidate)); + return validateXmlSchema(xmlFile, schemaToValidateAgainst); + } + + public static boolean validateXmlSchema(Source xmlFile, URL schemaToValidateAgainst) throws MalformedURLException, SAXException, IOException { SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); Schema schema = schemaFactory.newSchema(schemaToValidateAgainst); Validator validator = schema.newValidator(); diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java index e576398a474..c1bbc3bebc1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -1,9 +1,11 @@ package edu.harvard.iq.dataverse.pidproviders.doi.datacite; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetAuthor; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.DataverseServiceBean; @@ -19,12 +21,17 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import edu.harvard.iq.dataverse.util.xml.XmlValidator; import java.io.IOException; +import java.io.StringReader; +import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; +import javax.xml.transform.stream.StreamSource; + import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeAll; @@ -32,6 +39,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.xml.sax.SAXException; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -69,10 +77,32 @@ public static void setupMocks() { public void testDataCiteXMLCreation() throws IOException { DoiMetadata doiMetadata = new DoiMetadata(); doiMetadata.setTitle("A Title"); - List creators = new ArrayList(); - creators.add("Alice"); - creators.add("Bob"); - doiMetadata.setCreators(creators); + DatasetFieldType dft = new DatasetFieldType(DatasetFieldConstant.authorName, FieldType.TEXT, false); + dft.setDisplayFormat("#VALUE"); + DatasetFieldType dft2 = new DatasetFieldType(DatasetFieldConstant.authorAffiliation, FieldType.TEXT, false); + dft2.setDisplayFormat("#VALUE"); + DatasetAuthor alice = new DatasetAuthor(); + DatasetField df1 = new DatasetField(); + df1.setDatasetFieldType(dft); + df1.setSingleValue("Alice"); + alice.setName(df1); + DatasetField df2 = new DatasetField(); + df2.setDatasetFieldType(dft2); + df2.setSingleValue("Harvard University"); + alice.setAffiliation(df2); + DatasetAuthor bob = new DatasetAuthor(); + DatasetField df3 = new DatasetField(); + df3.setDatasetFieldType(dft); + df3.setSingleValue("Bob"); + bob.setName(df3); + DatasetField df4 = new DatasetField(); + df4.setDatasetFieldType(dft2); + df4.setSingleValue("QDR"); + bob.setAffiliation(df4); + List authors = new ArrayList<>(); + authors.add(alice); + authors.add(bob); + doiMetadata.setAuthors(authors); doiMetadata.setPublisher("Dataverse"); XmlMetadataTemplate template = new XmlMetadataTemplate(doiMetadata); @@ -102,6 +132,13 @@ public void testDataCiteXMLCreation() throws IOException { String xml = template.generateXML(d); System.out.println("Output is " + xml); + try { + StreamSource source = new StreamSource(new StringReader(xml)); + source.setSystemId("DataCite XML for test dataset"); + assertTrue(XmlValidator.validateXmlSchema(source, new URL("https://schema.datacite.org/meta/kernel-4/metadata.xsd"))); + } catch (SAXException e) { + System.out.println("Invalid schema: " + e.getMessage()); + } } From 3ec7a0b680ec5f04d650e830bb391c6be1f176f2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 23 May 2024 16:24:09 -0400 Subject: [PATCH 082/360] fix contributorType --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 74da57094c4..6e4d81d6248 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -535,7 +535,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin xmlw.writeStartElement(elementName); Map attributeMap = new HashMap(); if (StringUtils.isNotBlank(type)) { - attributeMap.put("contributorType", type); + xmlw.writeAttribute("contributorType", type); } // person name=, if (entityObject.getBoolean("isPerson")) { From 842dee678530391264b2869ec71ab70258901189 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 23 May 2024 17:21:57 -0400 Subject: [PATCH 083/360] add geolocations element and multiple geolocation --- .../pidproviders/doi/XmlMetadataTemplate.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 6e4d81d6248..d0986616bb4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1174,15 +1174,18 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X List places = dv.getGeographicCoverage(); if (places != null && !places.isEmpty()) { // geoLocationPlace - geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); - + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); for (String[] place : places) { + xmlw.startElement("geoLocation"); // + ArrayList placeList = new ArrayList(); for (String placePart : place) { placeList.add(placePart); } XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", Strings.join(placeList, ", ")); + xmlw.endElement(); // } + } boolean boundingBoxFound = false; boolean productionPlaceFound = false; @@ -1213,7 +1216,8 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X } } if (StringUtils.isNoneBlank(wLongitude, eLongitude, nLatitude, sLatitude)) { - geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); + xmlw.startElement("geoLocation"); // if (wLongitude.equals(eLongitude) && nLatitude.equals(sLatitude)) { // A point xmlw.writeStartElement("geoLocationPoint"); @@ -1230,15 +1234,18 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X xmlw.writeEndElement(); } + xmlw.endElement(); // } } case DatasetFieldConstant.productionPlace: productionPlaceFound = true; // geoLocationPlace - geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocation", geoLocationsWritten); + geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); List prodPlaces = dsf.getValues(); for (String prodPlace : prodPlaces) { + xmlw.startElement("geoLocation"); // XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", prodPlace); + xmlw.endElement(); // } break; } From 81a7c4a946ee4e54ae91913c6de6857fb6a553ba Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 23 May 2024 17:29:19 -0400 Subject: [PATCH 084/360] typos --- .../pidproviders/doi/XmlMetadataTemplate.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index d0986616bb4..127a1930860 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1176,14 +1176,14 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X // geoLocationPlace geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); for (String[] place : places) { - xmlw.startElement("geoLocation"); // + xmlw.writeStartElement("geoLocation"); // ArrayList placeList = new ArrayList(); for (String placePart : place) { placeList.add(placePart); } XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", Strings.join(placeList, ", ")); - xmlw.endElement(); // + xmlw.writeEndElement(); // } } @@ -1217,7 +1217,7 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X } if (StringUtils.isNoneBlank(wLongitude, eLongitude, nLatitude, sLatitude)) { geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); - xmlw.startElement("geoLocation"); // + xmlw.writeStartElement("geoLocation"); // if (wLongitude.equals(eLongitude) && nLatitude.equals(sLatitude)) { // A point xmlw.writeStartElement("geoLocationPoint"); @@ -1234,7 +1234,7 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X xmlw.writeEndElement(); } - xmlw.endElement(); // + xmlw.writeEndElement(); // } } case DatasetFieldConstant.productionPlace: @@ -1243,9 +1243,9 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X geoLocationsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "geoLocations", geoLocationsWritten); List prodPlaces = dsf.getValues(); for (String prodPlace : prodPlaces) { - xmlw.startElement("geoLocation"); // + xmlw.writeStartElement("geoLocation"); // XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", prodPlace); - xmlw.endElement(); // + xmlw.writeEndElement(); // } break; } From ed5eab0deb487ebfbb53157a40e2cf409d5f40ab Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 09:53:12 -0400 Subject: [PATCH 085/360] try execute inside the main method trying to avoid a separate tx boundary --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index f83041d87bd..fbff40a9c80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -167,9 +167,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { DeleteDatasetVersionCommand cmd; cmd = new DeleteDatasetVersionCommand(getRequest(), savedDataset); - ctxt.engine().submit(cmd); - // Running the command above reindexes the dataset, so we don't need to do it - // again in here. + cmd.execute(ctxt); // And update metadata at PID provider ctxt.engine().submit( From 39673f05e6d4394d3549c58a5d487a9c732113c2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 10:41:42 -0400 Subject: [PATCH 086/360] Fix subject, keyword --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 127a1930860..85e28670cfc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -305,8 +305,10 @@ private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLSt // fields if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersionForCopy(); - dv.getDatasetSubjects(); for (DatasetField dsf : dv.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.subject)) { + subjects.addAll(dsf.getValues()); + } if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.keyword)) { compoundKeywords = dsf.getDatasetFieldCompoundValues(); } else if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.topicClassification)) { @@ -330,7 +332,7 @@ private void writeSubjects(XMLStreamWriter xmlw, DvObject dvObject) throws XMLSt for (DatasetField subField : keywordFieldValue.getChildDatasetFields()) { switch (subField.getDatasetFieldType().getName()) { - case DatasetFieldConstant.keyword: + case DatasetFieldConstant.keywordValue: keyword = subField.getValue(); break; case DatasetFieldConstant.keywordVocab: From 36097d61bbf0c92aab48db01ff02e1c23b86be1a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 10:41:51 -0400 Subject: [PATCH 087/360] fix geo coverage --- src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index d723cf3d528..6648419216d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1342,9 +1342,8 @@ public List getGeographicCoverage() { } geoCoverages.add(coverageItem); } - + break; } - break; } return geoCoverages; } From a5d3b3e5a40b049176a6c3a205b1199c2117694e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 11:47:04 -0400 Subject: [PATCH 088/360] adjust funders to include grant number, add xml escaping for description --- .../pidproviders/doi/XmlMetadataTemplate.java | 70 +++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 85e28670cfc..3b6a5cb2906 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1082,7 +1082,7 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws X if (descriptions != null) { for (String description : descriptions) { descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, description); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, StringEscapeUtils.escapeXml10(description)); } } @@ -1272,21 +1272,67 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr dv = df.getOwner().getLatestVersionForCopy(); } if (dv != null) { - List funders = dv.getFunders(); - if (!funders.isEmpty()) { - - for (String funder : funders) { - if (!StringUtils.isBlank(funder)) { - fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); - xmlw.writeStartElement("fundingReference"); // - XmlWriterUtil.writeFullElement(xmlw, "funderName", funder); - xmlw.writeEndElement(); // + List retList = new ArrayList<>(); + for (DatasetField dsf : dv.getDatasetFields()) { + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.contributor)) { + boolean addFunder = false; + for (DatasetFieldCompoundValue contributorValue : dsf.getDatasetFieldCompoundValues()) { + String contributorName = null; + String contributorType = null; + for (DatasetField subField : contributorValue.getChildDatasetFields()) { + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.contributorName)) { + contributorName = subField.getDisplayValue(); + } + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.contributorType)) { + contributorType = subField.getRawValue(); + } + } + // SEK 02/12/2019 move outside loop to prevent contrib type to carry over to + // next contributor + // TODO: Consider how this will work in French, Chinese, etc. + if ("Funder".equals(contributorType)) { + if (!StringUtils.isBlank(contributorName)) { + fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); + xmlw.writeStartElement("fundingReference"); // + XmlWriterUtil.writeFullElement(xmlw, "funderName", contributorName); + xmlw.writeEndElement(); // + } + } } } - if (fundingReferenceWritten) { - xmlw.writeEndElement(); // + if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.grantNumber)) { + for (DatasetFieldCompoundValue grantObject : dsf.getDatasetFieldCompoundValues()) { + String funder = null; + String awardNumber = null; + for (DatasetField subField : grantObject.getChildDatasetFields()) { + // It would be nice to do something with grantNumberValue (the actual number) + // but schema.org doesn't support it. + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.grantNumberAgency)) { + String grantAgency = subField.getDisplayValue(); + funder = grantAgency; + } else if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.grantNumberValue)) { + String grantNumberValue = subField.getDisplayValue(); + awardNumber = grantNumberValue; + } + } + if (!StringUtils.isBlank(funder)) { + fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); + xmlw.writeStartElement("fundingReference"); // + XmlWriterUtil.writeFullElement(xmlw, "funderName", funder); + if (StringUtils.isNotBlank(awardNumber)) { + writeFullElement(xmlw, null, "awardNumber", null, awardNumber); + } + xmlw.writeEndElement(); // + } + + } } } + + if (fundingReferenceWritten) { + xmlw.writeEndElement(); // + } + } } } \ No newline at end of file From 8a12444d3b835a1df989bc674337897b7feaf1d2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 11:49:03 -0400 Subject: [PATCH 089/360] bug: add dataset descriptions --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 3b6a5cb2906..564768991cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1069,7 +1069,7 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws X if (dvObject instanceof Dataset d) { dv = d.getLatestVersionForCopy(); - dv.getDescriptions(); + descriptions = dv.getDescriptions(); } else if (dvObject instanceof DataFile df) { String description = df.getDescription(); if (description != null) { From f3e5dc1d00e1d68a734b4c593ae5b874bb5d14a2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 11:54:50 -0400 Subject: [PATCH 090/360] typo, add xml escape for funder --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 564768991cb..a4fd4585028 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1294,7 +1294,7 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr if (!StringUtils.isBlank(contributorName)) { fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); xmlw.writeStartElement("fundingReference"); // - XmlWriterUtil.writeFullElement(xmlw, "funderName", contributorName); + XmlWriterUtil.writeFullElement(xmlw, "funderName", StringEscapeUtils.escapeXml10(contributorName)); xmlw.writeEndElement(); // } } @@ -1318,9 +1318,9 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr if (!StringUtils.isBlank(funder)) { fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); xmlw.writeStartElement("fundingReference"); // - XmlWriterUtil.writeFullElement(xmlw, "funderName", funder); + XmlWriterUtil.writeFullElement(xmlw, "funderName", StringEscapeUtils.escapeXml10(funder)); if (StringUtils.isNotBlank(awardNumber)) { - writeFullElement(xmlw, null, "awardNumber", null, awardNumber); + writeFullElement(xmlw, null, "awardNumber", StringEscapeUtils.escapeXml10(awardNumber)); } xmlw.writeEndElement(); // } From 5610c950212f2d3d80d7144c37c98e6cd0b71c5e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 12:00:09 -0400 Subject: [PATCH 091/360] still typo --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index a4fd4585028..e2883cad1f9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1320,7 +1320,7 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr xmlw.writeStartElement("fundingReference"); // XmlWriterUtil.writeFullElement(xmlw, "funderName", StringEscapeUtils.escapeXml10(funder)); if (StringUtils.isNotBlank(awardNumber)) { - writeFullElement(xmlw, null, "awardNumber", StringEscapeUtils.escapeXml10(awardNumber)); + XmlWriterUtil.writeFullElement(xmlw, "awardNumber", StringEscapeUtils.escapeXml10(awardNumber)); } xmlw.writeEndElement(); // } From 7148b03360b00363f6550aed5d5d851ab7c2c356 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 14:14:48 -0400 Subject: [PATCH 092/360] mark contact as deprecated - unused --- .../java/edu/harvard/iq/dataverse/DatasetFieldConstant.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 22bad42df96..c3e385dcff2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -156,6 +156,8 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String confidentialityDeclaration="confidentialityDeclaration"; public final static String specialPermissions="specialPermissions"; public final static String restrictions="restrictions"; + @Deprecated + //Doesn't appear to be used and is not datasetContact public final static String contact="contact"; public final static String citationRequirements="citationRequirements"; public final static String depositorRequirements="depositorRequirements"; From 0470459316b3338bd940ea2f9afcf9ec1430eab1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 14:14:59 -0400 Subject: [PATCH 093/360] more fixes --- .../pidproviders/doi/XmlMetadataTemplate.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index e2883cad1f9..fd5a4ecf7fb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -427,7 +427,7 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X case DatasetFieldConstant.distributor: compoundDistributors = dsf.getDatasetFieldCompoundValues(); break; - case DatasetFieldConstant.contact: + case DatasetFieldConstant.datasetContact: compoundContacts = dsf.getDatasetFieldCompoundValues(); break; case DatasetFieldConstant.contributor: @@ -638,7 +638,7 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } if (releaseDate != null) { - String date = Util.getDateTimeFormat().format(releaseDate); + String date = Util.getDateFormat().format(releaseDate); datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Available"); @@ -660,6 +660,14 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } } if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { + if(StringUtils.isNotBlank(startDate)) { + Date start = Util.getDateTimeFormat().parse(startDate); + startDate = Util.getDateFormat().format(start); + } + if(StringUtils.isNotBlank(endDate)) { + Date end = Util.getDateTimeFormat().parse(endDate); + endDate = Util.getDateFormat().format(end); + } datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Collected"); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, (startDate + "/" + endDate).trim()); @@ -675,14 +683,14 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea private void writeLanguage(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { // Currently not supported. Spec indicates one 'primary' language. Could send // the first entry in DatasetFieldConstant.language or send iff there is only - // one entry, and/or default to the machine's default lang? + // one entry, and/or default to the machine's default lang, or the dataverse metadatalang? return; } // 10, ResourceType (with mandatory general type // description sub- property) (M) private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { - List kindOfDataValues = new ArrayList(); + List kindOfDataValues = new ArrayList(); Map attributes = new HashMap(); attributes.put("resourceTypeGeneral", "Dataset"); @@ -691,7 +699,7 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X for (DatasetField dsf : dv.getDatasetFields()) { switch (dsf.getDatasetFieldType().getName()) { case DatasetFieldConstant.kindOfData: - kindOfDataValues = dsf.getControlledVocabularyValues(); + kindOfDataValues.addAll(dsf.getValues()); break; } } @@ -701,8 +709,7 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); xmlw.writeEndElement(); } else { - for (ControlledVocabularyValue kindOfDataValue : kindOfDataValues) { - String resourceType = kindOfDataValue.getStrValue(); + for (String resourceType : kindOfDataValues) { if (StringUtils.isNotBlank(resourceType)) { XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, resourceType); } @@ -821,14 +828,16 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th * way those two fields are used for all identifier types. The code here is * ~best effort to interpret those fields. */ + logger.info("Found relpub: " + pubIdType + " " + identifier + " " + url); pubIdType = getCanonicalPublicationType(pubIdType); - +logger.info("Canonical type: " + pubIdType); // Prefer url if set, otherwise check identifier String relatedIdentifier = url; if (StringUtils.isBlank(relatedIdentifier)) { relatedIdentifier = identifier; } + logger.info("Related identifier: " + relatedIdentifier); // For types where we understand the protocol, get the canonical form if (pubIdType != null) { switch (pubIdType) { @@ -836,12 +845,15 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th if (!relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http")) { relatedIdentifier = "doi:" + relatedIdentifier; } + logger.info("Intermediate Related identifier: " + relatedIdentifier); try { GlobalId pid = PidUtil.parseAsGlobalID(relatedIdentifier); relatedIdentifier = pid.asRawIdentifier(); } catch (IllegalArgumentException e) { + logger.warning("Invalid DOI: " + e.getLocalizedMessage()); relatedIdentifier = null; } + logger.info("Final Related identifier: " + relatedIdentifier); break; case "Handle": if (!relatedIdentifier.startsWith("hdl:") || relatedIdentifier.startsWith("http")) { From c0265da5324c6f68e9356a44261ce6b166ded6b8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 14:22:29 -0400 Subject: [PATCH 094/360] catch parseexception --- .../pidproviders/doi/XmlMetadataTemplate.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index fd5a4ecf7fb..9ed417e77ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -5,6 +5,7 @@ import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; +import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -661,12 +662,20 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { if(StringUtils.isNotBlank(startDate)) { + try { Date start = Util.getDateTimeFormat().parse(startDate); startDate = Util.getDateFormat().format(start); + } catch (ParseException e) { + logger.warning("Could not parse date: " + startDate); + } } if(StringUtils.isNotBlank(endDate)) { + try { Date end = Util.getDateTimeFormat().parse(endDate); - endDate = Util.getDateFormat().format(end); + endDate = Util.getDateFormat().format(end); + } catch (ParseException e) { + logger.warning("Could not parse date: " + endDate); + }; } datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Collected"); From 2ff867850500aa9b2eb5712348b65cf48ed4b917 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 14:52:42 -0400 Subject: [PATCH 095/360] fix alternateIdentifier, related PID parsing, series --- .../pidproviders/doi/XmlMetadataTemplate.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 9ed417e77ce..7f861b3e42d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -755,7 +755,7 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) } if (altPids != null && !altPids.isEmpty()) { - alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); + alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternateIdentifiers", alternatesWritten); for (AlternativePersistentIdentifier altPid : altPids) { String identifierType = null; String identifier = null; @@ -775,7 +775,7 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) identifier = altPid.getAuthority() + altPid.getIdentifier(); break; } - attributes.put("alternativeIdentifierType", identifierType); + attributes.put("alternateIdentifierType", identifierType); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); } @@ -795,9 +795,9 @@ private void writeAlternateIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) break; } } - attributes.put("alternativeIdentifierType", identifierType); + attributes.put("alternateIdentifierType", identifierType); if (!StringUtils.isBlank(identifier)) { - alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternativeIdentifiers", alternatesWritten); + alternatesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "alternateIdentifiers", alternatesWritten); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "alternateIdentifier", attributes, identifier); } @@ -851,7 +851,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th if (pubIdType != null) { switch (pubIdType) { case "DOI": - if (!relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http")) { + if (!relatedIdentifier.startsWith("doi:") || !relatedIdentifier.startsWith("http")) { relatedIdentifier = "doi:" + relatedIdentifier; } logger.info("Intermediate Related identifier: " + relatedIdentifier); @@ -865,7 +865,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th logger.info("Final Related identifier: " + relatedIdentifier); break; case "Handle": - if (!relatedIdentifier.startsWith("hdl:") || relatedIdentifier.startsWith("http")) { + if (!relatedIdentifier.startsWith("hdl:") || !relatedIdentifier.startsWith("http")) { relatedIdentifier = "hdl:" + relatedIdentifier; } try { @@ -1158,7 +1158,7 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws X List childDsfs = dsfcv.getChildDatasetFields(); for (DatasetField childDsf : childDsfs) { - if (DatasetFieldConstant.seriesInformation.equals(childDsf.getDatasetFieldType().getName())) { + if (DatasetFieldConstant.seriesName.equals(childDsf.getDatasetFieldType().getName())) { String seriesInformation = childDsf.getValue(); if (StringUtils.isNotBlank(seriesInformation)) { descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); From 182f3d7bca310c54eb44f0452c76671b26b03824 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 14:53:47 -0400 Subject: [PATCH 096/360] catch PID update exception to avoid corrupt dataset --- .../CuratePublishedDatasetVersionCommand.java | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index fbff40a9c80..dd8b19e0c3b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.datavariable.VarGroup; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -151,7 +152,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { tempDataset.setThumbnailFile(publishedFmd.getDataFile()); } } - + if(logger.isLoggable(Level.FINE)) { + for(FileMetadata fmd: updateVersion.getFileMetadatas()) { + logger.fine("Id: " + fmd.getId() + " label: " + fmd.getLabel()); + } + } // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); @@ -170,28 +175,38 @@ public Dataset execute(CommandContext ctxt) throws CommandException { cmd.execute(ctxt); // And update metadata at PID provider - ctxt.engine().submit( - new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); - - //And the exported metadata files try { - ExportService instance = ExportService.getInstance(); - instance.exportAllFormats(getDataset()); - } catch (ExportException ex) { - // Just like with indexing, a failure to export is not a fatal condition. - logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while exporting metadata files:{0}", ex.getMessage()); + ctxt.engine().submit( + new UpdateDvObjectPIDMetadataCommand(savedDataset, getRequest())); + } catch (CommandException ex) { + //Make this non-fatal as after the DeleteDatasetVersionCommand, we can't roll back - for some reason no datasetfields remain in the DB + //(The old version doesn't need them and the new version doesn't get updated to include them?) + logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while updating PID metadata:{0}", ex.getMessage()); } - - // Update so that getDataset() in updateDatasetUser will get the up-to-date copy // (with no draft version) setDataset(savedDataset); updateDatasetUser(ctxt); - - - return savedDataset; } + @Override + public boolean onSuccess(CommandContext ctxt, Object r) { + boolean retVal = true; + Dataset d = (Dataset) r; + + ctxt.index().asyncIndexDataset(d, true); + + // And the exported metadata files + try { + ExportService instance = ExportService.getInstance(); + instance.exportAllFormats(d); + } catch (ExportException ex) { + // Just like with indexing, a failure to export is not a fatal condition. + retVal = false; + logger.log(Level.WARNING, "Curate Published DatasetVersion: exception while exporting metadata files:{0}", ex.getMessage()); + } + return retVal; + } } From be903555bcc3c21169b3a7343a076c331d308bf6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 16:24:40 -0400 Subject: [PATCH 097/360] try long sleep --- .../java/edu/harvard/iq/dataverse/DatasetServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index dab0ff43fcf..29de42f3578 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -944,9 +944,9 @@ public void callFinalizePublishCommandAsynchronously(Long datasetId, CommandCont // portion of the PublishDatasetCommand. I'm going to leave the 1 second // sleep below, for just in case reasons: -- L.A. try { - Thread.sleep(1000); + Thread.sleep(5000); } catch (Exception ex) { - logger.warning("Failed to sleep for a second."); + logger.warning("Failed to sleep for five seconds."); } logger.fine("Running FinalizeDatasetPublicationCommand, asynchronously"); Dataset theDataset = find(datasetId); From e458e8ca5c14f48d8da94d83c2cb40e76e9198d6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 16:53:28 -0400 Subject: [PATCH 098/360] set dv released before pid publicize, go back to short time --- .../java/edu/harvard/iq/dataverse/DatasetServiceBean.java | 4 ++-- .../command/impl/FinalizeDatasetPublicationCommand.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 29de42f3578..18bd6dc74ee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -944,9 +944,9 @@ public void callFinalizePublishCommandAsynchronously(Long datasetId, CommandCont // portion of the PublishDatasetCommand. I'm going to leave the 1 second // sleep below, for just in case reasons: -- L.A. try { - Thread.sleep(5000); + Thread.sleep(1000); } catch (Exception ex) { - logger.warning("Failed to sleep for five seconds."); + logger.warning("Failed to sleep for one second."); } logger.fine("Running FinalizeDatasetPublicationCommand, asynchronously"); Dataset theDataset = find(datasetId); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 287e877f6e0..299bb3168de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -211,7 +211,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if (theDataset.getLatestVersion().getVersionState() != RELEASED) { // some imported datasets may already be released. - + theDataset.getLatestVersion().setVersionState(RELEASED); if (!datasetExternallyReleased) { publicizeExternalIdentifier(theDataset, ctxt); // Will throw a CommandException, unless successful. @@ -220,7 +220,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // a failure - it will remove any locks, and it will send a // proper notification to the user(s). } - theDataset.getLatestVersion().setVersionState(RELEASED); } final Dataset ds = ctxt.em().merge(theDataset); From 27fe7b4d0dad6124d5e036c8ec5d36b31a371a9b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 24 May 2024 17:03:13 -0400 Subject: [PATCH 099/360] always use latest version for copy --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 7f861b3e42d..0adc9984b3d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -146,7 +146,7 @@ private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String languag List altTitles = new ArrayList<>(); // Only Datasets can have a subtitle or alternative titles if (dvObject instanceof Dataset d) { - DatasetVersion dv = d.getLatestVersion(); + DatasetVersion dv = d.getLatestVersionForCopy(); Optional subTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.subTitle)).findFirst(); if (subTitleField.isPresent()) { subTitle = subTitleField.get().getValue(); From 00a383007686b49b938f88415b476e377d054f98 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 09:53:40 -0400 Subject: [PATCH 100/360] handle deaccession, fix relatedIDtype for files --- .../edu/harvard/iq/dataverse/DataFile.java | 19 +++ .../pidproviders/doi/XmlMetadataTemplate.java | 117 +++++++++++------- 2 files changed, 90 insertions(+), 46 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 29a4a14c021..1a610d9ea6e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -1123,4 +1123,23 @@ private boolean tagExists(String tagLabel) { } return false; } + + public boolean isDeaccessioned() { + // return true, if all published versions were deaccessioned + boolean inDeaccessionedVersions = false; + for (FileMetadata fmd : getFileMetadatas()) { + DatasetVersion testDsv = fmd.getDatasetVersion(); + if (testDsv.isReleased()) { + return false; + } + // Also check for draft version + if (testDsv.isDraft()) { + return false; + } + if (testDsv.isDeaccessioned()) { + inDeaccessionedVersions = true; + } + } + return inDeaccessionedVersions; // since any published version would have already returned + } } // end of class diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 0adc9984b3d..96ee84fe13b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -100,30 +100,41 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM String metadataLanguage = null; // when set, otherwise = language? XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); xmlw.writeStartElement("resource"); - + boolean deaccessioned=false; + if(dvObject instanceof Dataset d) { + deaccessioned=d.isDeaccessioned(); + } else if (dvObject instanceof DataFile df) { + deaccessioned = df.isDeaccessioned(); + } xmlw.writeDefaultNamespace(XML_NAMESPACE); xmlw.writeAttribute("xmlns:xsi", XML_XSI); xmlw.writeAttribute("xsi:schemaLocation", XML_SCHEMA_LOCATION); writeIdentifier(xmlw, dvObject); - writeCreators(xmlw, doiMetadata.getAuthors()); - writeTitles(xmlw, dvObject, language); - writePublisher(xmlw, dvObject); - writePublicationYear(xmlw, dvObject); - writeSubjects(xmlw, dvObject); - writeContributors(xmlw, dvObject); - writeDates(xmlw, dvObject); - writeLanguage(xmlw, dvObject); + writeCreators(xmlw, doiMetadata.getAuthors(), deaccessioned); + writeTitles(xmlw, dvObject, language, deaccessioned); + writePublisher(xmlw, dvObject, deaccessioned); + writePublicationYear(xmlw, dvObject, deaccessioned); + if (!deaccessioned) { + writeSubjects(xmlw, dvObject); + writeContributors(xmlw, dvObject); + writeDates(xmlw, dvObject); + writeLanguage(xmlw, dvObject); + } writeResourceType(xmlw, dvObject); - writeAlternateIdentifiers(xmlw, dvObject); - writeRelatedIdentifiers(xmlw, dvObject); - writeSize(xmlw, dvObject); - writeFormats(xmlw, dvObject); - writeVersion(xmlw, dvObject); - writeAccessRights(xmlw, dvObject); - writeDescriptions(xmlw, dvObject); - writeGeoLocations(xmlw, dvObject); - writeFundingReferences(xmlw, dvObject); + if (!deaccessioned) { + writeAlternateIdentifiers(xmlw, dvObject); + writeRelatedIdentifiers(xmlw, dvObject); + writeSize(xmlw, dvObject); + writeFormats(xmlw, dvObject); + writeVersion(xmlw, dvObject); + writeAccessRights(xmlw, dvObject); + } + writeDescriptions(xmlw, dvObject, deaccessioned); + if (!deaccessioned) { + writeGeoLocations(xmlw, dvObject); + writeFundingReferences(xmlw, dvObject); + } xmlw.writeEndElement(); xmlw.flush(); } @@ -140,23 +151,29 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM * @return * @throws XMLStreamException */ - private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String language) throws XMLStreamException { - String title = doiMetadata.getTitle(); + private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String language, boolean deaccessioned) throws XMLStreamException { + String title = null; String subTitle = null; List altTitles = new ArrayList<>(); - // Only Datasets can have a subtitle or alternative titles - if (dvObject instanceof Dataset d) { - DatasetVersion dv = d.getLatestVersionForCopy(); - Optional subTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.subTitle)).findFirst(); - if (subTitleField.isPresent()) { - subTitle = subTitleField.get().getValue(); - } - Optional altTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.alternativeTitle)).findFirst(); - if (altTitleField.isPresent()) { - altTitles = altTitleField.get().getValues(); + + if (!deaccessioned) { + doiMetadata.getTitle(); + + // Only Datasets can have a subtitle or alternative titles + if (dvObject instanceof Dataset d) { + DatasetVersion dv = d.getLatestVersionForCopy(); + Optional subTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.subTitle)).findFirst(); + if (subTitleField.isPresent()) { + subTitle = subTitleField.get().getValue(); + } + Optional altTitleField = dv.getDatasetFields().stream().filter(f -> f.getDatasetFieldType().getName().equals(DatasetFieldConstant.alternativeTitle)).findFirst(); + if (altTitleField.isPresent()) { + altTitles = altTitleField.get().getValues(); + } } + } else { + title = AbstractDOIProvider.UNAVAILABLE; } - if (StringUtils.isNotBlank(title) || StringUtils.isNotBlank(subTitle) || (altTitles != null && !String.join("", altTitles).isBlank())) { xmlw.writeStartElement("titles"); if (StringUtils.isNotBlank(title)) { @@ -227,13 +244,13 @@ private void writeIdentifier(XMLStreamWriter xmlw, DvObject dvObject) throws XML * - the list of authors * @throws XMLStreamException */ - public void writeCreators(XMLStreamWriter xmlw, List authorList) throws XMLStreamException { + public void writeCreators(XMLStreamWriter xmlw, List authorList, boolean deaccessioned) throws XMLStreamException { // creators -> creator -> creatorName with nameType attribute, givenName, // familyName, nameIdentifier // write all creators xmlw.writeStartElement("creators"); // - if (authorList != null && !authorList.isEmpty()) { + if (!deaccessioned && authorList != null && !authorList.isEmpty()) { for (DatasetAuthor author : authorList) { String creatorName = StringEscapeUtils.escapeXml10(author.getName().getDisplayValue()); String affiliation = null; @@ -267,18 +284,21 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList) xmlw.writeEndElement(); // } - private void writePublisher(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + private void writePublisher(XMLStreamWriter xmlw, DvObject dvObject, boolean deaccessioned) throws XMLStreamException { // publisher should already be non null - :unav if it wasn't available + if(deaccessioned) { + doiMetadata.setPublisher(AbstractPidProvider.UNAVAILABLE); + } XmlWriterUtil.writeFullElement(xmlw, "publisher", doiMetadata.getPublisher()); } - private void writePublicationYear(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + private void writePublicationYear(XMLStreamWriter xmlw, DvObject dvObject, boolean deaccessioned) throws XMLStreamException { // Can't use "UNKNOWN" here because DataCite will respond with "[facet // 'pattern'] the value 'unknown' is not accepted by the pattern '[\d]{4}'" String pubYear = "9999"; // FIXME: Investigate why this.publisherYear is sometimes null now that pull // request #4606 has been merged. - if (doiMetadata.getPublisherYear() != null) { + if (! deaccessioned && (doiMetadata.getPublisherYear() != null)) { // Added to prevent a NullPointerException when trying to destroy datasets when // using DataCite rather than EZID. pubYear = doiMetadata.getPublisherYear(); @@ -926,6 +946,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th attributes.clear(); attributes.put("relationType", "IsPartOf"); + attributes.put("relatedIdentifierType", pubIdType); relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, pid.asRawIdentifier()); } @@ -1082,20 +1103,24 @@ private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) throws X xmlw.writeEndElement(); // } - private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { + private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject, boolean deaccessioned) throws XMLStreamException { // descriptions -> description with descriptionType attribute boolean descriptionsWritten = false; List descriptions = null; DatasetVersion dv = null; - - if (dvObject instanceof Dataset d) { - dv = d.getLatestVersionForCopy(); - descriptions = dv.getDescriptions(); - } else if (dvObject instanceof DataFile df) { - String description = df.getDescription(); - if (description != null) { - descriptions = new ArrayList(); - descriptions.add(description); + if(deaccessioned) { + descriptions = new ArrayList(); + descriptions.add(AbstractDOIProvider.UNAVAILABLE); + } else { + if (dvObject instanceof Dataset d) { + dv = d.getLatestVersionForCopy(); + descriptions = dv.getDescriptions(); + } else if (dvObject instanceof DataFile df) { + String description = df.getDescription(); + if (description != null) { + descriptions = new ArrayList(); + descriptions.add(description); + } } } Map attributes = new HashMap(); From 1faf0cd84c5ef52a8e88afb7989e238058870916 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 10:26:30 -0400 Subject: [PATCH 101/360] missed assignment for title --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 96ee84fe13b..b2008e14a89 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -157,7 +157,7 @@ private void writeTitles(XMLStreamWriter xmlw, DvObject dvObject, String languag List altTitles = new ArrayList<>(); if (!deaccessioned) { - doiMetadata.getTitle(); + title = doiMetadata.getTitle(); // Only Datasets can have a subtitle or alternative titles if (dvObject instanceof Dataset d) { From 23dd581c98b921908b2cdcca13d32e8731c76e7e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 10:54:06 -0400 Subject: [PATCH 102/360] fix creator for deaccessioned --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index b2008e14a89..8f962204302 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -249,8 +249,10 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, // familyName, nameIdentifier // write all creators xmlw.writeStartElement("creators"); // - - if (!deaccessioned && authorList != null && !authorList.isEmpty()) { + if(deaccessioned) { + authorList = null; + } + if (authorList != null && !authorList.isEmpty()) { for (DatasetAuthor author : authorList) { String creatorName = StringEscapeUtils.escapeXml10(author.getName().getDisplayValue()); String affiliation = null; From 3bbd2e9dfc4cd3d1e5cebd5a9cf7dbbfe52c1fa4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 11:51:50 -0400 Subject: [PATCH 103/360] correct fix for creators when deaccessioned --- .../pidproviders/doi/XmlMetadataTemplate.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8f962204302..a3eca9ef9a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -250,8 +250,10 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, // write all creators xmlw.writeStartElement("creators"); // if(deaccessioned) { + //skip the loop below authorList = null; } + boolean nothingWritten = true; if (authorList != null && !authorList.isEmpty()) { for (DatasetAuthor author : authorList) { String creatorName = StringEscapeUtils.escapeXml10(author.getName().getDisplayValue()); @@ -274,15 +276,17 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, if (StringUtils.isNotBlank(creatorName)) { JsonObject creatorObj = PersonOrOrgUtil.getPersonOrOrganization(creatorName, false, StringUtils.containsIgnoreCase(nameIdentifierScheme, "orcid")); + nothingWritten = false; writeEntityElements(xmlw, "creator", null, creatorObj, affiliation, nameIdentifier, nameIdentifierScheme); } - else { - // Authors unavailable - XmlWriterUtil.writeFullElement(xmlw, "creator", "creatorName", AbstractPidProvider.UNAVAILABLE); - } + } } + if (nothingWritten) { + // Authors unavailable + XmlWriterUtil.writeFullElement(xmlw, "creator", "creatorName", AbstractPidProvider.UNAVAILABLE); + } xmlw.writeEndElement(); // } From 4def6da32c207223fb2cf9d5aada50f921ddd474 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 12:00:15 -0400 Subject: [PATCH 104/360] remove bad value and lang --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index a3eca9ef9a0..2d09c67fea9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -285,7 +285,7 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, } if (nothingWritten) { // Authors unavailable - XmlWriterUtil.writeFullElement(xmlw, "creator", "creatorName", AbstractPidProvider.UNAVAILABLE); + XmlWriterUtil.writeFullElement(xmlw, "creator", AbstractPidProvider.UNAVAILABLE); } xmlw.writeEndElement(); // } From eac477ec3c2fef4f48759b110f6157081b301eff Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 12:46:07 -0400 Subject: [PATCH 105/360] add creatorName sub element for deaccession/no names case --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 2d09c67fea9..a660a80448a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -285,7 +285,9 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, } if (nothingWritten) { // Authors unavailable - XmlWriterUtil.writeFullElement(xmlw, "creator", AbstractPidProvider.UNAVAILABLE); + xmlw.writeStartElement("creator"); + XmlWriterUtil.writeFullElement(xmlw, "creatorName", AbstractPidProvider.UNAVAILABLE); + xmlw.writeEndElement("creator"); } xmlw.writeEndElement(); // } From 154ac8a91554be29492d62454f1b0e52501b5af2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 12:49:59 -0400 Subject: [PATCH 106/360] typo --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index a660a80448a..732a633116e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -287,7 +287,7 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, // Authors unavailable xmlw.writeStartElement("creator"); XmlWriterUtil.writeFullElement(xmlw, "creatorName", AbstractPidProvider.UNAVAILABLE); - xmlw.writeEndElement("creator"); + xmlw.writeEndElement(); } xmlw.writeEndElement(); // } From 9144f6c96ae0685a7ac719d2203f0aed3f71e85e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 13:16:32 -0400 Subject: [PATCH 107/360] fix resourceType - always 1 entry --- .../pidproviders/doi/XmlMetadataTemplate.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 732a633116e..8f6211c0730 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -736,23 +736,24 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X for (DatasetField dsf : dv.getDatasetFields()) { switch (dsf.getDatasetFieldType().getName()) { case DatasetFieldConstant.kindOfData: - kindOfDataValues.addAll(dsf.getValues()); + List vals = dsf.getValues(); + for(String val: vals) { + if(StringUtils.isNotBlank(val)) { + kindOfDataValues.add(val); + } + } break; } } - if (kindOfDataValues.isEmpty()) { + if (!kindOfDataValues.isEmpty()) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, String.join(", ", kindOfDataValues)); + + } else { // Write an attribute only element if there are no kindOfData values. xmlw.writeStartElement("resourceType"); xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); xmlw.writeEndElement(); - } else { - for (String resourceType : kindOfDataValues) { - if (StringUtils.isNotBlank(resourceType)) { - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, resourceType); - } - } } - } } From a5870fbaf3a89e9c6100b8e1f4371caa291c1e23 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 28 May 2024 13:26:41 -0400 Subject: [PATCH 108/360] Also handle file case for resourceType --- .../pidproviders/doi/XmlMetadataTemplate.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8f6211c0730..7d817d57a2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -745,16 +745,17 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X break; } } - if (!kindOfDataValues.isEmpty()) { - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, String.join(", ", kindOfDataValues)); - - } else { - // Write an attribute only element if there are no kindOfData values. - xmlw.writeStartElement("resourceType"); - xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); - xmlw.writeEndElement(); - } } + if (!kindOfDataValues.isEmpty()) { + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, String.join(", ", kindOfDataValues)); + + } else { + // Write an attribute only element if there are no kindOfData values. + xmlw.writeStartElement("resourceType"); + xmlw.writeAttribute("resourceTypeGeneral", attributes.get("resourceTypeGeneral")); + xmlw.writeEndElement(); + } + } /** From 24db2af2bfe9564eedc1ee9aedae8b9048bd551e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 31 May 2024 14:47:06 -0400 Subject: [PATCH 109/360] missed changes --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 7d817d57a2a..f5bd009e8d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -23,6 +23,7 @@ import javax.xml.stream.XMLStreamWriter; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringEscapeUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -39,6 +40,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetRelPublication; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; @@ -703,7 +705,7 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea endDate = Util.getDateFormat().format(end); } catch (ParseException e) { logger.warning("Could not parse date: " + endDate); - }; + }; } datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Collected"); @@ -853,7 +855,6 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th Map attributes = new HashMap(); if (dvObject instanceof Dataset dataset) { - List relatedPublications = dataset.getLatestVersionForCopy().getRelatedPublications(); if (!relatedPublications.isEmpty()) { for (DatasetRelPublication relatedPub : relatedPublications) { @@ -967,7 +968,6 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } } - static HashMap relatedIdentifierTypeMap = new HashMap(); private static String getCanonicalPublicationType(String pubIdType) { From f0fd61ad555369ef1af9a1529797cf8d73d6efde Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 31 May 2024 15:05:02 -0400 Subject: [PATCH 110/360] simplify - util checks for null and empty --- .../dataverse/export/ddi/DdiExportUtil.java | 46 +++++-------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index c0e3057696a..f5efc448090 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -371,7 +371,7 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ xmlw.writeStartElement("verStmt"); - XmlWriterUtil.writeAttribute(xmlw,"source","archive"); + xmlw.writeAttribute("source","archive"); xmlw.writeStartElement("version"); XmlWriterUtil.writeAttribute(xmlw,"date", datasetVersionDTO.getReleaseTime().substring(0, 10)); XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); @@ -842,9 +842,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!authorName.isEmpty()){ xmlw.writeStartElement("AuthEnty"); - if(!authorAffiliation.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"affiliation",authorAffiliation); - } + XmlWriterUtil.writeAttribute(xmlw,"affiliation",authorAffiliation); xmlw.writeCharacters(authorName); xmlw.writeEndElement(); //AuthEnty } @@ -865,9 +863,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!contributorName.isEmpty()){ xmlw.writeStartElement("othId"); - if(!contributorType.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"role", contributorType); - } + XmlWriterUtil.writeAttribute(xmlw,"role", contributorType); xmlw.writeCharacters(contributorName); xmlw.writeEndElement(); //othId } @@ -907,12 +903,8 @@ private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO // TODO: Since datasetContactEmail is a required field but datasetContactName is not consider not checking if datasetContactName is empty so we can write out datasetContactEmail. if (!datasetContactName.isEmpty()){ xmlw.writeStartElement("contact"); - if(!datasetContactAffiliation.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"affiliation",datasetContactAffiliation); - } - if(!datasetContactEmail.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"email",datasetContactEmail); - } + XmlWriterUtil.writeAttribute(xmlw,"affiliation",datasetContactAffiliation); + XmlWriterUtil.writeAttribute(xmlw,"email",datasetContactEmail); xmlw.writeCharacters(datasetContactName); xmlw.writeEndElement(); //AuthEnty } @@ -957,9 +949,7 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT xmlw.writeStartElement("producer"); XmlWriterUtil.writeAttribute(xmlw, "affiliation", producerAffiliation); XmlWriterUtil.writeAttribute(xmlw, "abbr", producerAbbreviation); - /*if (!producerLogo.isEmpty()) { - XmlWriterUtil.writeAttribute(xmlw, "role", producerLogo); - }*/ + //XmlWriterUtil.writeAttribute(xmlw, "role", producerLogo); xmlw.writeCharacters(producerName); xmlw.writeEndElement(); //AuthEnty } @@ -1139,9 +1129,7 @@ private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!descriptionText.isEmpty()){ xmlw.writeStartElement("abstract"); - if(!descriptionDate.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"date",descriptionDate); - } + XmlWriterUtil.writeAttribute(xmlw,"date",descriptionDate); if(DvObjectContainer.isMetadataLanguageSet(lang)) { xmlw.writeAttribute("xml:lang", lang); } @@ -1176,9 +1164,7 @@ private static void writeGrantElement(XMLStreamWriter xmlw, DatasetVersionDTO da } if (!grantNumber.isEmpty()){ xmlw.writeStartElement("grantNo"); - if(!grantAgency.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"agency",grantAgency); - } + XmlWriterUtil.writeAttribute(xmlw,"agency",grantAgency); xmlw.writeCharacters(grantNumber); xmlw.writeEndElement(); //grantno } @@ -1210,9 +1196,7 @@ private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!otherId.isEmpty()){ xmlw.writeStartElement("IDNo"); - if(!otherIdAgency.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"agency",otherIdAgency); - } + XmlWriterUtil.writeAttribute(xmlw,"agency",otherIdAgency); xmlw.writeCharacters(otherId); xmlw.writeEndElement(); //IDNo } @@ -1244,9 +1228,7 @@ private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!softwareName.isEmpty()){ xmlw.writeStartElement("software"); - if(!softwareVersion.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"version",softwareVersion); - } + XmlWriterUtil.writeAttribute(xmlw,"version",softwareVersion); xmlw.writeCharacters(softwareName); xmlw.writeEndElement(); //software } @@ -1359,12 +1341,8 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da } if (!notesText.isEmpty()) { xmlw.writeStartElement("notes"); - if(!notesType.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"type",notesType); - } - if(!notesSubject.isEmpty()){ - XmlWriterUtil.writeAttribute(xmlw,"subject",notesSubject); - } + XmlWriterUtil.writeAttribute(xmlw,"type",notesType); + XmlWriterUtil.writeAttribute(xmlw,"subject",notesSubject); xmlw.writeCharacters(notesText); xmlw.writeEndElement(); } From ead153f502ef77258d20f6faf4a0fc8282a74687 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 10 Jun 2024 12:18:45 -0400 Subject: [PATCH 111/360] typo in DOI parsing logic --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index f5bd009e8d7..e6c1a1ae6b4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -882,7 +882,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th if (pubIdType != null) { switch (pubIdType) { case "DOI": - if (!relatedIdentifier.startsWith("doi:") || !relatedIdentifier.startsWith("http")) { + if (!(relatedIdentifier.startsWith("doi:") || relatedIdentifier.startsWith("http"))) { relatedIdentifier = "doi:" + relatedIdentifier; } logger.info("Intermediate Related identifier: " + relatedIdentifier); From ea75216025ca358f1c9d7d3c11d324b6dbe4f0f2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 10 Jun 2024 12:58:46 -0400 Subject: [PATCH 112/360] only files in latestversionforcopy --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index e6c1a1ae6b4..ae7c21b3308 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -43,6 +43,7 @@ import edu.harvard.iq.dataverse.DatasetRelPublication; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.api.Util; @@ -934,10 +935,12 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } } } - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { + List fmds = dataset.getLatestVersionForCopy().getFileMetadatas(); + if (!(fmds==null) && fmds.isEmpty()) { attributes.clear(); attributes.put("relationType", "HasPart"); - for (DataFile dataFile : dataset.getFiles()) { + for (FileMetadata fmd : fmds) { + DataFile dataFile = fmd.getDataFile(); GlobalId pid = dataFile.getGlobalId(); if (pid != null) { String pubIdType = getCanonicalPublicationType(pid.getProtocol()); From b6bd530db70dfbc78017445e4e2af79233b6e899 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 11 Jun 2024 16:40:58 -0400 Subject: [PATCH 113/360] fix date parsing, clear bad values --- .../dataverse/pidproviders/doi/XmlMetadataTemplate.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index ae7c21b3308..9ba1e4e3116 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -692,22 +692,27 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } } if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { + //Minimal clean-up - useful? Parse/format would remove unused chars, and an exception would clear the date so we don't send nonsense if(StringUtils.isNotBlank(startDate)) { try { - Date start = Util.getDateTimeFormat().parse(startDate); + Date start = Util.getDateFormat().parse(startDate); startDate = Util.getDateFormat().format(start); } catch (ParseException e) { logger.warning("Could not parse date: " + startDate); + startDate = null; } } if(StringUtils.isNotBlank(endDate)) { try { - Date end = Util.getDateTimeFormat().parse(endDate); + Date end = Util.getDateFormat().parse(endDate); endDate = Util.getDateFormat().format(end); } catch (ParseException e) { logger.warning("Could not parse date: " + endDate); + endDate = null; }; } + } + if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Collected"); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, (startDate + "/" + endDate).trim()); From e1383d77d3304418ebf3decfd257ca0610994ce1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 13:10:04 -0400 Subject: [PATCH 114/360] relationType entry in citation block --- conf/solr/9.3.0/schema.xml | 2 + ...dataset-create-new-all-default-fields.json | 2935 ++++++++--------- scripts/api/data/metadatablocks/citation.tsv | 105 +- .../iq/dataverse/DatasetFieldConstant.java | 1 + .../dublincore/DublinCoreExportUtil.java | 35 +- .../export/openaire/OpenAireExportUtil.java | 9 +- .../java/propertyFiles/citation.properties | 2 + .../export/OpenAireExportUtilTest.java | 2 +- 8 files changed, 1502 insertions(+), 1589 deletions(-) diff --git a/conf/solr/9.3.0/schema.xml b/conf/solr/9.3.0/schema.xml index 5dde750573d..32f10d0a621 100644 --- a/conf/solr/9.3.0/schema.xml +++ b/conf/solr/9.3.0/schema.xml @@ -349,6 +349,7 @@ + @@ -589,6 +590,7 @@ + diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index cc856c6372f..e522ab32b1d 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -1,1527 +1,1410 @@ { - "datasetVersion": { - "license": { - "name": "CC0 1.0", - "uri": "http://creativecommons.org/publicdomain/zero/1.0" - }, - "metadataBlocks": { - "citation": { - "displayName": "Citation Metadata", - "fields": [ - { - "typeName": "title", - "multiple": false, - "typeClass": "primitive", - "value": "Replication Data for: Title" - }, - { - "typeName": "subtitle", - "multiple": false, - "typeClass": "primitive", - "value": "Subtitle" - }, - { - "typeName": "alternativeTitle", - "multiple": true, - "typeClass": "primitive", - "value": ["Alternative Title"] - }, - { - "typeName": "alternativeURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://AlternativeURL.org" - }, - { - "typeName": "otherId", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "otherIdAgency": { - "typeName": "otherIdAgency", - "multiple": false, - "typeClass": "primitive", - "value": "OtherIDAgency1" - }, - "otherIdValue": { - "typeName": "otherIdValue", - "multiple": false, - "typeClass": "primitive", - "value": "OtherIDIdentifier1" - } - }, - { - "otherIdAgency": { - "typeName": "otherIdAgency", - "multiple": false, - "typeClass": "primitive", - "value": "OtherIDAgency2" - }, - "otherIdValue": { - "typeName": "otherIdValue", - "multiple": false, - "typeClass": "primitive", - "value": "OtherIDIdentifier2" - } - } - ] - }, - { - "typeName": "author", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "authorName": { - "typeName": "authorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastAuthor1, FirstAuthor1" - }, - "authorAffiliation": { - "typeName": "authorAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "AuthorAffiliation1" - }, - "authorIdentifierScheme": { - "typeName": "authorIdentifierScheme", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "ORCID" - }, - "authorIdentifier": { - "typeName": "authorIdentifier", - "multiple": false, - "typeClass": "primitive", - "value": "AuthorIdentifier1" - } - }, - { - "authorName": { - "typeName": "authorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastAuthor2, FirstAuthor2" - }, - "authorAffiliation": { - "typeName": "authorAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "AuthorAffiliation2" - }, - "authorIdentifierScheme": { - "typeName": "authorIdentifierScheme", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "ISNI" - }, - "authorIdentifier": { - "typeName": "authorIdentifier", - "multiple": false, - "typeClass": "primitive", - "value": "AuthorIdentifier2" - } - } - ] - }, - { - "typeName": "datasetContact", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "datasetContactName": { - "typeName": "datasetContactName", - "multiple": false, - "typeClass": "primitive", - "value": "LastContact1, FirstContact1" - }, - "datasetContactAffiliation": { - "typeName": "datasetContactAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "ContactAffiliation1" - }, - "datasetContactEmail": { - "typeName": "datasetContactEmail", - "multiple": false, - "typeClass": "primitive", - "value": "ContactEmail1@mailinator.com" - } - }, - { - "datasetContactName": { - "typeName": "datasetContactName", - "multiple": false, - "typeClass": "primitive", - "value": "LastContact2, FirstContact2" - }, - "datasetContactAffiliation": { - "typeName": "datasetContactAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "ContactAffiliation2" - }, - "datasetContactEmail": { - "typeName": "datasetContactEmail", - "multiple": false, - "typeClass": "primitive", - "value": "ContactEmail2@mailinator.com" - } - } - ] - }, - { - "typeName": "dsDescription", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "dsDescriptionValue": { - "typeName": "dsDescriptionValue", - "multiple": false, - "typeClass": "primitive", - "value": "DescriptionText1" - }, - "dsDescriptionDate": { - "typeName": "dsDescriptionDate", - "multiple": false, - "typeClass": "primitive", - "value": "1000-01-01" - } - }, - { - "dsDescriptionValue": { - "typeName": "dsDescriptionValue", - "multiple": false, - "typeClass": "primitive", - "value": "DescriptionText2" - }, - "dsDescriptionDate": { - "typeName": "dsDescriptionDate", - "multiple": false, - "typeClass": "primitive", - "value": "1000-02-02" - } - } - ] - }, - { - "typeName": "subject", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Agricultural Sciences", - "Business and Management", - "Engineering", - "Law" - ] - }, - { - "typeName": "keyword", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "keywordValue": { - "typeName": "keywordValue", - "multiple": false, - "typeClass": "primitive", - "value": "KeywordTerm1" - }, - "keywordTermURI": { - "typeName": "keywordTermURI", - "multiple": false, - "typeClass": "primitive", - "value": "http://keywordTermURI1.org" - }, - "keywordVocabulary": { - "typeName": "keywordVocabulary", - "multiple": false, - "typeClass": "primitive", - "value": "KeywordVocabulary1" - }, - "keywordVocabularyURI": { - "typeName": "keywordVocabularyURI", - "multiple": false, - "typeClass": "primitive", - "value": "http://KeywordVocabularyURL1.org" - } - }, - { - "keywordValue": { - "typeName": "keywordValue", - "multiple": false, - "typeClass": "primitive", - "value": "KeywordTerm2" - }, - "keywordTermURI": { - "typeName": "keywordTermURI", - "multiple": false, - "typeClass": "primitive", - "value": "http://keywordTermURI2.org" - }, - "keywordVocabulary": { - "typeName": "keywordVocabulary", - "multiple": false, - "typeClass": "primitive", - "value": "KeywordVocabulary2" - }, - "keywordVocabularyURI": { - "typeName": "keywordVocabularyURI", - "multiple": false, - "typeClass": "primitive", - "value": "http://KeywordVocabularyURL2.org" - } - } - ] - }, - { - "typeName": "topicClassification", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "topicClassValue": { - "typeName": "topicClassValue", - "multiple": false, - "typeClass": "primitive", - "value": "Topic Classification Term1" - }, - "topicClassVocab": { - "typeName": "topicClassVocab", - "multiple": false, - "typeClass": "primitive", - "value": "Topic Classification Vocab1" - }, - "topicClassVocabURI": { - "typeName": "topicClassVocabURI", - "multiple": false, - "typeClass": "primitive", - "value": "https://TopicClassificationURL1.com" - } - }, - { - "topicClassValue": { - "typeName": "topicClassValue", - "multiple": false, - "typeClass": "primitive", - "value": "Topic Classification Term2" - }, - "topicClassVocab": { - "typeName": "topicClassVocab", - "multiple": false, - "typeClass": "primitive", - "value": "Topic Classification Vocab2" - }, - "topicClassVocabURI": { - "typeName": "topicClassVocabURI", - "multiple": false, - "typeClass": "primitive", - "value": "https://TopicClassificationURL2.com" - } - } - ] - }, - { - "typeName": "publication", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "publicationCitation": { - "typeName": "publicationCitation", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationCitation1" - }, - "publicationIDType": { - "typeName": "publicationIDType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "ark" - }, - "publicationIDNumber": { - "typeName": "publicationIDNumber", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationIDNumber1" - }, - "publicationURL": { - "typeName": "publicationURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://RelatedPublicationURL1.org" - } - }, - { - "publicationCitation": { - "typeName": "publicationCitation", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationCitation2" - }, - "publicationIDType": { - "typeName": "publicationIDType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "arXiv" - }, - "publicationIDNumber": { - "typeName": "publicationIDNumber", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationIDNumber2" - }, - "publicationURL": { - "typeName": "publicationURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://RelatedPublicationURL2.org" - } - } - ] - }, - { - "typeName": "notesText", - "multiple": false, - "typeClass": "primitive", - "value": "Notes1" - }, - { - "typeName": "language", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Abkhaz", - "Afar" - ] - }, - { - "typeName": "producer", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "producerName": { - "typeName": "producerName", - "multiple": false, - "typeClass": "primitive", - "value": "LastProducer1, FirstProducer1" - }, - "producerAffiliation": { - "typeName": "producerAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "ProducerAffiliation1" - }, - "producerAbbreviation": { - "typeName": "producerAbbreviation", - "multiple": false, - "typeClass": "primitive", - "value": "ProducerAbbreviation1" - }, - "producerURL": { - "typeName": "producerURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://ProducerURL1.org" - }, - "producerLogoURL": { - "typeName": "producerLogoURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://ProducerLogoURL1.org" - } - }, - { - "producerName": { - "typeName": "producerName", - "multiple": false, - "typeClass": "primitive", - "value": "LastProducer2, FirstProducer2" - }, - "producerAffiliation": { - "typeName": "producerAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "ProducerAffiliation2" - }, - "producerAbbreviation": { - "typeName": "producerAbbreviation", - "multiple": false, - "typeClass": "primitive", - "value": "ProducerAbbreviation2" - }, - "producerURL": { - "typeName": "producerURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://ProducerURL2.org" - }, - "producerLogoURL": { - "typeName": "producerLogoURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://ProducerLogoURL2.org" - } - } - ] - }, - { - "typeName": "productionDate", - "multiple": false, - "typeClass": "primitive", - "value": "1003-01-01" - }, - { - "typeName": "productionPlace", - "multiple": true, - "typeClass": "primitive", - "value": ["ProductionPlace"] - }, - { - "typeName": "contributor", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "contributorType": { - "typeName": "contributorType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "Data Collector" - }, - "contributorName": { - "typeName": "contributorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastContributor1, FirstContributor1" - } - }, - { - "contributorType": { - "typeName": "contributorType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "Data Curator" - }, - "contributorName": { - "typeName": "contributorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastContributor2, FirstContributor2" - } - } - ] - }, - { - "typeName": "grantNumber", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "grantNumberAgency": { - "typeName": "grantNumberAgency", - "multiple": false, - "typeClass": "primitive", - "value": "GrantInformationGrantAgency1" - }, - "grantNumberValue": { - "typeName": "grantNumberValue", - "multiple": false, - "typeClass": "primitive", - "value": "GrantInformationGrantNumber1" - } - }, - { - "grantNumberAgency": { - "typeName": "grantNumberAgency", - "multiple": false, - "typeClass": "primitive", - "value": "GrantInformationGrantAgency2" - }, - "grantNumberValue": { - "typeName": "grantNumberValue", - "multiple": false, - "typeClass": "primitive", - "value": "GrantInformationGrantNumber2" - } - } - ] - }, - { - "typeName": "distributor", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "distributorName": { - "typeName": "distributorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastDistributor1, FirstDistributor1" - }, - "distributorAffiliation": { - "typeName": "distributorAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "DistributorAffiliation1" - }, - "distributorAbbreviation": { - "typeName": "distributorAbbreviation", - "multiple": false, - "typeClass": "primitive", - "value": "DistributorAbbreviation1" - }, - "distributorURL": { - "typeName": "distributorURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://DistributorURL1.org" - }, - "distributorLogoURL": { - "typeName": "distributorLogoURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://DistributorLogoURL1.org" - } - }, - { - "distributorName": { - "typeName": "distributorName", - "multiple": false, - "typeClass": "primitive", - "value": "LastDistributor2, FirstDistributor2" - }, - "distributorAffiliation": { - "typeName": "distributorAffiliation", - "multiple": false, - "typeClass": "primitive", - "value": "DistributorAffiliation2" - }, - "distributorAbbreviation": { - "typeName": "distributorAbbreviation", - "multiple": false, - "typeClass": "primitive", - "value": "DistributorAbbreviation2" - }, - "distributorURL": { - "typeName": "distributorURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://DistributorURL2.org" - }, - "distributorLogoURL": { - "typeName": "distributorLogoURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://DistributorLogoURL2.org" - } - } - ] - }, - { - "typeName": "distributionDate", - "multiple": false, - "typeClass": "primitive", - "value": "1004-01-01" - }, - { - "typeName": "depositor", - "multiple": false, - "typeClass": "primitive", - "value": "LastDepositor, FirstDepositor" - }, - { - "typeName": "dateOfDeposit", - "multiple": false, - "typeClass": "primitive", - "value": "1002-01-01" - }, - { - "typeName": "timePeriodCovered", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "timePeriodCoveredStart": { - "typeName": "timePeriodCoveredStart", - "multiple": false, - "typeClass": "primitive", - "value": "1005-01-01" - }, - "timePeriodCoveredEnd": { - "typeName": "timePeriodCoveredEnd", - "multiple": false, - "typeClass": "primitive", - "value": "1005-01-02" - } - }, - { - "timePeriodCoveredStart": { - "typeName": "timePeriodCoveredStart", - "multiple": false, - "typeClass": "primitive", - "value": "1005-02-01" - }, - "timePeriodCoveredEnd": { - "typeName": "timePeriodCoveredEnd", - "multiple": false, - "typeClass": "primitive", - "value": "1005-02-02" - } - } - ] - }, - { - "typeName": "dateOfCollection", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "dateOfCollectionStart": { - "typeName": "dateOfCollectionStart", - "multiple": false, - "typeClass": "primitive", - "value": "1006-01-01" - }, - "dateOfCollectionEnd": { - "typeName": "dateOfCollectionEnd", - "multiple": false, - "typeClass": "primitive", - "value": "1006-01-01" - } - }, - { - "dateOfCollectionStart": { - "typeName": "dateOfCollectionStart", - "multiple": false, - "typeClass": "primitive", - "value": "1006-02-01" - }, - "dateOfCollectionEnd": { - "typeName": "dateOfCollectionEnd", - "multiple": false, - "typeClass": "primitive", - "value": "1006-02-02" - } - } - ] - }, - { - "typeName": "kindOfData", - "multiple": true, - "typeClass": "primitive", - "value": [ - "KindOfData1", - "KindOfData2" - ] - }, - { - "typeName": "series", - "multiple": true, - "typeClass": "compound", - "value": [{ - "seriesName": { - "typeName": "seriesName", - "multiple": false, - "typeClass": "primitive", - "value": "SeriesName" - }, - "seriesInformation": { - "typeName": "seriesInformation", - "multiple": false, - "typeClass": "primitive", - "value": "SeriesInformation" - } - }] - }, - { - "typeName": "software", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "softwareName": { - "typeName": "softwareName", - "multiple": false, - "typeClass": "primitive", - "value": "SoftwareName1" - }, - "softwareVersion": { - "typeName": "softwareVersion", - "multiple": false, - "typeClass": "primitive", - "value": "SoftwareVersion1" - } - }, - { - "softwareName": { - "typeName": "softwareName", - "multiple": false, - "typeClass": "primitive", - "value": "SoftwareName2" - }, - "softwareVersion": { - "typeName": "softwareVersion", - "multiple": false, - "typeClass": "primitive", - "value": "SoftwareVersion2" - } - } - ] - }, - { - "typeName": "relatedMaterial", - "multiple": true, - "typeClass": "primitive", - "value": [ - "RelatedMaterial1", - "RelatedMaterial2" - ] - }, - { - "typeName": "relatedDatasets", - "multiple": true, - "typeClass": "primitive", - "value": [ - "RelatedDatasets1", - "RelatedDatasets2" - ] - }, - { - "typeName": "otherReferences", - "multiple": true, - "typeClass": "primitive", - "value": [ - "OtherReferences1", - "OtherReferences2" - ] - }, - { - "typeName": "dataSources", - "multiple": true, - "typeClass": "primitive", - "value": [ - "DataSources1", - "DataSources2" - ] - }, - { - "typeName": "originOfSources", - "multiple": false, - "typeClass": "primitive", - "value": "OriginOfSources" - }, - { - "typeName": "characteristicOfSources", - "multiple": false, - "typeClass": "primitive", - "value": "CharacteristicOfSourcesNoted" - }, - { - "typeName": "accessToSources", - "multiple": false, - "typeClass": "primitive", - "value": "DocumentationAndAccessToSources" - } - ] - }, - "geospatial": { - "displayName": "Geospatial Metadata", - "fields": [ - { - "typeName": "geographicCoverage", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "country": { - "typeName": "country", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "Afghanistan" - }, - "state": { - "typeName": "state", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageStateProvince1" - }, - "city": { - "typeName": "city", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageCity1" - }, - "otherGeographicCoverage": { - "typeName": "otherGeographicCoverage", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageOther1" - } - }, - { - "country": { - "typeName": "country", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "Albania" - }, - "state": { - "typeName": "state", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageStateProvince2" - }, - "city": { - "typeName": "city", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageCity2" - }, - "otherGeographicCoverage": { - "typeName": "otherGeographicCoverage", - "multiple": false, - "typeClass": "primitive", - "value": "GeographicCoverageOther2" - } - } - ] - }, - { - "typeName": "geographicUnit", - "multiple": true, - "typeClass": "primitive", - "value": [ - "GeographicUnit1", - "GeographicUnit2" - ] - }, - { - "typeName": "geographicBoundingBox", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "westLongitude": { - "typeName": "westLongitude", - "multiple": false, - "typeClass": "primitive", - "value": "-72" - }, - "eastLongitude": { - "typeName": "eastLongitude", - "multiple": false, - "typeClass": "primitive", - "value": "-70" - }, - "northLatitude": { - "typeName": "northLatitude", - "multiple": false, - "typeClass": "primitive", - "value": "43" - }, - "southLatitude": { - "typeName": "southLatitude", - "multiple": false, - "typeClass": "primitive", - "value": "42" - } - }, - { - "westLongitude": { - "typeName": "westLongitude", - "multiple": false, - "typeClass": "primitive", - "value": "-18" - }, - "eastLongitude": { - "typeName": "eastLongitude", - "multiple": false, - "typeClass": "primitive", - "value": "-13" - }, - "northLatitude": { - "typeName": "northLatitude", - "multiple": false, - "typeClass": "primitive", - "value": "29" - }, - "southLatitude": { - "typeName": "southLatitude", - "multiple": false, - "typeClass": "primitive", - "value": "28" - } - } - ] - } - ] - }, - "socialscience": { - "displayName": "Social Science and Humanities Metadata", - "fields": [ - { - "typeName": "unitOfAnalysis", - "multiple": true, - "typeClass": "primitive", - "value": [ - "UnitOfAnalysis1", - "UnitOfAnalysis2" - ] - }, - { - "typeName": "universe", - "multiple": true, - "typeClass": "primitive", - "value": [ - "Universe1", - "Universe2" - ] - }, - { - "typeName": "timeMethod", - "multiple": false, - "typeClass": "primitive", - "value": "TimeMethod" - }, - { - "typeName": "dataCollector", - "multiple": false, - "typeClass": "primitive", - "value": "LastDataCollector1, FirstDataCollector1" - }, - { - "typeName": "collectorTraining", - "multiple": false, - "typeClass": "primitive", - "value": "CollectorTraining" - }, - { - "typeName": "frequencyOfDataCollection", - "multiple": false, - "typeClass": "primitive", - "value": "Frequency" - }, - { - "typeName": "samplingProcedure", - "multiple": false, - "typeClass": "primitive", - "value": "SamplingProcedure" - }, - { - "typeName": "targetSampleSize", - "multiple": false, - "typeClass": "compound", - "value": { - "targetSampleActualSize": { - "typeName": "targetSampleActualSize", - "multiple": false, - "typeClass": "primitive", - "value": "100" - }, - "targetSampleSizeFormula": { - "typeName": "targetSampleSizeFormula", - "multiple": false, - "typeClass": "primitive", - "value": "TargetSampleSizeFormula" - } - } - }, - { - "typeName": "deviationsFromSampleDesign", - "multiple": false, - "typeClass": "primitive", - "value": "MajorDeviationsForSampleDesign" - }, - { - "typeName": "collectionMode", - "multiple": true, - "typeClass": "primitive", - "value": ["CollectionMode"] - }, - { - "typeName": "researchInstrument", - "multiple": false, - "typeClass": "primitive", - "value": "TypeOfResearchInstrument" - }, - { - "typeName": "dataCollectionSituation", - "multiple": false, - "typeClass": "primitive", - "value": "CharacteristicsOfDataCollectionSituation" - }, - { - "typeName": "actionsToMinimizeLoss", - "multiple": false, - "typeClass": "primitive", - "value": "ActionsToMinimizeLosses" - }, - { - "typeName": "controlOperations", - "multiple": false, - "typeClass": "primitive", - "value": "ControlOperations" - }, - { - "typeName": "weighting", - "multiple": false, - "typeClass": "primitive", - "value": "Weighting" - }, - { - "typeName": "cleaningOperations", - "multiple": false, - "typeClass": "primitive", - "value": "CleaningOperations" - }, - { - "typeName": "datasetLevelErrorNotes", - "multiple": false, - "typeClass": "primitive", - "value": "StudyLevelErrorNotes" - }, - { - "typeName": "responseRate", - "multiple": false, - "typeClass": "primitive", - "value": "ResponseRate" - }, - { - "typeName": "samplingErrorEstimates", - "multiple": false, - "typeClass": "primitive", - "value": "EstimatesOfSamplingError" - }, - { - "typeName": "otherDataAppraisal", - "multiple": false, - "typeClass": "primitive", - "value": "OtherFormsOfDataAppraisal" - }, - { - "typeName": "socialScienceNotes", - "multiple": false, - "typeClass": "compound", - "value": { - "socialScienceNotesType": { - "typeName": "socialScienceNotesType", - "multiple": false, - "typeClass": "primitive", - "value": "NotesType" - }, - "socialScienceNotesSubject": { - "typeName": "socialScienceNotesSubject", - "multiple": false, - "typeClass": "primitive", - "value": "NotesSubject" - }, - "socialScienceNotesText": { - "typeName": "socialScienceNotesText", - "multiple": false, - "typeClass": "primitive", - "value": "NotesText" - } - } - } - ] - }, - "astrophysics": { - "displayName": "Astronomy and Astrophysics Metadata", - "fields": [ - { - "typeName": "astroType", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Image", - "Mosaic", - "EventList", - "Cube" - ] - }, - { - "typeName": "astroFacility", - "multiple": true, - "typeClass": "primitive", - "value": [ - "Facility1", - "Facility2" - ] - }, - { - "typeName": "astroInstrument", - "multiple": true, - "typeClass": "primitive", - "value": [ - "Instrument1", - "Instrument2" - ] - }, - { - "typeName": "astroObject", - "multiple": true, - "typeClass": "primitive", - "value": [ - "Object1", - "Object2" - ] - }, - { - "typeName": "resolution.Spatial", - "multiple": false, - "typeClass": "primitive", - "value": "SpatialResolution" - }, - { - "typeName": "resolution.Spectral", - "multiple": false, - "typeClass": "primitive", - "value": "SpectralResolution" - }, - { - "typeName": "resolution.Temporal", - "multiple": false, - "typeClass": "primitive", - "value": "TimeResolution" - }, - { - "typeName": "coverage.Spectral.Bandpass", - "multiple": true, - "typeClass": "primitive", - "value": [ - "Bandpass1", - "Bandpass2" - ] - }, - { - "typeName": "coverage.Spectral.CentralWavelength", - "multiple": true, - "typeClass": "primitive", - "value": [ - "3001", - "3002" - ] - }, - { - "typeName": "coverage.Spectral.Wavelength", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "coverage.Spectral.MinimumWavelength": { - "typeName": "coverage.Spectral.MinimumWavelength", - "multiple": false, - "typeClass": "primitive", - "value": "4001" - }, - "coverage.Spectral.MaximumWavelength": { - "typeName": "coverage.Spectral.MaximumWavelength", - "multiple": false, - "typeClass": "primitive", - "value": "4002" - } - }, - { - "coverage.Spectral.MinimumWavelength": { - "typeName": "coverage.Spectral.MinimumWavelength", - "multiple": false, - "typeClass": "primitive", - "value": "4003" - }, - "coverage.Spectral.MaximumWavelength": { - "typeName": "coverage.Spectral.MaximumWavelength", - "multiple": false, - "typeClass": "primitive", - "value": "4004" - } - } - ] - }, - { - "typeName": "coverage.Temporal", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "coverage.Temporal.StartTime": { - "typeName": "coverage.Temporal.StartTime", - "multiple": false, - "typeClass": "primitive", - "value": "1007-01-01" - }, - "coverage.Temporal.StopTime": { - "typeName": "coverage.Temporal.StopTime", - "multiple": false, - "typeClass": "primitive", - "value": "1007-01-02" - } - }, - { - "coverage.Temporal.StartTime": { - "typeName": "coverage.Temporal.StartTime", - "multiple": false, - "typeClass": "primitive", - "value": "1007-02-01" - }, - "coverage.Temporal.StopTime": { - "typeName": "coverage.Temporal.StopTime", - "multiple": false, - "typeClass": "primitive", - "value": "1007-02-02" - } - } - ] - }, - { - "typeName": "coverage.Spatial", - "multiple": true, - "typeClass": "primitive", - "value": [ - "SkyCoverage1", - "SkyCoverage2" - ] - }, - { - "typeName": "coverage.Depth", - "multiple": false, - "typeClass": "primitive", - "value": "200" - }, - { - "typeName": "coverage.ObjectDensity", - "multiple": false, - "typeClass": "primitive", - "value": "300" - }, - { - "typeName": "coverage.ObjectCount", - "multiple": false, - "typeClass": "primitive", - "value": "400" - }, - { - "typeName": "coverage.SkyFraction", - "multiple": false, - "typeClass": "primitive", - "value": "500" - }, - { - "typeName": "coverage.Polarization", - "multiple": false, - "typeClass": "primitive", - "value": "Polarization" - }, - { - "typeName": "redshiftType", - "multiple": false, - "typeClass": "primitive", - "value": "RedshiftType" - }, - { - "typeName": "resolution.Redshift", - "multiple": false, - "typeClass": "primitive", - "value": "600" - }, - { - "typeName": "coverage.RedshiftValue", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "coverage.Redshift.MinimumValue": { - "typeName": "coverage.Redshift.MinimumValue", - "multiple": false, - "typeClass": "primitive", - "value": "701" - }, - "coverage.Redshift.MaximumValue": { - "typeName": "coverage.Redshift.MaximumValue", - "multiple": false, - "typeClass": "primitive", - "value": "702" - } - }, - { - "coverage.Redshift.MinimumValue": { - "typeName": "coverage.Redshift.MinimumValue", - "multiple": false, - "typeClass": "primitive", - "value": "703" - }, - "coverage.Redshift.MaximumValue": { - "typeName": "coverage.Redshift.MaximumValue", - "multiple": false, - "typeClass": "primitive", - "value": "704" - } - } - ] - } - ] - }, - "biomedical": { - "displayName": "Life Sciences Metadata", - "fields": [ - { - "typeName": "studyDesignType", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Case Control", - "Cross Sectional", - "Cohort Study", - "Not Specified" - ] - }, - { - "typeName": "studyFactorType", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Age", - "Biomarkers", - "Cell Surface Markers", - "Developmental Stage" - ] - }, - { - "typeName": "studyAssayOrganism", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "Arabidopsis thaliana", - "Bos taurus", - "Caenorhabditis elegans", - "Danio rerio (zebrafish)" - ] - }, - { - "typeName": "studyAssayOtherOrganism", - "multiple": true, - "typeClass": "primitive", - "value": [ - "OtherOrganism1", - "OtherOrganism2" - ] - }, - { - "typeName": "studyAssayMeasurementType", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "genome sequencing", - "cell sorting", - "clinical chemistry analysis", - "DNA methylation profiling" - ] - }, - { - "typeName": "studyAssayOtherMeasurmentType", - "multiple": true, - "typeClass": "primitive", - "value": [ - "OtherMeasurementType1", - "OtherMeasurementType2" - ] - }, - { - "typeName": "studyAssayTechnologyType", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "culture based drug susceptibility testing, single concentration", - "culture based drug susceptibility testing, two concentrations", - "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", - "flow cytometry" - ] - }, - { - "typeName": "studyAssayPlatform", - "multiple": true, - "typeClass": "controlledVocabulary", - "value": [ - "210-MS GC Ion Trap (Varian)", - "220-MS GC Ion Trap (Varian)", - "225-MS GC Ion Trap (Varian)", - "300-MS quadrupole GC/MS (Varian)" - ] - }, - { - "typeName": "studyAssayCellType", - "multiple": true, - "typeClass": "primitive", - "value": [ - "CellType1", - "CellType2" - ] - } - ] - }, - "journal": { - "displayName": "Journal Metadata", - "fields": [ - { - "typeName": "journalVolumeIssue", - "multiple": true, - "typeClass": "compound", - "value": [ - { - "journalVolume": { - "typeName": "journalVolume", - "multiple": false, - "typeClass": "primitive", - "value": "JournalVolume1" - }, - "journalIssue": { - "typeName": "journalIssue", - "multiple": false, - "typeClass": "primitive", - "value": "JournalIssue1" - }, - "journalPubDate": { - "typeName": "journalPubDate", - "multiple": false, - "typeClass": "primitive", - "value": "1008-01-01" - } - }, - { - "journalVolume": { - "typeName": "journalVolume", - "multiple": false, - "typeClass": "primitive", - "value": "JournalVolume2" - }, - "journalIssue": { - "typeName": "journalIssue", - "multiple": false, - "typeClass": "primitive", - "value": "JournalIssue2" - }, - "journalPubDate": { - "typeName": "journalPubDate", - "multiple": false, - "typeClass": "primitive", - "value": "1008-02-01" - } - } - ] - }, - { - "typeName": "journalArticleType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "abstract" - } - ] - } - } - } -} + "datasetVersion" : { + "license" : { + "name" : "CC0 1.0", + "uri" : "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks" : { + "citation" : { + "displayName" : "Citation Metadata", + "fields" : [{ + "typeName" : "title", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Replication Data for: Title" + }, { + "typeName" : "subtitle", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Subtitle" + }, { + "typeName" : "alternativeTitle", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Alternative Title" + ] + }, { + "typeName" : "alternativeURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://AlternativeURL.org" + }, { + "typeName" : "otherId", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "otherIdAgency" : { + "typeName" : "otherIdAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDAgency1" + }, + "otherIdValue" : { + "typeName" : "otherIdValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDIdentifier1" + } + }, { + "otherIdAgency" : { + "typeName" : "otherIdAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDAgency2" + }, + "otherIdValue" : { + "typeName" : "otherIdValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherIDIdentifier2" + } + } + ] + }, { + "typeName" : "author", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "authorName" : { + "typeName" : "authorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastAuthor1, FirstAuthor1" + }, + "authorAffiliation" : { + "typeName" : "authorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorAffiliation1" + }, + "authorIdentifierScheme" : { + "typeName" : "authorIdentifierScheme", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ORCID" + }, + "authorIdentifier" : { + "typeName" : "authorIdentifier", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorIdentifier1" + } + }, { + "authorName" : { + "typeName" : "authorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastAuthor2, FirstAuthor2" + }, + "authorAffiliation" : { + "typeName" : "authorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorAffiliation2" + }, + "authorIdentifierScheme" : { + "typeName" : "authorIdentifierScheme", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ISNI" + }, + "authorIdentifier" : { + "typeName" : "authorIdentifier", + "multiple" : false, + "typeClass" : "primitive", + "value" : "AuthorIdentifier2" + } + } + ] + }, { + "typeName" : "datasetContact", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "datasetContactName" : { + "typeName" : "datasetContactName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContact1, FirstContact1" + }, + "datasetContactAffiliation" : { + "typeName" : "datasetContactAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactAffiliation1" + }, + "datasetContactEmail" : { + "typeName" : "datasetContactEmail", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactEmail1@mailinator.com" + } + }, { + "datasetContactName" : { + "typeName" : "datasetContactName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContact2, FirstContact2" + }, + "datasetContactAffiliation" : { + "typeName" : "datasetContactAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactAffiliation2" + }, + "datasetContactEmail" : { + "typeName" : "datasetContactEmail", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ContactEmail2@mailinator.com" + } + } + ] + }, { + "typeName" : "dsDescription", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "dsDescriptionValue" : { + "typeName" : "dsDescriptionValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DescriptionText1" + }, + "dsDescriptionDate" : { + "typeName" : "dsDescriptionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1000-01-01" + } + }, { + "dsDescriptionValue" : { + "typeName" : "dsDescriptionValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DescriptionText2" + }, + "dsDescriptionDate" : { + "typeName" : "dsDescriptionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1000-02-02" + } + } + ] + }, { + "typeName" : "subject", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Agricultural Sciences", + "Business and Management", + "Engineering", + "Law" + ] + }, { + "typeName" : "keyword", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "keywordValue" : { + "typeName" : "keywordValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordTerm1" + }, + "keywordTermURI" : { + "typeName" : "keywordTermURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://keywordTermURI1.org" + }, + "keywordVocabulary" : { + "typeName" : "keywordVocabulary", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordVocabulary1" + }, + "keywordVocabularyURI" : { + "typeName" : "keywordVocabularyURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://KeywordVocabularyURL1.org" + } + }, { + "keywordValue" : { + "typeName" : "keywordValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordTerm2" + }, + "keywordTermURI" : { + "typeName" : "keywordTermURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://keywordTermURI2.org" + }, + "keywordVocabulary" : { + "typeName" : "keywordVocabulary", + "multiple" : false, + "typeClass" : "primitive", + "value" : "KeywordVocabulary2" + }, + "keywordVocabularyURI" : { + "typeName" : "keywordVocabularyURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://KeywordVocabularyURL2.org" + } + } + ] + }, { + "typeName" : "topicClassification", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "topicClassValue" : { + "typeName" : "topicClassValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Term1" + }, + "topicClassVocab" : { + "typeName" : "topicClassVocab", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Vocab1" + }, + "topicClassVocabURI" : { + "typeName" : "topicClassVocabURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "https://TopicClassificationURL1.com" + } + }, { + "topicClassValue" : { + "typeName" : "topicClassValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Term2" + }, + "topicClassVocab" : { + "typeName" : "topicClassVocab", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Topic Classification Vocab2" + }, + "topicClassVocabURI" : { + "typeName" : "topicClassVocabURI", + "multiple" : false, + "typeClass" : "primitive", + "value" : "https://TopicClassificationURL2.com" + } + } + ] + }, { + "typeName" : "publication", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "publicationRelationType" : { + "typeName" : "publicationRelationType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "IsSupplementTo" + }, + "publicationCitation" : { + "typeName" : "publicationCitation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationCitation1" + }, + "publicationIDType" : { + "typeName" : "publicationIDType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "ark" + }, + "publicationIDNumber" : { + "typeName" : "publicationIDNumber", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationIDNumber1" + }, + "publicationURL" : { + "typeName" : "publicationURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://RelatedPublicationURL1.org" + } + }, { + "publicationCitation" : { + "typeName" : "publicationCitation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationCitation2" + }, + "publicationIDType" : { + "typeName" : "publicationIDType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "arXiv" + }, + "publicationIDNumber" : { + "typeName" : "publicationIDNumber", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RelatedPublicationIDNumber2" + }, + "publicationURL" : { + "typeName" : "publicationURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://RelatedPublicationURL2.org" + } + } + ] + }, { + "typeName" : "notesText", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Notes1" + }, { + "typeName" : "language", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Abkhaz", + "Afar" + ] + }, { + "typeName" : "producer", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "producerName" : { + "typeName" : "producerName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastProducer1, FirstProducer1" + }, + "producerAffiliation" : { + "typeName" : "producerAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAffiliation1" + }, + "producerAbbreviation" : { + "typeName" : "producerAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAbbreviation1" + }, + "producerURL" : { + "typeName" : "producerURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerURL1.org" + }, + "producerLogoURL" : { + "typeName" : "producerLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerLogoURL1.org" + } + }, { + "producerName" : { + "typeName" : "producerName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastProducer2, FirstProducer2" + }, + "producerAffiliation" : { + "typeName" : "producerAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAffiliation2" + }, + "producerAbbreviation" : { + "typeName" : "producerAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ProducerAbbreviation2" + }, + "producerURL" : { + "typeName" : "producerURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerURL2.org" + }, + "producerLogoURL" : { + "typeName" : "producerLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://ProducerLogoURL2.org" + } + } + ] + }, { + "typeName" : "productionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1003-01-01" + }, { + "typeName" : "productionPlace", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "ProductionPlace" + ] + }, { + "typeName" : "contributor", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "contributorType" : { + "typeName" : "contributorType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Data Collector" + }, + "contributorName" : { + "typeName" : "contributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContributor1, FirstContributor1" + } + }, { + "contributorType" : { + "typeName" : "contributorType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Data Curator" + }, + "contributorName" : { + "typeName" : "contributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastContributor2, FirstContributor2" + } + } + ] + }, { + "typeName" : "grantNumber", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "grantNumberAgency" : { + "typeName" : "grantNumberAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantAgency1" + }, + "grantNumberValue" : { + "typeName" : "grantNumberValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantNumber1" + } + }, { + "grantNumberAgency" : { + "typeName" : "grantNumberAgency", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantAgency2" + }, + "grantNumberValue" : { + "typeName" : "grantNumberValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GrantInformationGrantNumber2" + } + } + ] + }, { + "typeName" : "distributor", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "distributorName" : { + "typeName" : "distributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDistributor1, FirstDistributor1" + }, + "distributorAffiliation" : { + "typeName" : "distributorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAffiliation1" + }, + "distributorAbbreviation" : { + "typeName" : "distributorAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAbbreviation1" + }, + "distributorURL" : { + "typeName" : "distributorURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorURL1.org" + }, + "distributorLogoURL" : { + "typeName" : "distributorLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorLogoURL1.org" + } + }, { + "distributorName" : { + "typeName" : "distributorName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDistributor2, FirstDistributor2" + }, + "distributorAffiliation" : { + "typeName" : "distributorAffiliation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAffiliation2" + }, + "distributorAbbreviation" : { + "typeName" : "distributorAbbreviation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DistributorAbbreviation2" + }, + "distributorURL" : { + "typeName" : "distributorURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorURL2.org" + }, + "distributorLogoURL" : { + "typeName" : "distributorLogoURL", + "multiple" : false, + "typeClass" : "primitive", + "value" : "http://DistributorLogoURL2.org" + } + } + ] + }, { + "typeName" : "distributionDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1004-01-01" + }, { + "typeName" : "depositor", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDepositor, FirstDepositor" + }, { + "typeName" : "dateOfDeposit", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1002-01-01" + }, { + "typeName" : "timePeriodCovered", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "timePeriodCoveredStart" : { + "typeName" : "timePeriodCoveredStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-01-01" + }, + "timePeriodCoveredEnd" : { + "typeName" : "timePeriodCoveredEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-01-02" + } + }, { + "timePeriodCoveredStart" : { + "typeName" : "timePeriodCoveredStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-02-01" + }, + "timePeriodCoveredEnd" : { + "typeName" : "timePeriodCoveredEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1005-02-02" + } + } + ] + }, { + "typeName" : "dateOfCollection", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "dateOfCollectionStart" : { + "typeName" : "dateOfCollectionStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-01-01" + }, + "dateOfCollectionEnd" : { + "typeName" : "dateOfCollectionEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-01-01" + } + }, { + "dateOfCollectionStart" : { + "typeName" : "dateOfCollectionStart", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-02-01" + }, + "dateOfCollectionEnd" : { + "typeName" : "dateOfCollectionEnd", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1006-02-02" + } + } + ] + }, { + "typeName" : "kindOfData", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "KindOfData1", + "KindOfData2" + ] + }, { + "typeName" : "series", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "seriesName" : { + "typeName" : "seriesName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SeriesName" + }, + "seriesInformation" : { + "typeName" : "seriesInformation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SeriesInformation" + } + } + ] + }, { + "typeName" : "software", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "softwareName" : { + "typeName" : "softwareName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareName1" + }, + "softwareVersion" : { + "typeName" : "softwareVersion", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareVersion1" + } + }, { + "softwareName" : { + "typeName" : "softwareName", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareName2" + }, + "softwareVersion" : { + "typeName" : "softwareVersion", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SoftwareVersion2" + } + } + ] + }, { + "typeName" : "relatedMaterial", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "RelatedMaterial1", + "RelatedMaterial2" + ] + }, { + "typeName" : "relatedDatasets", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "RelatedDatasets1", + "RelatedDatasets2" + ] + }, { + "typeName" : "otherReferences", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherReferences1", + "OtherReferences2" + ] + }, { + "typeName" : "dataSources", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "DataSources1", + "DataSources2" + ] + }, { + "typeName" : "originOfSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OriginOfSources" + }, { + "typeName" : "characteristicOfSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CharacteristicOfSourcesNoted" + }, { + "typeName" : "accessToSources", + "multiple" : false, + "typeClass" : "primitive", + "value" : "DocumentationAndAccessToSources" + } + ] + }, + "geospatial" : { + "displayName" : "Geospatial Metadata", + "fields" : [{ + "typeName" : "geographicCoverage", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "country" : { + "typeName" : "country", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Afghanistan" + }, + "state" : { + "typeName" : "state", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageStateProvince1" + }, + "city" : { + "typeName" : "city", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageCity1" + }, + "otherGeographicCoverage" : { + "typeName" : "otherGeographicCoverage", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageOther1" + } + }, { + "country" : { + "typeName" : "country", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "Albania" + }, + "state" : { + "typeName" : "state", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageStateProvince2" + }, + "city" : { + "typeName" : "city", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageCity2" + }, + "otherGeographicCoverage" : { + "typeName" : "otherGeographicCoverage", + "multiple" : false, + "typeClass" : "primitive", + "value" : "GeographicCoverageOther2" + } + } + ] + }, { + "typeName" : "geographicUnit", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "GeographicUnit1", + "GeographicUnit2" + ] + }, { + "typeName" : "geographicBoundingBox", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "westLongitude" : { + "typeName" : "westLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-72" + }, + "eastLongitude" : { + "typeName" : "eastLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-70" + }, + "northLatitude" : { + "typeName" : "northLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "43" + }, + "southLatitude" : { + "typeName" : "southLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "42" + } + }, { + "westLongitude" : { + "typeName" : "westLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-18" + }, + "eastLongitude" : { + "typeName" : "eastLongitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "-13" + }, + "northLatitude" : { + "typeName" : "northLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "29" + }, + "southLatitude" : { + "typeName" : "southLatitude", + "multiple" : false, + "typeClass" : "primitive", + "value" : "28" + } + } + ] + } + ] + }, + "socialscience" : { + "displayName" : "Social Science and Humanities Metadata", + "fields" : [{ + "typeName" : "unitOfAnalysis", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "UnitOfAnalysis1", + "UnitOfAnalysis2" + ] + }, { + "typeName" : "universe", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Universe1", + "Universe2" + ] + }, { + "typeName" : "timeMethod", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TimeMethod" + }, { + "typeName" : "dataCollector", + "multiple" : false, + "typeClass" : "primitive", + "value" : "LastDataCollector1, FirstDataCollector1" + }, { + "typeName" : "collectorTraining", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CollectorTraining" + }, { + "typeName" : "frequencyOfDataCollection", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Frequency" + }, { + "typeName" : "samplingProcedure", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SamplingProcedure" + }, { + "typeName" : "targetSampleSize", + "multiple" : false, + "typeClass" : "compound", + "value" : { + "targetSampleActualSize" : { + "typeName" : "targetSampleActualSize", + "multiple" : false, + "typeClass" : "primitive", + "value" : "100" + }, + "targetSampleSizeFormula" : { + "typeName" : "targetSampleSizeFormula", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TargetSampleSizeFormula" + } + } + }, { + "typeName" : "deviationsFromSampleDesign", + "multiple" : false, + "typeClass" : "primitive", + "value" : "MajorDeviationsForSampleDesign" + }, { + "typeName" : "collectionMode", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "CollectionMode" + ] + }, { + "typeName" : "researchInstrument", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TypeOfResearchInstrument" + }, { + "typeName" : "dataCollectionSituation", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CharacteristicsOfDataCollectionSituation" + }, { + "typeName" : "actionsToMinimizeLoss", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ActionsToMinimizeLosses" + }, { + "typeName" : "controlOperations", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ControlOperations" + }, { + "typeName" : "weighting", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Weighting" + }, { + "typeName" : "cleaningOperations", + "multiple" : false, + "typeClass" : "primitive", + "value" : "CleaningOperations" + }, { + "typeName" : "datasetLevelErrorNotes", + "multiple" : false, + "typeClass" : "primitive", + "value" : "StudyLevelErrorNotes" + }, { + "typeName" : "responseRate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "ResponseRate" + }, { + "typeName" : "samplingErrorEstimates", + "multiple" : false, + "typeClass" : "primitive", + "value" : "EstimatesOfSamplingError" + }, { + "typeName" : "otherDataAppraisal", + "multiple" : false, + "typeClass" : "primitive", + "value" : "OtherFormsOfDataAppraisal" + }, { + "typeName" : "socialScienceNotes", + "multiple" : false, + "typeClass" : "compound", + "value" : { + "socialScienceNotesType" : { + "typeName" : "socialScienceNotesType", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesType" + }, + "socialScienceNotesSubject" : { + "typeName" : "socialScienceNotesSubject", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesSubject" + }, + "socialScienceNotesText" : { + "typeName" : "socialScienceNotesText", + "multiple" : false, + "typeClass" : "primitive", + "value" : "NotesText" + } + } + } + ] + }, + "astrophysics" : { + "displayName" : "Astronomy and Astrophysics Metadata", + "fields" : [{ + "typeName" : "astroType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Image", + "Mosaic", + "EventList", + "Cube" + ] + }, { + "typeName" : "astroFacility", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Facility1", + "Facility2" + ] + }, { + "typeName" : "astroInstrument", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Instrument1", + "Instrument2" + ] + }, { + "typeName" : "astroObject", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Object1", + "Object2" + ] + }, { + "typeName" : "resolution.Spatial", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SpatialResolution" + }, { + "typeName" : "resolution.Spectral", + "multiple" : false, + "typeClass" : "primitive", + "value" : "SpectralResolution" + }, { + "typeName" : "resolution.Temporal", + "multiple" : false, + "typeClass" : "primitive", + "value" : "TimeResolution" + }, { + "typeName" : "coverage.Spectral.Bandpass", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "Bandpass1", + "Bandpass2" + ] + }, { + "typeName" : "coverage.Spectral.CentralWavelength", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "3001", + "3002" + ] + }, { + "typeName" : "coverage.Spectral.Wavelength", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Spectral.MinimumWavelength" : { + "typeName" : "coverage.Spectral.MinimumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4001" + }, + "coverage.Spectral.MaximumWavelength" : { + "typeName" : "coverage.Spectral.MaximumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4002" + } + }, { + "coverage.Spectral.MinimumWavelength" : { + "typeName" : "coverage.Spectral.MinimumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4003" + }, + "coverage.Spectral.MaximumWavelength" : { + "typeName" : "coverage.Spectral.MaximumWavelength", + "multiple" : false, + "typeClass" : "primitive", + "value" : "4004" + } + } + ] + }, { + "typeName" : "coverage.Temporal", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Temporal.StartTime" : { + "typeName" : "coverage.Temporal.StartTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-01-01" + }, + "coverage.Temporal.StopTime" : { + "typeName" : "coverage.Temporal.StopTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-01-02" + } + }, { + "coverage.Temporal.StartTime" : { + "typeName" : "coverage.Temporal.StartTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-02-01" + }, + "coverage.Temporal.StopTime" : { + "typeName" : "coverage.Temporal.StopTime", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1007-02-02" + } + } + ] + }, { + "typeName" : "coverage.Spatial", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "SkyCoverage1", + "SkyCoverage2" + ] + }, { + "typeName" : "coverage.Depth", + "multiple" : false, + "typeClass" : "primitive", + "value" : "200" + }, { + "typeName" : "coverage.ObjectDensity", + "multiple" : false, + "typeClass" : "primitive", + "value" : "300" + }, { + "typeName" : "coverage.ObjectCount", + "multiple" : false, + "typeClass" : "primitive", + "value" : "400" + }, { + "typeName" : "coverage.SkyFraction", + "multiple" : false, + "typeClass" : "primitive", + "value" : "500" + }, { + "typeName" : "coverage.Polarization", + "multiple" : false, + "typeClass" : "primitive", + "value" : "Polarization" + }, { + "typeName" : "redshiftType", + "multiple" : false, + "typeClass" : "primitive", + "value" : "RedshiftType" + }, { + "typeName" : "resolution.Redshift", + "multiple" : false, + "typeClass" : "primitive", + "value" : "600" + }, { + "typeName" : "coverage.RedshiftValue", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "coverage.Redshift.MinimumValue" : { + "typeName" : "coverage.Redshift.MinimumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "701" + }, + "coverage.Redshift.MaximumValue" : { + "typeName" : "coverage.Redshift.MaximumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "702" + } + }, { + "coverage.Redshift.MinimumValue" : { + "typeName" : "coverage.Redshift.MinimumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "703" + }, + "coverage.Redshift.MaximumValue" : { + "typeName" : "coverage.Redshift.MaximumValue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "704" + } + } + ] + } + ] + }, + "biomedical" : { + "displayName" : "Life Sciences Metadata", + "fields" : [{ + "typeName" : "studyDesignType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Case Control", + "Cross Sectional", + "Cohort Study", + "Not Specified" + ] + }, { + "typeName" : "studyFactorType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Age", + "Biomarkers", + "Cell Surface Markers", + "Developmental Stage" + ] + }, { + "typeName" : "studyAssayOrganism", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "Arabidopsis thaliana", + "Bos taurus", + "Caenorhabditis elegans", + "Danio rerio (zebrafish)" + ] + }, { + "typeName" : "studyAssayOtherOrganism", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherOrganism1", + "OtherOrganism2" + ] + }, { + "typeName" : "studyAssayMeasurementType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "genome sequencing", + "cell sorting", + "clinical chemistry analysis", + "DNA methylation profiling" + ] + }, { + "typeName" : "studyAssayOtherMeasurmentType", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "OtherMeasurementType1", + "OtherMeasurementType2" + ] + }, { + "typeName" : "studyAssayTechnologyType", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "culture based drug susceptibility testing, single concentration", + "culture based drug susceptibility testing, two concentrations", + "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", + "flow cytometry" + ] + }, { + "typeName" : "studyAssayPlatform", + "multiple" : true, + "typeClass" : "controlledVocabulary", + "value" : [ + "210-MS GC Ion Trap (Varian)", + "220-MS GC Ion Trap (Varian)", + "225-MS GC Ion Trap (Varian)", + "300-MS quadrupole GC/MS (Varian)" + ] + }, { + "typeName" : "studyAssayCellType", + "multiple" : true, + "typeClass" : "primitive", + "value" : [ + "CellType1", + "CellType2" + ] + } + ] + }, + "journal" : { + "displayName" : "Journal Metadata", + "fields" : [{ + "typeName" : "journalVolumeIssue", + "multiple" : true, + "typeClass" : "compound", + "value" : [{ + "journalVolume" : { + "typeName" : "journalVolume", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalVolume1" + }, + "journalIssue" : { + "typeName" : "journalIssue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalIssue1" + }, + "journalPubDate" : { + "typeName" : "journalPubDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1008-01-01" + } + }, { + "journalVolume" : { + "typeName" : "journalVolume", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalVolume2" + }, + "journalIssue" : { + "typeName" : "journalIssue", + "multiple" : false, + "typeClass" : "primitive", + "value" : "JournalIssue2" + }, + "journalPubDate" : { + "typeName" : "journalPubDate", + "multiple" : false, + "typeClass" : "primitive", + "value" : "1008-02-01" + } + } + ] + }, { + "typeName" : "journalArticleType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "abstract" + } + ] + } + } + } +} \ No newline at end of file diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 18354f2b1f7..db4f1c9f30c 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -31,55 +31,56 @@ topicClassVocab Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 27 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation topicClassVocabURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 28 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation publication Related Publication The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab none 29 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy - publicationCitation Citation The full bibliographic citation for the related publication textbox 30 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation - publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 31 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme - publicationIDNumber Identifier The identifier for a related publication text 32 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier - publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 33 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE publication citation https://schema.org/distribution - notesText Notes Additional information about the Dataset textbox 34 FALSE FALSE FALSE FALSE TRUE FALSE citation - language Language A language that the Dataset's files is written in text 35 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language - producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 36 FALSE FALSE TRUE FALSE FALSE FALSE citation - producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 37 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation - producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 39 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerURL URL The URL of the producer's website https:// url 40 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerLogoURL Logo URL The URL of the producer's logo https:// url 41
FALSE FALSE FALSE FALSE FALSE FALSE producer citation - productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 42 TRUE FALSE FALSE TRUE FALSE FALSE citation - productionPlace Production Location The location where the data and any related materials were produced or collected text 43 TRUE FALSE TRUE TRUE FALSE FALSE citation - contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 44 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor - contributorType Type Indicates the type of contribution made to the dataset text 45 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation - contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 46 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation - grantNumber Funding Information Information about the Dataset's financial support none 47 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor - grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 49 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 50 FALSE FALSE TRUE FALSE FALSE FALSE citation - distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 51 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation - distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 53 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorURL URL The URL of the distributor's webpage https:// url 54 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 55
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 56 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 57 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 58 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted - timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 59 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage - timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - dateOfCollection Date of Collection The dates when the data were collected or generated none 62 ; FALSE FALSE TRUE FALSE FALSE FALSE citation - dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 64 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 65 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData - series Series Information about the dataset series to which the Dataset belong none 66 : FALSE FALSE TRUE FALSE FALSE FALSE citation - seriesName Name The name of the dataset series text 67 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation - seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 68 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation - software Software Information about the software used to generate the Dataset none 69 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name The name of software used to generate the Dataset text 70 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 71 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation - relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation - otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 74 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references - dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 75 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom - originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation - characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation - accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation + publicationRelationType Relation Type The nature of the relationship between this Dataset and the related publication text 30 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://datacite.org/schema/kernel-4/simpleTypes#relationType + publicationCitation Citation The full bibliographic citation for the related publication textbox 31 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation + publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 32 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme + publicationIDNumber Identifier The identifier for a related publication text 33 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier + publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 34 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE publication citation https://schema.org/distribution + notesText Notes Additional information about the Dataset textbox 35 FALSE FALSE FALSE FALSE TRUE FALSE citation + language Language A language that the Dataset's files is written in text 36 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language + producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 37 FALSE FALSE TRUE FALSE FALSE FALSE citation + producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 38 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation + producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 39 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 40 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerURL URL The URL of the producer's website https:// url 41 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerLogoURL Logo URL The URL of the producer's logo https:// url 42
FALSE FALSE FALSE FALSE FALSE FALSE producer citation + productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 43 TRUE FALSE FALSE TRUE FALSE FALSE citation + productionPlace Production Location The location where the data and any related materials were produced or collected text 44 TRUE FALSE TRUE TRUE FALSE FALSE citation + contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 45 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor + contributorType Type Indicates the type of contribution made to the dataset text 46 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation + contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 47 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation + grantNumber Funding Information Information about the Dataset's financial support none 48 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor + grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 49 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 50 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 51 FALSE FALSE TRUE FALSE FALSE FALSE citation + distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 52 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation + distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 53 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 54 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorURL URL The URL of the distributor's webpage https:// url 55 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 56
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 57 TRUE FALSE FALSE TRUE FALSE FALSE citation + depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 58 FALSE FALSE FALSE FALSE FALSE FALSE citation + dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 59 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted + timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 60 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage + timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 62 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + dateOfCollection Date of Collection The dates when the data were collected or generated none 63 ; FALSE FALSE TRUE FALSE FALSE FALSE citation + dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 64 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 65 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 66 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData + series Series Information about the dataset series to which the Dataset belong none 67 : FALSE FALSE TRUE FALSE FALSE FALSE citation + seriesName Name The name of the dataset series text 68 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation + seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 69 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation + software Software Information about the software used to generate the Dataset none 70 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy + softwareName Name The name of software used to generate the Dataset text 71 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 72 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 73 FALSE FALSE TRUE FALSE FALSE FALSE citation + relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 74 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation + otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 75 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references + dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 76 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom + originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation + accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 79 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 @@ -325,3 +326,9 @@ language Zhuang, Chuang zha 183 zha za Zhuang Chuang language Zulu zul 184 zul zu language Not applicable 185 + publicationRelationType IsCitedBy RT1 1 + publicationRelationType Cites RT2 2 + publicationRelationType IsSupplementTo RT3 3 + publicationRelationType IsSupplementedBy RT4 4 + publicationRelationType IsReferencedBy RT5 5 + publicationRelationType References RT6 6 \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 1313f3415ab..53ab6c7bef7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -40,6 +40,7 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String note = "note"; + public final static String publicationRelationType = "publicationRelationType"; public final static String publicationCitation = "publicationCitation"; public final static String publicationIDType = "publicationIDType"; public final static String publicationIDNumber = "publicationIDNumber"; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java index 6b7cb844f3e..e74a2f26af6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java @@ -28,6 +28,8 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; +import org.apache.commons.lang3.StringUtils; + /** * * @author skraffmi @@ -301,26 +303,35 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO String IDType = ""; String IDNo = ""; String url = ""; + String relationType = null; for (Iterator iterator = foo.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); - if (DatasetFieldConstant.publicationCitation.equals(next.getTypeName())) { - citation = next.getSinglePrimitive(); - } - if (DatasetFieldConstant.publicationIDType.equals(next.getTypeName())) { - IDType = next.getSinglePrimitive(); - } - if (DatasetFieldConstant.publicationIDNumber.equals(next.getTypeName())) { - IDNo = next.getSinglePrimitive(); - } - if (DatasetFieldConstant.publicationURL.equals(next.getTypeName())) { - url = next.getSinglePrimitive(); + switch (next.getTypeName()) { + case DatasetFieldConstant.publicationCitation: + citation = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationIDType: + IDType = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationIDNumber: + IDNo = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationURL: + url = next.getSinglePrimitive(); + break; + case DatasetFieldConstant.publicationRelationType: + relationType = next.getSinglePrimitive(); + break; } } + if(StringUtils.isBlank(relationType)) { + relationType = "isReferencedBy"; + } pubString = appendCommaSeparatedValue(citation, IDType); pubString = appendCommaSeparatedValue(pubString, IDNo); pubString = appendCommaSeparatedValue(pubString, url); if (!pubString.isEmpty()){ - xmlw.writeStartElement(dcFlavor+":"+"isReferencedBy"); + xmlw.writeStartElement(dcFlavor+":" + relationType); xmlw.writeCharacters(pubString); xmlw.writeEndElement(); //relPubl } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index b4b5e597365..dd01750942d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -932,6 +932,7 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe String relatedIdentifierType = null; String relatedIdentifier = null; // is used when relatedIdentifierType variable is not URL String relatedURL = null; // is used when relatedIdentifierType variable is URL + String relationType = null; // is used when relatedIdentifierType variable is URL for (Iterator iterator = fieldDTOs.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); @@ -944,6 +945,9 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe if (DatasetFieldConstant.publicationURL.equals(next.getTypeName())) { relatedURL = next.getSinglePrimitive(); } + if (DatasetFieldConstant.publicationRelationType.equals(next.getTypeName())) { + relationType = next.getSinglePrimitive(); + } } if (StringUtils.isNotBlank(relatedIdentifierType)) { @@ -956,7 +960,10 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe } relatedIdentifier_map.put("relatedIdentifierType", relatedIdentifierType); - relatedIdentifier_map.put("relationType", "IsCitedBy"); + if(relationType== null) { + relationType = "IsCitedBy"; + } + relatedIdentifier_map.put("relationType", relationType); if (StringUtils.containsIgnoreCase(relatedIdentifierType, "url")) { writeFullElement(xmlw, null, "relatedIdentifier", relatedIdentifier_map, relatedURL, language); diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index cb864eb78e9..a52a599cff3 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -31,6 +31,7 @@ datasetfieldtype.topicClassValue.title=Term datasetfieldtype.topicClassVocab.title=Controlled Vocabulary Name datasetfieldtype.topicClassVocabURI.title=Controlled Vocabulary URL datasetfieldtype.publication.title=Related Publication +datasetfieldtype.publicationRelationType.title=Relation Type datasetfieldtype.publicationCitation.title=Citation datasetfieldtype.publicationIDType.title=Identifier Type datasetfieldtype.publicationIDNumber.title=Identifier @@ -110,6 +111,7 @@ datasetfieldtype.topicClassValue.description=A topic or subject term datasetfieldtype.topicClassVocab.description=The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) datasetfieldtype.topicClassVocabURI.description=The URL where one can access information about the term's controlled vocabulary datasetfieldtype.publication.description=The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab +datasetfieldtype.publicationRelationType.description=The nature of the relationship between this Dataset and the related publication datasetfieldtype.publicationCitation.description=The full bibliographic citation for the related publication datasetfieldtype.publicationIDType.description=The type of identifier that uniquely identifies a related publication datasetfieldtype.publicationIDNumber.description=The identifier for a related publication diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 2da15147255..8350c5d9875 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -617,7 +617,7 @@ public void testWriteRelatedIdentifierElement() throws XMLStreamException, IOExc //then assertEquals("" - + "" + + "" + "RelatedPublicationIDNumber1" + "" + "RelatedPublicationIDNumber2" From 93faadeb7aaa8093369d327cc0394c49ee090850 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 13:24:58 -0400 Subject: [PATCH 115/360] missing element for openaireutil test --- .../harvard/iq/dataverse/export/dataset-all-defaults.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index 431f069cb03..6b3c7689bbf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -324,6 +324,12 @@ "typeClass": "compound", "value": [ { + "publicationRelationType": { + "typeName": "publicationRelationType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "IsSupplementTo" + }, "publicationCitation": { "typeName": "publicationCitation", "multiple": false, From c9084e3058045fb54fd960a203588f371ce59b2c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 13:57:02 -0400 Subject: [PATCH 116/360] contributor type null fix --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 9ba1e4e3116..92bcf8b481f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -546,7 +546,10 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X contributor = subField.getValue(); break; case DatasetFieldConstant.contributorType: - contributorType = subField.getValue().replace(" ", ""); + contributorType = subField.getValue(); + if(contributorType!=null) { + contributorType = contributorType.replace(" ", ""); + } break; } } From cdd6d6fb4357fe2e63dbf597faf57d59c8625670 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 16:05:40 -0400 Subject: [PATCH 117/360] add relationType to base code and DataCite XML --- .../iq/dataverse/DatasetRelPublication.java | 29 ++++++++++----- .../harvard/iq/dataverse/DatasetVersion.java | 37 +++++++++++++------ .../pidproviders/doi/XmlMetadataTemplate.java | 12 ++++-- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetRelPublication.java b/src/main/java/edu/harvard/iq/dataverse/DatasetRelPublication.java index 7680ebc16db..a0696ab38d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetRelPublication.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetRelPublication.java @@ -6,7 +6,6 @@ package edu.harvard.iq.dataverse; - /** * * @author skraffmiller @@ -25,10 +24,12 @@ public class DatasetRelPublication { private String description; private boolean replicationData; private int displayOrder; + private String relationType; public int getDisplayOrder() { return displayOrder; } + public void setDisplayOrder(int displayOrder) { this.displayOrder = displayOrder; } @@ -64,8 +65,7 @@ public String getUrl() { public void setUrl(String url) { this.url = url; } - - + public String getTitle() { return title; } @@ -82,12 +82,21 @@ public void setDescription(String description) { this.description = description; } - public boolean isEmpty() { - return ((text==null || text.trim().equals("")) - && (!replicationData) - && (idType==null || idType.trim().equals("")) - && (idNumber==null || idNumber.trim().equals("")) - && (url==null || url.trim().equals(""))); - } + public void setRelationType(String type) { + relationType = type; + + } + + public String getRelationType() { + return relationType; + } + + public boolean isEmpty() { + return ((text == null || text.trim().equals("")) + && (!replicationData) + && (idType == null || idType.trim().equals("")) + && (idNumber == null || idNumber.trim().equals("")) + && (url == null || url.trim().equals(""))); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 6648419216d..eb6fdd4e923 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1356,24 +1356,37 @@ public List getRelatedPublications() { for (DatasetFieldCompoundValue publication : dsf.getDatasetFieldCompoundValues()) { DatasetRelPublication relatedPublication = new DatasetRelPublication(); for (DatasetField subField : publication.getChildDatasetFields()) { - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationCitation)) { - String citation = subField.getDisplayValue(); - relatedPublication.setText(citation); - } - - - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationURL)) { - // We have to avoid using subField.getDisplayValue() here - because the DisplayFormatType - // for this url metadata field is likely set up so that the display value is automatically - // turned into a clickable HTML HREF block, which we don't want to end in our Schema.org JSON-LD output. - // So we want to use the raw value of the field instead, with - // minimal HTML sanitation, just in case (this would be done on all URLs in getDisplayValue()). + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.publicationCitation: + relatedPublication.setText(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationURL: + // We have to avoid using subField.getDisplayValue() here - because the DisplayFormatType + // for this url metadata field is likely set up so that the display value is automatically + // turned into a clickable HTML HREF block, which we don't want to end in our Schema.org + // JSON-LD output. So we want to use the raw value of the field instead, with minimal HTML + // sanitation, just in case (this would be done on all URLs in getDisplayValue()). String url = subField.getValue(); if (StringUtils.isBlank(url) || DatasetField.NA_VALUE.equals(url)) { relatedPublication.setUrl(""); } else { relatedPublication.setUrl(MarkupChecker.sanitizeBasicHTML(url)); } + break; + case DatasetFieldConstant.publicationIDType: + // QDR idType has a trailing : now (Aug 2021) + // Get sanitized value without any display modifications + subField.getDatasetFieldType().setDisplayFormat("#VALUE"); + relatedPublication.setIdType(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationIDNumber: + // Get sanitized value without any display modifications + subField.getDatasetFieldType().setDisplayFormat("#VALUE"); + relatedPublication.setIdNumber(subField.getDisplayValue()); + break; + case DatasetFieldConstant.publicationRelationType: + relatedPublication.setRelationType(subField.getDisplayValue()); + break; } } relatedPublications.add(relatedPublication); diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 92bcf8b481f..03d4de99691 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.ParseException; import java.util.ArrayList; @@ -872,6 +874,10 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th String pubIdType = relatedPub.getIdType(); String identifier = relatedPub.getIdNumber(); String url = relatedPub.getUrl(); + String relationType = relatedPub.getRelationType(); + if(StringUtils.isBlank(relationType)) { + relationType = "IsSupplementTo"; + } /* * Note - with identifier and url fields, it's not clear that there's a single * way those two fields are used for all identifier types. The code here is @@ -921,13 +927,13 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th // For non-URL types, if a URL is given, split the string to get a schemeUri try { - URL relatedUrl = new URL(relatedIdentifier); + URL relatedUrl = new URI(relatedIdentifier).toURL(); String protocol = relatedUrl.getProtocol(); String authority = relatedUrl.getAuthority(); String site = String.format("%s://%s", protocol, authority); relatedIdentifier = relatedIdentifier.substring(site.length()); attributes.put("schemeURI", site); - } catch (MalformedURLException e) { + } catch (URISyntaxException | MalformedURLException e) { // Just an identifier } } @@ -937,7 +943,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th if (pubIdType != null) { attributes.put("relatedIdentifierType", pubIdType); } - attributes.put("relationType", "IsSupplementTo"); + attributes.put("relationType", relationType); relatedIdentifiersWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "relatedIdentifiers", relatedIdentifiersWritten); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "relatedIdentifier", attributes, relatedIdentifier); } From 360d3fac049239cfc4f41b6be83b8ecc0b16b475 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 16:05:57 -0400 Subject: [PATCH 118/360] add relationType to above fold display --- .../iq/dataverse/DatasetVersionUI.java | 32 +++++++++++++------ src/main/webapp/dataset.xhtml | 6 ++-- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java index 55b98c178bb..975de391d8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java @@ -62,14 +62,14 @@ public void setMetadataBlocksForEdit(TreeMap> public DatasetVersionUI initDatasetVersionUI(DatasetVersion datasetVersion, boolean createBlanks) { /*takes in the values of a dataset version and apportions them into lists for - viewing and editng in the dataset page. + viewing and editing in the dataset page. */ setDatasetVersion(datasetVersion); //this.setDatasetAuthors(new ArrayList()); this.setDatasetRelPublications(new ArrayList<>()); - // loop through vaues to get fields for view mode + // loop through values to get fields for view mode for (DatasetField dsf : datasetVersion.getDatasetFields()) { //Special Handling for various fields displayed above tabs in dataset page view. if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.title)) { @@ -114,17 +114,23 @@ public DatasetVersionUI initDatasetVersionUI(DatasetVersion datasetVersion, boo datasetRelPublication.setTitle(dsf.getDatasetFieldType().getLocaleTitle()); datasetRelPublication.setDescription(dsf.getDatasetFieldType().getLocaleDescription()); for (DatasetField subField : relPubVal.getChildDatasetFields()) { - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationCitation)) { + String value = subField.getValue(); + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.publicationCitation: datasetRelPublication.setText(subField.getValue()); - } - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationIDNumber)) { + break; + case DatasetFieldConstant.publicationIDNumber: datasetRelPublication.setIdNumber(subField.getValue()); - } - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationIDType)) { + break; + case DatasetFieldConstant.publicationIDType: datasetRelPublication.setIdType(subField.getValue()); - } - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.publicationURL)) { + break; + case DatasetFieldConstant.publicationURL: datasetRelPublication.setUrl(subField.getValue()); + break; + case DatasetFieldConstant.publicationRelationType: + datasetRelPublication.setRelationType(subField.getValue()); + break; } } this.getDatasetRelPublications().add(datasetRelPublication); @@ -263,6 +269,14 @@ public String getRelPublicationUrl() { } } + public String getRelPublicationRelationType() { + if (!this.datasetRelPublications.isEmpty()) { + return this.getDatasetRelPublications().get(0).getRelationType(); + } else { + return ""; + } + } + public String getUNF() { //todo get UNF to calculate and display here. return ""; diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 13faf9d7f20..4fd91f24c36 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -631,6 +631,7 @@ or !empty DatasetPage.datasetVersionUI.keywordDisplay or !empty DatasetPage.datasetVersionUI.subject.value or !empty DatasetPage.datasetVersionUI.relPublicationCitation + or !empty DatasetPage.datasetVersionUI.relPublicationUrl or !empty DatasetPage.datasetVersionUI.notes.value) and !empty DatasetPage.datasetSummaryFields}"> @@ -650,8 +651,9 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{DatasetPage.datasetVersionUI.datasetRelPublications.get(0).description}">
- - #{DatasetPage.datasetVersionUI.relPublicationId} + + #{DatasetPage.datasetVersionUI.relPublicationId} + From 53ded9e21d81848fc9749acd2e07140b964d21cb Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 16:24:26 -0400 Subject: [PATCH 119/360] typos --- src/main/webapp/dataset.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4fd91f24c36..5c9ccde70a6 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -651,9 +651,9 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{DatasetPage.datasetVersionUI.datasetRelPublications.get(0).description}"> - + #{DatasetPage.datasetVersionUI.relPublicationId} - + From 6aade3a7df43647792a6627d41f871f85c69a676 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 14 Jun 2024 16:43:17 -0400 Subject: [PATCH 120/360] handle blank id no and styling in above fold summary --- src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/resources/css/structure.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 5c9ccde70a6..427a8c88f6c 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -652,7 +652,7 @@ - #{DatasetPage.datasetVersionUI.relPublicationId} + #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 64d07038f2f..274755046e2 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -1163,7 +1163,7 @@ a { color:#3174AF; } -.label-warning a, .bg-citation a, .terms .help-block a, #panelCollapseTOU .help-block a { +.label-warning a, .bg-citation a, .terms .help-block a, #panelCollapseTOU .help-block a, #dataset-summary-metadata a { text-decoration: underline; } .pagination>.disabled>a { From 347971fc3619b5c122fca0b345350b5ba2c24de3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 17 Jun 2024 16:19:57 -0400 Subject: [PATCH 121/360] skip blanks in geo place name entries --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 9ba1e4e3116..f81db1463d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1246,7 +1246,9 @@ private void writeGeoLocations(XMLStreamWriter xmlw, DvObject dvObject) throws X ArrayList placeList = new ArrayList(); for (String placePart : place) { - placeList.add(placePart); + if (!StringUtils.isBlank(placePart)) { + placeList.add(placePart); + } } XmlWriterUtil.writeFullElement(xmlw, "geoLocationPlace", Strings.join(placeList, ", ")); xmlw.writeEndElement(); // From bc5686bd2ccc376e1f12a9020ebd0adcce1ff87e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 17 Jun 2024 16:22:35 -0400 Subject: [PATCH 122/360] use ; to separate kindOfData / resourceTypes --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index f81db1463d6..0c19ba0a6df 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -755,7 +755,7 @@ private void writeResourceType(XMLStreamWriter xmlw, DvObject dvObject) throws X } } if (!kindOfDataValues.isEmpty()) { - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, String.join(", ", kindOfDataValues)); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "resourceType", attributes, String.join(";", kindOfDataValues)); } else { // Write an attribute only element if there are no kindOfData values. From db934bb9af639b71a971e5df5cf9533b766fb35d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 18 Jun 2024 10:29:02 -0400 Subject: [PATCH 123/360] add Time Period as Other Date --- .../pidproviders/doi/XmlMetadataTemplate.java | 68 +++++++++++++------ 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 0c19ba0a6df..8ee1c261bf2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -6,6 +6,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -630,6 +631,7 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea String dateOfDeposit = null; Date releaseDate = null; List datesOfCollection = new ArrayList(); + List timePeriods = new ArrayList(); if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersionForCopy(); @@ -647,6 +649,10 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea break; case DatasetFieldConstant.dateOfCollection: datesOfCollection = dsf.getDatasetFieldCompoundValues(); + break; + case DatasetFieldConstant.timePeriodCovered: + timePeriods = dsf.getDatasetFieldCompoundValues(); + break; } } } @@ -691,30 +697,40 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea break; } } + // Minimal clean-up - useful? Parse/format would remove unused chars, and an + // exception would clear the date so we don't send nonsense + startDate =cleanUpDate(startDate); + endDate =cleanUpDate(endDate); if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { - //Minimal clean-up - useful? Parse/format would remove unused chars, and an exception would clear the date so we don't send nonsense - if(StringUtils.isNotBlank(startDate)) { - try { - Date start = Util.getDateFormat().parse(startDate); - startDate = Util.getDateFormat().format(start); - } catch (ParseException e) { - logger.warning("Could not parse date: " + startDate); - startDate = null; - } - } - if(StringUtils.isNotBlank(endDate)) { - try { - Date end = Util.getDateFormat().parse(endDate); - endDate = Util.getDateFormat().format(end); - } catch (ParseException e) { - logger.warning("Could not parse date: " + endDate); - endDate = null; - }; + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + attributes.put("dateType", "Collected"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, (startDate + "/" + endDate).trim()); + } + } + } + if (timePeriods != null) { + for (DatasetFieldCompoundValue timePeriodFieldValue : timePeriods) { + String startDate = null; + String endDate = null; + + for (DatasetField subField : timePeriodFieldValue.getChildDatasetFields()) { + switch (subField.getDatasetFieldType().getName()) { + case DatasetFieldConstant.timePeriodCoveredStart: + startDate = subField.getValue(); + break; + case DatasetFieldConstant.timePeriodCoveredEnd: + endDate = subField.getValue(); + break; } } + // Minimal clean-up - useful? Parse/format would remove unused chars, and an + // exception would clear the date so we don't send nonsense + startDate = cleanUpDate(startDate); + endDate = cleanUpDate(endDate); if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); - attributes.put("dateType", "Collected"); + attributes.put("dateType", "Other"); + attributes.put("dateInformation", "Time period covered by the data"); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, (startDate + "/" + endDate).trim()); } } @@ -724,6 +740,20 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } } + private String cleanUpDate(String date) { + if(StringUtils.isBlank(date)) { + return null; + } + try { + SimpleDateFormat sdf = Util.getDateFormat(); + Date start = sdf.parse(date); + date = sdf.format(start); + } catch (ParseException e) { + logger.warning("Could not parse date: " + date); + date = null; + } + } + // 9, Language (MA), language private void writeLanguage(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStreamException { // Currently not supported. Spec indicates one 'primary' language. Could send From 9efe5971a62b200ac3de0cca25340dda9ef9eb5e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 18 Jun 2024 15:48:11 -0400 Subject: [PATCH 124/360] support available and updated dates for dataset and file --- .../pidproviders/doi/XmlMetadataTemplate.java | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8ee1c261bf2..eab832d50fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -630,12 +630,37 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea String dateOfProduction = null; String dateOfDeposit = null; Date releaseDate = null; + String publicationDate = null; + boolean isAnUpdate=false; List datesOfCollection = new ArrayList(); List timePeriods = new ArrayList(); - if (dvObject instanceof Dataset d) { + if (dvObject instanceof DataFile df) { + // Find the first released version the file is in to give a published date + List fmds = df.getFileMetadatas(); + for (FileMetadata fmd : fmds) { + DatasetVersion dv = fmd.getDatasetVersion(); + if (dv.isReleased()) { + publicationDate = Util.getDateFormat().format(dv.getReleaseTime()); + break; + } + } + // And the last update is the most recent + for (int i = fmds.size() - 1; i >= 0; i--) { + DatasetVersion dv = fmds.get(i).getDatasetVersion(); + if (dv.isReleased()) { + releaseDate = dv.getReleaseTime(); + break; + } + } + } else if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersionForCopy(); + Long versionNumber = dv.getVersionNumber(); + if (versionNumber != null && !(versionNumber.equals(1) && dv.getMinorVersionNumber().equals(0))) { + isAnUpdate = true; + } releaseDate = dv.getReleaseTime(); + publicationDate = d.getPublicationDateFormattedYYYYMMDD(); for (DatasetField dsf : dv.getDatasetFields()) { switch (dsf.getDatasetFieldType().getName()) { case DatasetFieldConstant.distributionDate: @@ -675,11 +700,17 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, dateOfDeposit); } - if (releaseDate != null) { - String date = Util.getDateFormat().format(releaseDate); + if (publicationDate != null) { datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Available"); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, publicationDate); + } + if (isAnUpdate) { + String date = Util.getDateFormat().format(releaseDate); + datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); + + attributes.put("dateType", "Updated"); XmlWriterUtil.writeFullElementWithAttributes(xmlw, "date", attributes, date); } if (datesOfCollection != null) { @@ -699,8 +730,8 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } // Minimal clean-up - useful? Parse/format would remove unused chars, and an // exception would clear the date so we don't send nonsense - startDate =cleanUpDate(startDate); - endDate =cleanUpDate(endDate); + startDate = cleanUpDate(startDate); + endDate = cleanUpDate(endDate); if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) { datesWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "dates", datesWritten); attributes.put("dateType", "Collected"); @@ -741,17 +772,17 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } private String cleanUpDate(String date) { - if(StringUtils.isBlank(date)) { - return null; - } - try { - SimpleDateFormat sdf = Util.getDateFormat(); - Date start = sdf.parse(date); - date = sdf.format(start); - } catch (ParseException e) { - logger.warning("Could not parse date: " + date); - date = null; + String newDate = null; + if (!StringUtils.isBlank(date)) { + try { + SimpleDateFormat sdf = Util.getDateFormat(); + Date start = sdf.parse(date); + newDate = sdf.format(start); + } catch (ParseException e) { + logger.warning("Could not parse date: " + date); + } } + return newDate; } // 9, Language (MA), language From de3731479eb30205867a0b30fdf3b00a552bb359 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 18 Jun 2024 16:24:15 -0400 Subject: [PATCH 125/360] fix file updated logic --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index eab832d50fd..95fd6de309d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -638,9 +638,11 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea if (dvObject instanceof DataFile df) { // Find the first released version the file is in to give a published date List fmds = df.getFileMetadatas(); + DatasetVersion initialVersion = null; for (FileMetadata fmd : fmds) { DatasetVersion dv = fmd.getDatasetVersion(); if (dv.isReleased()) { + initialVersion = dv; publicationDate = Util.getDateFormat().format(dv.getReleaseTime()); break; } @@ -648,8 +650,9 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea // And the last update is the most recent for (int i = fmds.size() - 1; i >= 0; i--) { DatasetVersion dv = fmds.get(i).getDatasetVersion(); - if (dv.isReleased()) { + if (dv.isReleased() && !dv.equals(initialVersion)) { releaseDate = dv.getReleaseTime(); + isAnUpdate=true; break; } } From 6357d92dbf87011898210ef9a0f2ea56535e4dd4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 18 Jun 2024 16:11:27 -0400 Subject: [PATCH 126/360] fix no relType styling --- .../edu/harvard/iq/dataverse/DatasetVersionUI.java | 10 +++++++--- src/main/webapp/dataset.xhtml | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java index 975de391d8c..225201c1f4c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java @@ -271,10 +271,14 @@ public String getRelPublicationUrl() { public String getRelPublicationRelationType() { if (!this.datasetRelPublications.isEmpty()) { - return this.getDatasetRelPublications().get(0).getRelationType(); - } else { - return ""; + //Add ': ' formatting if relationType exists + String relationType = this.getDatasetRelPublications().get(0).getRelationType(); + if (!relationType.isEmpty()) { + return relationType + ": "; + } } + return ""; + } public String getUNF() { diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 427a8c88f6c..b46688bf27e 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -631,7 +631,6 @@ or !empty DatasetPage.datasetVersionUI.keywordDisplay or !empty DatasetPage.datasetVersionUI.subject.value or !empty DatasetPage.datasetVersionUI.relPublicationCitation - or !empty DatasetPage.datasetVersionUI.relPublicationUrl or !empty DatasetPage.datasetVersionUI.notes.value) and !empty DatasetPage.datasetSummaryFields}"> @@ -651,9 +650,10 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{DatasetPage.datasetVersionUI.datasetRelPublications.get(0).description}">
- - #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} - + + + #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} + From c624c0c92fb555f82a33f69a12103967531a201d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 18 Jun 2024 17:29:38 -0400 Subject: [PATCH 127/360] handle null --- src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java index 225201c1f4c..0a1afe86901 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java @@ -16,6 +16,8 @@ import java.util.List; import java.util.TreeMap; +import org.apache.commons.lang3.StringUtils; + import jakarta.ejb.EJB; import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; @@ -273,7 +275,7 @@ public String getRelPublicationRelationType() { if (!this.datasetRelPublications.isEmpty()) { //Add ': ' formatting if relationType exists String relationType = this.getDatasetRelPublications().get(0).getRelationType(); - if (!relationType.isEmpty()) { + if (!StringUtils.isBlank(relationType)) { return relationType + ": "; } } From 0c2cff14eff0f94905ec87b799a4496d8dee921b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 24 Jun 2024 12:52:19 -0400 Subject: [PATCH 128/360] add HasPart rels - logic issue --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 95fd6de309d..70a3dadedea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -1005,7 +1005,7 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th } } List fmds = dataset.getLatestVersionForCopy().getFileMetadatas(); - if (!(fmds==null) && fmds.isEmpty()) { + if (!((fmds==null) && fmds.isEmpty())) { attributes.clear(); attributes.put("relationType", "HasPart"); for (FileMetadata fmd : fmds) { From a177a08eab9d7d3856fc99027153e2d6ceb82870 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 24 Jun 2024 17:00:13 -0400 Subject: [PATCH 129/360] catch additional exception type --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 70a3dadedea..3697eeeab8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -982,13 +982,13 @@ private void writeRelatedIdentifiers(XMLStreamWriter xmlw, DvObject dvObject) th // For non-URL types, if a URL is given, split the string to get a schemeUri try { - URL relatedUrl = new URL(relatedIdentifier); + URL relatedUrl = new URI(relatedIdentifier).toURL(); String protocol = relatedUrl.getProtocol(); String authority = relatedUrl.getAuthority(); String site = String.format("%s://%s", protocol, authority); relatedIdentifier = relatedIdentifier.substring(site.length()); attributes.put("schemeURI", site); - } catch (MalformedURLException e) { + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { // Just an identifier } } From fed231ff6782ef7b050edb2ed1ee79e75e18d695 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 26 Jun 2024 13:11:49 -0400 Subject: [PATCH 130/360] fix rendering with a span --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index b46688bf27e..9b4882b8487 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -652,7 +652,7 @@ - #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} + #{empty DatasetPage.datasetVersionUI.relPublicationId ? DatasetPage.datasetVersionUI.relPublicationUrl:DatasetPage.datasetVersionUI.relPublicationId} From 08843d8beb309ab9980abe0203c08b4f58d9af12 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 26 Jun 2024 13:35:44 -0400 Subject: [PATCH 131/360] missing imports, null check --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 3697eeeab8f..14ffed91c36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -547,7 +549,10 @@ private void writeContributors(XMLStreamWriter xmlw, DvObject dvObject) throws X contributor = subField.getValue(); break; case DatasetFieldConstant.contributorType: - contributorType = subField.getValue().replace(" ", ""); + contributorType = subField.getValue(); + if(contributorType!=null) { + contributorType = contributorType.replace(" ", ""); + } break; } } From 5df4720277928789f14f94faa2ca4b108a23082d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 3 Jul 2024 14:36:15 -0400 Subject: [PATCH 132/360] avoid making non-managed fields hidden --- .../iq/dataverse/DatasetFieldServiceBean.java | 13 +++++++++++++ .../edu/harvard/iq/dataverse/SettingsWrapper.java | 12 ++++++++++++ src/main/webapp/metadataFragment.xhtml | 7 +++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 34595728fa7..317b6347c00 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -84,6 +84,9 @@ public class DatasetFieldServiceBean implements java.io.Serializable { //Note that for primitive fields, the prent and term-uri-field are the same and these maps have the same entry Map cvocMapByTermUri = null; + //Flat list of cvoc managed fields + Set managedFieldSet = null; + //The hash of the existing CVocConf setting. Used to determine when the setting has changed and it needs to be re-parsed to recreate the cvocMaps String oldHash = null; @@ -282,6 +285,10 @@ public Map getCVocConf(boolean byTermUriField){ String cvocSetting = settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf); if (cvocSetting == null || cvocSetting.isEmpty()) { oldHash=null; + //Release old maps + cvocMap=null; + cvocMapByTermUri=null; + managedFieldSet = null; return new HashMap<>(); } String newHash = DigestUtils.md5Hex(cvocSetting); @@ -291,6 +298,7 @@ public Map getCVocConf(boolean byTermUriField){ oldHash=newHash; cvocMap=new HashMap<>(); cvocMapByTermUri=new HashMap<>(); + managedFieldSet = new HashSet<>(); try (JsonReader jsonReader = Json.createReader(new StringReader(settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf)))) { JsonArray cvocConfJsonArray = jsonReader.readArray(); @@ -331,6 +339,7 @@ public Map getCVocConf(boolean byTermUriField){ + managedFields.getString(s)); } else { logger.fine("Found: " + dft.getName()); + managedFieldSet.add(dft.getName()); } } } @@ -342,6 +351,10 @@ public Map getCVocConf(boolean byTermUriField){ return byTermUriField ? cvocMapByTermUri : cvocMap; } + public Set getManagedFieldSet() { + return managedFieldSet; + } + /** * Adds information about the external vocabulary term being used in this DatasetField to the ExternalVocabularyValue table if it doesn't already exist. * @param df - the primitive/parent compound field containing a newly saved value diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 48196591b19..6e187ae6cef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -98,6 +98,7 @@ public class SettingsWrapper implements java.io.Serializable { //External Vocabulary support private Map cachedCvocMap = null; private Map cachedCvocByTermFieldMap = null; + private Set managedFieldSet; private Long zipDownloadLimit = null; @@ -806,6 +807,17 @@ public Map getCVocConf(boolean byTermField) { } } + public boolean isManagedField(String fieldName) { + + if(managedFieldSet == null) { + managedFieldSet = fieldService.getManagedFieldSet(); + } + if(managedFieldSet == null) { + return false; + } + return managedFieldSet.contains(fieldName); + } + public String getMetricsUrl() { if (metricsUrl == null) { metricsUrl = getValueForKey(SettingsServiceBean.Key.MetricsUrl); diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 43d54f64c43..a2e67fb80c1 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -171,8 +171,11 @@ rendered="#{(cvocOnDsf or cvocOnCvPart) and not ( cvPart.key.datasetFieldType.name.equals(cvocConf.get(cvPart.key.datasetFieldType.id).getString('term-uri-field')) or cvPart.key.datasetFieldType.name.equals(cvocConf.get(dsf.datasetFieldType.id).getString('term-uri-field')))}" - styleClass="hidden"> - + > + + + + From c4f75cfbd537eee2e9c84d436014f0986b8e5b7b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 3 Jul 2024 15:48:14 -0400 Subject: [PATCH 133/360] fix/refactor managedField display --- .../iq/dataverse/DatasetFieldServiceBean.java | 16 +++++++++------- .../harvard/iq/dataverse/SettingsWrapper.java | 12 ++++++------ src/main/webapp/metadataFragment.xhtml | 11 ++++------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 317b6347c00..0f9f9b39f1c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -84,8 +84,8 @@ public class DatasetFieldServiceBean implements java.io.Serializable { //Note that for primitive fields, the prent and term-uri-field are the same and these maps have the same entry Map cvocMapByTermUri = null; - //Flat list of cvoc managed fields - Set managedFieldSet = null; + //Flat list of cvoc term-uri and managed fields by Id + Set cvocFieldSet = null; //The hash of the existing CVocConf setting. Used to determine when the setting has changed and it needs to be re-parsed to recreate the cvocMaps String oldHash = null; @@ -288,7 +288,7 @@ public Map getCVocConf(boolean byTermUriField){ //Release old maps cvocMap=null; cvocMapByTermUri=null; - managedFieldSet = null; + cvocFieldSet = null; return new HashMap<>(); } String newHash = DigestUtils.md5Hex(cvocSetting); @@ -298,7 +298,7 @@ public Map getCVocConf(boolean byTermUriField){ oldHash=newHash; cvocMap=new HashMap<>(); cvocMapByTermUri=new HashMap<>(); - managedFieldSet = new HashSet<>(); + cvocFieldSet = new HashSet<>(); try (JsonReader jsonReader = Json.createReader(new StringReader(settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf)))) { JsonArray cvocConfJsonArray = jsonReader.readArray(); @@ -315,11 +315,13 @@ public Map getCVocConf(boolean byTermUriField){ if (termUriField.equals(dft.getName())) { logger.fine("Found primitive field for term uri : " + dft.getName() + ": " + dft.getId()); cvocMapByTermUri.put(dft.getId(), jo); + cvocFieldSet.add(dft.getId()); } } else { DatasetFieldType childdft = findByNameOpt(jo.getString("term-uri-field")); logger.fine("Found term child field: " + childdft.getName()+ ": " + childdft.getId()); cvocMapByTermUri.put(childdft.getId(), jo); + cvocFieldSet.add(childdft.getId()); if (childdft.getParentDatasetFieldType() != dft) { logger.warning("Term URI field (" + childdft.getDisplayName() + ") not a child of parent: " + dft.getDisplayName()); @@ -339,7 +341,7 @@ public Map getCVocConf(boolean byTermUriField){ + managedFields.getString(s)); } else { logger.fine("Found: " + dft.getName()); - managedFieldSet.add(dft.getName()); + cvocFieldSet.add(dft.getId()); } } } @@ -351,8 +353,8 @@ public Map getCVocConf(boolean byTermUriField){ return byTermUriField ? cvocMapByTermUri : cvocMap; } - public Set getManagedFieldSet() { - return managedFieldSet; + public Set getCvocFieldSet() { + return cvocFieldSet; } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 6e187ae6cef..222d2881cd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -98,7 +98,7 @@ public class SettingsWrapper implements java.io.Serializable { //External Vocabulary support private Map cachedCvocMap = null; private Map cachedCvocByTermFieldMap = null; - private Set managedFieldSet; + private Set cvocFieldSet; private Long zipDownloadLimit = null; @@ -807,15 +807,15 @@ public Map getCVocConf(boolean byTermField) { } } - public boolean isManagedField(String fieldName) { + public boolean isCvocField(Long fieldId) { - if(managedFieldSet == null) { - managedFieldSet = fieldService.getManagedFieldSet(); + if(cvocFieldSet == null) { + cvocFieldSet = fieldService.getCvocFieldSet(); } - if(managedFieldSet == null) { + if(cvocFieldSet == null) { return false; } - return managedFieldSet.contains(fieldName); + return cvocFieldSet.contains(fieldId); } public String getMetricsUrl() { diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index a2e67fb80c1..494940370e1 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -129,8 +129,8 @@ - - + + - - - - + styleClass="hidden"> + From 512da5cdbe762507d5ab486e89429e1c6c2020ca Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 8 Jul 2024 09:30:59 -0400 Subject: [PATCH 134/360] lower logging for plain text ext cvv case --- .../java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 0f9f9b39f1c..5fae2310fb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -487,7 +487,8 @@ public JsonObject getExternalVocabularyValue(String termUri) { logger.warning("Problem parsing external vocab value for uri: " + termUri + " : " + e.getMessage()); } } catch (NoResultException nre) { - logger.warning("No external vocab value for uri: " + termUri); + //Could just be a plain text value + logger.fine("No external vocab value for uri: " + termUri); } return null; } From 22b8ca7a43f8f681c7dc0a6a407e319d1e832710 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Jul 2024 15:18:21 -0400 Subject: [PATCH 135/360] add cvoc to managed internal cvv field --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 494940370e1..a6d409268df 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -381,7 +381,7 @@ -
+
From e20463211bbca844611ed5d2745a3389ea9d7303 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Jul 2024 15:18:55 -0400 Subject: [PATCH 136/360] increase priority/specificity to avoid conflict --- src/main/webapp/resources/css/structure.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 64d07038f2f..17a0e69d85d 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -1069,7 +1069,7 @@ progress::-webkit-progress-value { .select2-selection__clear, .select2-selection__rendered { vertical-align: -webkit-baseline-middle; } -.select2-results li { +.select2-results ul > li { font-weight:400; font-size:14px; } From 920898472d799d17d1431415a6bc4bbc3b586f51 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 11 Jul 2024 16:15:22 -0400 Subject: [PATCH 137/360] add/prioritize getting child field in cvocConf --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index a6d409268df..09704b4e2c7 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -377,7 +377,7 @@ - + From bb56118a51073ba7c11112c06f2293a40d088c2b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 12 Jul 2024 15:59:52 -0400 Subject: [PATCH 138/360] only show hidden cvoc fields if term uri field exists --- .../harvard/iq/dataverse/DatasetFieldCompoundValue.java | 9 +++++++++ src/main/webapp/metadataFragment.xhtml | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index c679cd7edad..c03baec73af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -225,6 +225,15 @@ public Pair getLinkComponents() { return linkComponents.get(parentDatasetField.getDatasetFieldType().getName()); } + public boolean hasChildOfType(String name) { + for (DatasetField child : childDatasetFields) { + if (child.getDatasetFieldType().getName().equals(name)) { + return true; + } + } + return false; + } + private Map removeLastComma(Map mapIn) { Iterator> itr = mapIn.entrySet().iterator(); diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 09704b4e2c7..f6aad71f408 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -126,11 +126,11 @@ - + - + From 95250a2945e220655d9f0fe126699d57e9c19e54 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 12 Jul 2024 16:04:36 -0400 Subject: [PATCH 139/360] typo --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index f6aad71f408..1d247c4ffd1 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -129,7 +129,7 @@ - + From 9063739fb2baaa874132f68f93a6bb49e898f00a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 12 Jul 2024 17:55:48 -0400 Subject: [PATCH 140/360] use getString --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 1d247c4ffd1..0da2953a0d0 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -126,7 +126,7 @@ - + From 4b904cfd503b852b9fa6c2933b0112aa37392c08 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 18 Jul 2024 10:09:25 -0400 Subject: [PATCH 141/360] tweak ror styling --- src/main/webapp/resources/css/structure.css | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 17a0e69d85d..db1e85f7631 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -670,6 +670,16 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container pre { white-space: pre-wrap !important; } + +#metadata_author img { + vertical-align: text-top; + margin-right:1px; + margin-left: 3px; +} + +#metadata_author img.ror { + vertical-align: middle; +} /* TRUNCATION */ div.more-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} button.desc-more-link {margin:0; padding:0;} From 1ccf834ebb52447e597b9b830265cb9cbc03ab56 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 21 Jul 2024 12:28:13 -0400 Subject: [PATCH 142/360] release note --- doc/release-notes/10711-CVoc Updates.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/10711-CVoc Updates.md diff --git a/doc/release-notes/10711-CVoc Updates.md b/doc/release-notes/10711-CVoc Updates.md new file mode 100644 index 00000000000..f747bedb049 --- /dev/null +++ b/doc/release-notes/10711-CVoc Updates.md @@ -0,0 +1 @@ +Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). From 6c87b9e1cca70525b7a26a69ae74f5109466314c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 5 Jul 2024 15:48:43 -0400 Subject: [PATCH 143/360] fix ROR identification --- .../harvard/iq/dataverse/ExternalIdentifier.java | 4 +++- .../pidproviders/doi/XmlMetadataTemplate.java | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java index 0b7285c017e..8c4fb6b1325 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java @@ -12,7 +12,9 @@ public enum ExternalIdentifier { GND("GND", "https://d-nb.info/gnd/%s", "^1[01]?\\d{7}[0-9X]|[47]\\d{6}-\\d|[1-9]\\d{0,7}-[0-9X]|3\\d{7}[0-9X]$"), // note: DAI is missing from this list, because it doesn't have resolvable URL ResearcherID("ResearcherID", "https://publons.com/researcher/%s/", "^[A-Z\\d][A-Z\\d-]+[A-Z\\d]$"), - ScopusID("ScopusID", "https://www.scopus.com/authid/detail.uri?authorId=%s", "^\\d*$"); + ScopusID("ScopusID", "https://www.scopus.com/authid/detail.uri?authorId=%s", "^\\d*$"), + //Requiring ROR to be URL form as we use it where there is no id type field and matching any 9 digit number starting with 0 seems a bit aggressive + ROR("ROR", "https://ror.org/%s", "^(https:\\/\\/ror.org\\/)0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"); private String name; private String template; diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 14ffed91c36..8049ac1fe5f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -46,6 +46,7 @@ import edu.harvard.iq.dataverse.DatasetRelPublication; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.ExternalIdentifier; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; @@ -262,10 +263,10 @@ public void writeCreators(XMLStreamWriter xmlw, List authorList, boolean nothingWritten = true; if (authorList != null && !authorList.isEmpty()) { for (DatasetAuthor author : authorList) { - String creatorName = StringEscapeUtils.escapeXml10(author.getName().getDisplayValue()); + String creatorName = author.getName().getDisplayValue(); String affiliation = null; - if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - affiliation = StringEscapeUtils.escapeXml10(author.getAffiliation().getDisplayValue()); + if (author.getAffiliation() != null && !author.getAffiliation().getValue().isEmpty()) { + affiliation = author.getAffiliation().getValue(); } String nameIdentifier = null; String nameIdentifierScheme = null; @@ -609,8 +610,14 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin } if (StringUtils.isNotBlank(affiliation)) { + boolean isROR=false; + ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; + if (externalIdentifier.isValidIdentifier(affiliation)) { + isROR=true; + } + attributeMap.clear(); - if (affiliation.startsWith("https://ror.org/")) { + if (isROR) { attributeMap.put("schemeURI", "https://ror.org"); attributeMap.put("affiliationIdentifierScheme", "ROR"); From 182cfdd7cc93452fdf822c1ad5907271fd9dca4c Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 9 Jul 2024 15:23:38 -0400 Subject: [PATCH 144/360] passthrough for ext cvv/ROR affiliation update --- .../pidproviders/PidProviderFactoryBean.java | 9 ++++++++ .../pidproviders/doi/XmlMetadataTemplate.java | 22 ++++++++++++++++++- .../doi/datacite/DataCiteDOIProvider.java | 6 +++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java index 40044408c63..2e3b96b2dd7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -23,9 +23,11 @@ import jakarta.ejb.Singleton; import jakarta.ejb.Startup; import jakarta.inject.Inject; +import jakarta.json.JsonObject; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.GlobalId; @@ -58,6 +60,9 @@ public class PidProviderFactoryBean { protected DvObjectServiceBean dvObjectService; @Inject SystemConfig systemConfig; + //QDR - adding to support use of ExternalVocabularyValues in DataCite provider/ DataCite XML generation + @EJB + DatasetFieldServiceBean datasetFieldServiceBean; private ServiceLoader loader; private Map pidProviderFactoryMap = new HashMap<>(); @@ -247,4 +252,8 @@ public PidProvider getDefaultPidGenerator() { return PidUtil.getPidProvider(protocol, authority, shoulder); } } + + public JsonObject getExternalVocabularyValue(String name) { + return datasetFieldServiceBean.getExternalVocabularyValue(name); + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8049ac1fe5f..1544dcd1373 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -58,8 +58,10 @@ import edu.harvard.iq.dataverse.export.DDIExporter; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -79,6 +81,8 @@ public class XmlMetadataTemplate { public static final String XML_SCHEMA_VERSION = "4.5"; private DoiMetadata doiMetadata; + //QDR - used to get ROR name from ExternalVocabularyValue via pidProvider.get + private PidProvider pidProvider = null; public XmlMetadataTemplate() { } @@ -105,6 +109,14 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM // Could/should use dataset metadata language for metadata from DvObject itself? String language = null; // machine locale? e.g. for Publisher which is global String metadataLanguage = null; // when set, otherwise = language? + + //QDR - used to get ROR name from ExternalVocabularyValue via pidProvider.get + GlobalId pid = null; + pid = dvObject.getGlobalId(); + if ((pid == null) && (dvObject instanceof DataFile df)) { + pid = df.getOwner().getGlobalId(); + } + pidProvider = PidUtil.getPidProvider(pid.getProviderId()); XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); xmlw.writeStartElement("resource"); boolean deaccessioned=false; @@ -611,9 +623,16 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin if (StringUtils.isNotBlank(affiliation)) { boolean isROR=false; + String orgName = affiliation; ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; if (externalIdentifier.isValidIdentifier(affiliation)) { isROR=true; + if(pidProvider instanceof DataCiteDOIProvider dcProvider) { + JsonObject jo = dcProvider.getExternalVocabularyValue(affiliation); + if(jo!=null) { + orgName = jo.getString("termName"); + } + } } attributeMap.clear(); @@ -621,8 +640,9 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin attributeMap.put("schemeURI", "https://ror.org"); attributeMap.put("affiliationIdentifierScheme", "ROR"); + attributeMap.put("affiliationIdentifier", affiliation); } - XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(affiliation)); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(orgName)); } xmlw.writeEndElement(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java index cd765933796..0250266bbdd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -17,6 +17,8 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import jakarta.json.JsonObject; + import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpStatus; @@ -339,5 +341,9 @@ public boolean updateIdentifier(DvObject dvObject) { return false; } } + + public JsonObject getExternalVocabularyValue(String name) { + return pidProviderService.getExternalVocabularyValue(name); + } } From e2b88ca58e3f119b745cfea4a80bf8f27a137853 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 5 Jul 2024 15:49:29 -0400 Subject: [PATCH 145/360] Merge branch 'datacite_xml_improvements' into datacite_plus_relPubRelType --- src/main/webapp/dataset.xhtml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4a952aa0412..4ed53e53208 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -631,6 +631,7 @@ or !empty DatasetPage.datasetVersionUI.keywordDisplay or !empty DatasetPage.datasetVersionUI.subject.value or !empty DatasetPage.datasetVersionUI.relPublicationCitation + or !empty DatasetPage.datasetVersionUI.relPublicationUrl or !empty DatasetPage.datasetVersionUI.notes.value) and !empty DatasetPage.datasetSummaryFields}"> From 2124ce685a470a919663141bd763600e99974229 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 5 Jul 2024 16:05:14 -0400 Subject: [PATCH 146/360] add relation type values --- src/main/java/propertyFiles/citation.properties | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index a52a599cff3..69f01769b39 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -273,6 +273,12 @@ controlledvocabulary.publicationIDType.upc=upc controlledvocabulary.publicationIDType.url=url controlledvocabulary.publicationIDType.urn=urn controlledvocabulary.publicationIDType.dash-nrs=DASH-NRS +controlledVocabulary.publicationRelationType.IsCitedBy=Is Cited By +controlledVocabulary.publicationRelationType.Cites=Cites +controlledVocabulary.publicationRelationType.IsSupplementTo=Is Supplement To +controlledVocabulary.publicationRelationType.IsSupplementedBy=Is Supplemented By +controlledVocabulary.publicationRelationType.IsReferencedBy=Is Referenced By +controlledVocabulary.publicationRelationType.References=References controlledvocabulary.contributorType.data_collector=Data Collector controlledvocabulary.contributorType.data_curator=Data Curator controlledvocabulary.contributorType.data_manager=Data Manager From 6e1919aaafe527f8f99c8302d9d0f713d51243fa Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 8 Jul 2024 09:31:10 -0400 Subject: [PATCH 147/360] fix case --- src/main/java/propertyFiles/citation.properties | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index 69f01769b39..1b65b5d391e 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -273,12 +273,12 @@ controlledvocabulary.publicationIDType.upc=upc controlledvocabulary.publicationIDType.url=url controlledvocabulary.publicationIDType.urn=urn controlledvocabulary.publicationIDType.dash-nrs=DASH-NRS -controlledVocabulary.publicationRelationType.IsCitedBy=Is Cited By -controlledVocabulary.publicationRelationType.Cites=Cites -controlledVocabulary.publicationRelationType.IsSupplementTo=Is Supplement To -controlledVocabulary.publicationRelationType.IsSupplementedBy=Is Supplemented By -controlledVocabulary.publicationRelationType.IsReferencedBy=Is Referenced By -controlledVocabulary.publicationRelationType.References=References +controlledVocabulary.publicationRelationType.iscitedby=Is Cited By +controlledVocabulary.publicationRelationType.cites=Cites +controlledVocabulary.publicationRelationType.issupplementto=Is Supplement To +controlledVocabulary.publicationRelationType.issupplementedby=Is Supplemented By +controlledVocabulary.publicationRelationType.isreferencedby=Is Referenced By +controlledVocabulary.publicationRelationType.references=References controlledvocabulary.contributorType.data_collector=Data Collector controlledvocabulary.contributorType.data_curator=Data Curator controlledvocabulary.contributorType.data_manager=Data Manager From 65ec69f97c74078e4db5d7194e1a5fd6a99f9b50 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 5 Aug 2024 17:17:31 -0400 Subject: [PATCH 148/360] a quick experimental AddReplaceFileHelper implementation of adding Globus-uploaded files #10623 --- .../iq/dataverse/EditDatafilesPage.java | 8 ++- .../dataverse/globus/GlobusServiceBean.java | 51 ++++++++++++++----- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 993cb02b66b..99a44058809 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2121,8 +2121,12 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { } /** - * Using information from the DropBox choose, ingest the chosen files - * https://www.dropbox.com/developers/dropins/chooser/js + * External, aka "Direct" Upload. + * The file(s) have been uploaded to physical storage (such as S3) directly, + * this call is to create and add the DataFiles to the Dataset on the Dataverse + * side. The method does NOT finalize saving the datafiles in the database - + * that will happen when the user clicks 'Save', similar to how the "normal" + * uploads are handled. * * @param event */ diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index fb50214c259..02bc4667ea6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -61,6 +61,9 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; import edu.harvard.iq.dataverse.dataaccess.StorageIO; +import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -70,6 +73,7 @@ import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import jakarta.ws.rs.core.Response; @Stateless @Named("GlobusServiceBean") @@ -81,6 +85,8 @@ public class GlobusServiceBean implements java.io.Serializable { protected SettingsServiceBean settingsSvc; @Inject DataverseSession session; + @Inject + DataverseRequestServiceBean dataverseRequestSvc; @EJB protected AuthenticationServiceBean authSvc; @EJB @@ -92,7 +98,13 @@ public class GlobusServiceBean implements java.io.Serializable { @EJB FileDownloadServiceBean fileDownloadService; @EJB - DataFileServiceBean dataFileService; + DataFileServiceBean dataFileSvc; + @EJB + PermissionServiceBean permissionSvc; + @EJB + IngestServiceBean ingestSvc; + @EJB + SystemConfig systemConfig; private static final Logger logger = Logger.getLogger(GlobusServiceBean.class.getCanonicalName()); private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); @@ -764,7 +776,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files"); logger.fine("Size: " + newfilesJsonArray.size()); logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0))); - JsonArrayBuilder jsonDataSecondAPI = Json.createArrayBuilder(); + JsonArrayBuilder addFilesJsonData = Json.createArrayBuilder(); for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { @@ -790,7 +802,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S path = Json.createPatchBuilder() .add("/mimeType", newfileJsonObject.get(0).getString("mime")).build(); fileJsonObject = path.apply(fileJsonObject); - jsonDataSecondAPI.add(fileJsonObject); + addFilesJsonData.add(fileJsonObject); countSuccess++; // } else { // globusLogger.info(fileName @@ -805,20 +817,32 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S } } - String newjsonData = jsonDataSecondAPI.build().toString(); + String newjsonData = addFilesJsonData.build().toString(); - globusLogger.info("Successfully generated new JsonData for Second API call"); + globusLogger.info("Successfully generated new JsonData for addFiles call"); - String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " + /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" + datasetIdentifier + " -F jsonData='" + newjsonData + "'"; - System.out.println("*******====command ==== " + command); + System.out.println("*******====command ==== " + command);*/ // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of // calling API - - String output = addFilesAsync(command, globusLogger); - if (output.equalsIgnoreCase("ok")) { + + // a quick experimental AddReplaceFileHelper implementation: + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( + dataverseRequestSvc.getDataverseRequest(), + this.ingestSvc, + this.datasetSvc, + this.dataFileSvc, + this.permissionSvc, + this.commandEngine, + this.systemConfig + ); + + Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser); + + if (Response.Status.OK.equals(addFilesResponse.getStatusInfo())) { // if(!taskSkippedFiles) if (countError == 0) { userNotificationService.sendNotification((AuthenticatedUser) authUser, @@ -830,10 +854,10 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), countSuccess + " files added out of " + countAll, true); } - globusLogger.info("Successfully completed api/datasets/:persistentId/addFiles call "); + globusLogger.info("Successfully completed addFiles call "); } else { globusLogger.log(Level.SEVERE, - "******* Error while executing api/datasets/:persistentId/add call ", command); + "******* Error while executing addFiles ", newjsonData); } } @@ -848,6 +872,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S e.printStackTrace(); globusLogger.info("Exception from globusUpload call " + e.getMessage()); datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); + // } } if (ruleId != null) { @@ -1261,7 +1286,7 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, Long fileId = Long.parseLong(idAsString); // If we need to create a GuestBookResponse record, we have to // look up the DataFile object for this file: - df = dataFileService.findCheapAndEasy(fileId); + df = dataFileSvc.findCheapAndEasy(fileId); selectedFiles.add(df); if (!doNotSaveGuestbookResponse) { guestbookResponse.setDataFile(df); From 04951609c20f954573eb1d50eebe8ef08d464ed8 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 6 Aug 2024 16:40:59 -0400 Subject: [PATCH 149/360] no need to try to calculate checksums if this globus storage isn't dataverse-accessible. #10623 --- .../dataverse/globus/GlobusServiceBean.java | 41 +++++++++++-------- .../timer/DataverseTimerServiceBean.java | 4 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 02bc4667ea6..d6b56b51fa5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -1160,25 +1160,30 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger) // ToDo: what if the file does not exist in s3 // ToDo: what if checksum calculation failed - do { - try { - StorageIO dataFileStorageIO = DataAccess.getDirectStorageIO(fullPath); - in = dataFileStorageIO.getInputStream(); - checksumVal = FileUtil.calculateChecksum(in, DataFile.ChecksumType.MD5); - count = 3; - } catch (IOException ioex) { - count = 3; - logger.fine(ioex.getMessage()); - globusLogger.info( - "DataFile (fullPath " + fullPath + ") does not appear to be accessible within Dataverse: "); - } catch (Exception ex) { - count = count + 1; - ex.printStackTrace(); - logger.info(ex.getMessage()); - Thread.sleep(5000); - } + String storageDriverId = DataAccess.getDriverIdAndStorageLocation(fullPath)[0]; - } while (count < 3); + if (StorageIO.isDataverseAccessible(storageDriverId)) { + do { + try { + StorageIO dataFileStorageIO = DataAccess.getDirectStorageIO(fullPath); + in = dataFileStorageIO.getInputStream(); + checksumVal = FileUtil.calculateChecksum(in, DataFile.ChecksumType.MD5); + count = 3; + } catch (IOException ioex) { + count = 3; + logger.fine(ioex.getMessage()); + globusLogger.info( + "DataFile (fullPath " + fullPath + ") does not appear to be accessible within Dataverse: "); + } catch (Exception ex) { + count = count + 1; + ex.printStackTrace(); + logger.info(ex.getMessage()); + Thread.sleep(5000); + } + + + } while (count < 3); + } if (checksumVal.length() == 0) { checksumVal = "Not available in Dataverse"; diff --git a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java index 6eb3a8df0bc..a783b211b36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java @@ -120,13 +120,13 @@ public void handleTimeout(jakarta.ejb.Timer timer) { } try { - logger.log(Level.INFO,"Handling timeout on " + InetAddress.getLocalHost().getCanonicalHostName()); + logger.log(Level.FINE,"Handling timeout on " + InetAddress.getLocalHost().getCanonicalHostName()); } catch (UnknownHostException ex) { Logger.getLogger(DataverseTimerServiceBean.class.getName()).log(Level.SEVERE, null, ex); } if (timer.getInfo() instanceof MotherTimerInfo) { - logger.info("Behold! I am the Master Timer, king of all timers! I'm here to create all the lesser timers!"); + logger.fine("Behold! I am the Master Timer, king of all timers! I'm here to create all the lesser timers!"); removeHarvestTimers(); for (HarvestingClient client : harvestingClientService.getAllHarvestingClients()) { createHarvestTimer(client); From ba661387bba24e770ae5c0f9cade3b64db7614f3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Aug 2024 10:43:45 -0400 Subject: [PATCH 150/360] more globus mods (work in progress). #10623 --- .../harvard/iq/dataverse/api/Datasets.java | 7 +- .../dataverse/globus/GlobusServiceBean.java | 485 +++++++++++------- .../{GlobusTask.java => GlobusTaskState.java} | 6 +- .../iq/dataverse/globus/GlobusUtil.java | 33 ++ .../dataverse/ingest/IngestServiceBean.java | 5 +- .../iq/dataverse/settings/FeatureFlags.java | 5 + .../iq/dataverse/util/SystemConfig.java | 9 + 7 files changed, 363 insertions(+), 187 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/globus/{GlobusTask.java => GlobusTaskState.java} (93%) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4b919c5ed82..b93257bc0c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -4009,6 +4009,7 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, logger.info(" ==== (api addGlobusFilesToDataset) jsonData ====== " + jsonData); if (!systemConfig.isHTTPUpload()) { + // @todo why isHTTPUpload()? - shouldn't it be checking isGlobusUpload() here? return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); } @@ -4075,7 +4076,11 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, String requestUrl = SystemConfig.getDataverseSiteUrlStatic(); // Async Call - globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser); + try { + globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser); + } catch (IllegalArgumentException ex) { + return badRequest("Invalid parameters: "+ex.getMessage()); + } return ok("Async call to Globus Upload started "); diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index d6b56b51fa5..eb1eb47611a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -22,7 +22,6 @@ import jakarta.json.JsonString; import jakarta.json.JsonValue.ValueType; import jakarta.json.stream.JsonParsingException; -import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.HttpMethod; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; @@ -33,7 +32,6 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLEncoder; import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.time.Duration; @@ -62,10 +60,10 @@ import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; @@ -73,6 +71,8 @@ import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; import jakarta.ws.rs.core.Response; @Stateless @@ -105,6 +105,8 @@ public class GlobusServiceBean implements java.io.Serializable { IngestServiceBean ingestSvc; @EJB SystemConfig systemConfig; + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; private static final Logger logger = Logger.getLogger(GlobusServiceBean.class.getCanonicalName()); private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); @@ -391,19 +393,33 @@ private void monitorTemporaryPermissions(String ruleId, long datasetId) { * @return * @throws MalformedURLException */ - public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger) throws MalformedURLException { + public GlobusTaskState getTask(String accessToken, String taskId, Logger globusLogger) { - URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId); + Logger myLogger = globusLogger != null ? globusLogger : logger; + + URL url; + try { + url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId); + } catch (MalformedURLException mue) { + myLogger.warning("Malformed URL exception when trying to contact Globus. Globus API url: " + + "https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + + taskId); + return null; + } MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "GET", null); - GlobusTask task = null; + GlobusTaskState task = null; if (result.status == 200) { - task = parseJson(result.jsonResponse, GlobusTask.class, false); + task = parseJson(result.jsonResponse, GlobusTaskState.class, false); } if (result.status != 200) { - globusLogger.warning("Cannot find information for the task " + taskId + " : Reason : " + // @todo It should probably retry it 2-3 times before giving up; + // similarly, it should probably differentiate between a "no such task" + // response and something intermittent like a server/network error or + // an expired token... i.e. something that's recoverable (?) + myLogger.warning("Cannot find information for the task " + taskId + " : Reason : " + result.jsonResponse.toString()); } @@ -646,11 +662,17 @@ private String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List< @Asynchronous @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, String httpRequestUrl, - AuthenticatedUser authUser) throws ExecutionException, InterruptedException, MalformedURLException { + AuthenticatedUser authUser) throws IllegalArgumentException, ExecutionException, InterruptedException, MalformedURLException { - Integer countAll = 0; - Integer countSuccess = 0; - Integer countError = 0; + // Before we do anything else, let's do some basic validation of what + // we've been passed: + + JsonArray filesJsonArray = jsonData.getJsonArray("files"); + + if (filesJsonArray == null || filesJsonArray.size() < 1) { + throw new IllegalArgumentException("No valid json entries supplied for the files being uploaded"); + } + String logTimestamp = logFormatter.format(new Date()); Logger globusLogger = Logger.getLogger( "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimestamp); @@ -674,11 +696,13 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S } logger.fine("json: " + JsonUtil.prettyPrint(jsonData)); + + globusLogger.info("Globus upload initiated"); String taskIdentifier = jsonData.getString("taskIdentifier"); GlobusEndpoint endpoint = getGlobusEndpoint(dataset); - GlobusTask task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); + GlobusTaskState task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw"); logger.fine("Found rule: " + ruleId); if (ruleId != null) { @@ -688,15 +712,43 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S rulesCache.invalidate(ruleId); } } - + // Wait before first check Thread.sleep(5000); + + if (FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) { + + // Save the task information in the database so that the Globus monitoring + // service can continue checking on its progress. + + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token, new Timestamp(new Date().getTime())); + em.persist(taskInProgress); + + // Save the metadata entries that define the files that are being uploaded + // in the database. These entries will be used once/if the uploads + // completes successfully to add the files to the dataset. + + for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { + ExternalFileUploadInProgress fileUploadRecord = new ExternalFileUploadInProgress(taskIdentifier, fileJsonObject.toString()); + + em.persist(fileUploadRecord); + } + + return; + } + + + // the old implementation that relies on looping continuosly, + // sleeping-then-checking the task status repeatedly: + // globus task status check + // (the method below performs continuous looped checks of the remote + // Globus API, monitoring it for as long as it takes for the task to + // finish one way or another!) task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); + // @todo null check, or make sure it's never null String taskStatus = getTaskStatus(task); - globusLogger.info("Starting a globusUpload "); - if (ruleId != null) { // Transfer is complete, so delete rule deletePermission(ruleId, dataset, globusLogger); @@ -739,138 +791,11 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S } else { try { - // - - List inputList = new ArrayList(); - JsonArray filesJsonArray = jsonData.getJsonArray("files"); - - if (filesJsonArray != null) { - String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/" - + dataset.getIdentifierForFileStorage(); - - for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { - - // storageIdentifier s3://gcs5-bucket1:1781cfeb8a7-748c270a227c from - // externalTool - String storageIdentifier = fileJsonObject.getString("storageIdentifier"); - String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); - String storeId = parts[0]; - // If this is an S3 store, we need to split out the bucket name - String[] bits = parts[1].split(":"); - String bucketName = ""; - if (bits.length > 1) { - bucketName = bits[0]; - } - String fileId = bits[bits.length - 1]; - - // fullpath s3://gcs5-bucket1/10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873 - // or globus:///10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873 - String fullPath = storeId + "://" + bucketName + "/" + datasetIdentifier + "/" + fileId; - String fileName = fileJsonObject.getString("fileName"); - - inputList.add(fileId + "IDsplit" + fullPath + "IDsplit" + fileName); - } - - // calculateMissingMetadataFields: checksum, mimetype - JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, globusLogger); - JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files"); - logger.fine("Size: " + newfilesJsonArray.size()); - logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0))); - JsonArrayBuilder addFilesJsonData = Json.createArrayBuilder(); - - for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { - - countAll++; - String storageIdentifier = fileJsonObject.getString("storageIdentifier"); - String fileName = fileJsonObject.getString("fileName"); - String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); - // If this is an S3 store, we need to split out the bucket name - String[] bits = parts[1].split(":"); - if (bits.length > 1) { - } - String fileId = bits[bits.length - 1]; - - List newfileJsonObject = IntStream.range(0, newfilesJsonArray.size()) - .mapToObj(index -> ((JsonObject) newfilesJsonArray.get(index)).getJsonObject(fileId)) - .filter(Objects::nonNull).collect(Collectors.toList()); - if (newfileJsonObject != null) { - logger.fine("List Size: " + newfileJsonObject.size()); - // if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) { - JsonPatch path = Json.createPatchBuilder() - .add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build(); - fileJsonObject = path.apply(fileJsonObject); - path = Json.createPatchBuilder() - .add("/mimeType", newfileJsonObject.get(0).getString("mime")).build(); - fileJsonObject = path.apply(fileJsonObject); - addFilesJsonData.add(fileJsonObject); - countSuccess++; - // } else { - // globusLogger.info(fileName - // + " will be skipped from adding to dataset by second API due to missing - // values "); - // countError++; - // } - } else { - globusLogger.info(fileName - + " will be skipped from adding to dataset by second API due to missing values "); - countError++; - } - } - - String newjsonData = addFilesJsonData.build().toString(); - - globusLogger.info("Successfully generated new JsonData for addFiles call"); - - /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " - + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" - + datasetIdentifier + " -F jsonData='" + newjsonData + "'"; - System.out.println("*******====command ==== " + command);*/ - - // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of - // calling API - - // a quick experimental AddReplaceFileHelper implementation: - AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( - dataverseRequestSvc.getDataverseRequest(), - this.ingestSvc, - this.datasetSvc, - this.dataFileSvc, - this.permissionSvc, - this.commandEngine, - this.systemConfig - ); - - Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser); - - if (Response.Status.OK.equals(addFilesResponse.getStatusInfo())) { - // if(!taskSkippedFiles) - if (countError == 0) { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADCOMPLETED, - dataset.getId(), countSuccess + " files added out of " + countAll, true); - } else { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), - countSuccess + " files added out of " + countAll, true); - } - globusLogger.info("Successfully completed addFiles call "); - } else { - globusLogger.log(Level.SEVERE, - "******* Error while executing addFiles ", newjsonData); - } - - } - - globusLogger.info("Files processed: " + countAll.toString()); - globusLogger.info("Files added successfully: " + countSuccess.toString()); - globusLogger.info("Files failures: " + countError.toString()); - globusLogger.info("Finished upload via Globus job."); - + processUploadedFiles(filesJsonArray, dataset, authUser, globusLogger); } catch (Exception e) { - logger.info("Exception from globusUpload call "); + logger.info("Exception from processUploadedFiles call "); e.printStackTrace(); - globusLogger.info("Exception from globusUpload call " + e.getMessage()); + globusLogger.info("Exception from processUploadedFiles call " + e.getMessage()); datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); // } @@ -883,7 +808,164 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S fileHandler.close(); } } + /** + * The code in this method is copy-and-pasted from the previous Borealis + * implemenation + * @todo see if it can be refactored and simplified a bit, the json manipulation + * specifically (?) + * @param filesJsonArray JsonArray containing files metadata entries as passed to /addGlobusFiles + * @param dataset the dataset + * @param authUser the user that should be be performing the addFiles call + * finalizing adding the files to the Dataset. Note that this + * user will need to be obtained from the saved api token, when this + * method is called via the TaskMonitoringService + * @param myLogger the Logger; if null, the main logger of the service bean will be used + * @throws IOException, InterruptedException, ExecutionException @todo may need to throw more exceptions (?) + */ + private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, AuthenticatedUser authUser, Logger myLogger) throws IOException, InterruptedException, ExecutionException { + myLogger = myLogger != null ? myLogger : logger; + + Integer countAll = 0; + Integer countSuccess = 0; + Integer countError = 0; + + List inputList = new ArrayList(); + + String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/" + + dataset.getIdentifierForFileStorage(); + + for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { + + // storageIdentifier s3://gcs5-bucket1:1781cfeb8a7-748c270a227c from + // externalTool + String storageIdentifier = fileJsonObject.getString("storageIdentifier"); + String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); + String storeId = parts[0]; + // If this is an S3 store, we need to split out the bucket name + String[] bits = parts[1].split(":"); + String bucketName = ""; + if (bits.length > 1) { + bucketName = bits[0]; + } + String fileId = bits[bits.length - 1]; + + // fullpath s3://gcs5-bucket1/10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873 + // or globus:///10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873 + String fullPath = storeId + "://" + bucketName + "/" + datasetIdentifier + "/" + fileId; + String fileName = fileJsonObject.getString("fileName"); + + inputList.add(fileId + "IDsplit" + fullPath + "IDsplit" + fileName); + } + + // calculateMissingMetadataFields: checksum, mimetype + JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, myLogger); + JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files"); + logger.fine("Size: " + newfilesJsonArray.size()); + logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0))); + JsonArrayBuilder addFilesJsonData = Json.createArrayBuilder(); + + for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) { + + countAll++; + String storageIdentifier = fileJsonObject.getString("storageIdentifier"); + String fileName = fileJsonObject.getString("fileName"); + String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier); + // If this is an S3 store, we need to split out the bucket name + String[] bits = parts[1].split(":"); + if (bits.length > 1) { + } + String fileId = bits[bits.length - 1]; + + List newfileJsonObject = IntStream.range(0, newfilesJsonArray.size()) + .mapToObj(index -> ((JsonObject) newfilesJsonArray.get(index)).getJsonObject(fileId)) + .filter(Objects::nonNull).collect(Collectors.toList()); + if (newfileJsonObject != null) { + logger.fine("List Size: " + newfileJsonObject.size()); + // if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) { + JsonPatch path = Json.createPatchBuilder() + .add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build(); + fileJsonObject = path.apply(fileJsonObject); + path = Json.createPatchBuilder() + .add("/mimeType", newfileJsonObject.get(0).getString("mime")).build(); + fileJsonObject = path.apply(fileJsonObject); + addFilesJsonData.add(fileJsonObject); + countSuccess++; + // } else { + // globusLogger.info(fileName + // + " will be skipped from adding to dataset by second API due to missing + // values "); + // countError++; + // } + } else { + myLogger.info(fileName + + " will be skipped from adding to dataset in the final AddReplaceFileHelper.addFiles() call. "); + countError++; + } + } + + String newjsonData = addFilesJsonData.build().toString(); + + myLogger.info("Successfully generated new JsonData for addFiles call"); + + /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " + + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" + + datasetIdentifier + " -F jsonData='" + newjsonData + "'"; + System.out.println("*******====command ==== " + command);*/ + // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of + // calling API + // a quick experimental AddReplaceFileHelper implementation: + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( + dataverseRequestSvc.getDataverseRequest(), + this.ingestSvc, + this.datasetSvc, + this.dataFileSvc, + this.permissionSvc, + this.commandEngine, + this.systemConfig + ); + + Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser); + + if (Response.Status.OK.equals(addFilesResponse.getStatusInfo())) { + // if(!taskSkippedFiles) + if (countError == 0) { + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADCOMPLETED, + dataset.getId(), countSuccess + " files added out of " + countAll, true); + } else { + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), + countSuccess + " files added out of " + countAll, true); + } + myLogger.info("Successfully completed addFiles call "); + } else { + myLogger.log(Level.SEVERE, + "******* Error while executing addFiles ", newjsonData); + } + myLogger.info("Files processed: " + countAll); + myLogger.info("Files added successfully: " + countSuccess); + myLogger.info("Files failures: " + countError); + myLogger.info("Finished upload via Globus job."); + + } + + /** + * I don't think this method is needed at all. (I suspect that it's a remnant + * from the times when *multiple* individual /add calls needed to be performed + * for each file being added. So this was part of a framework that attempted + * to run this calls in parallel, potentially speeding things up (similarly to + * how the checksums are being calculated in parallel for multiple files). + * As of now, this method doesn't do anything "asynchronous" - there is one + * /addFiles call, and the method below will wait for it to complete, via the + * CompletableFuture.get(). (L.A.) + * @param curlCommand + * @param globusLogger + * @return + * @throws ExecutionException + * @throws InterruptedException + */ public String addFilesAsync(String curlCommand, Logger globusLogger) throws ExecutionException, InterruptedException { CompletableFuture addFilesFuture = CompletableFuture.supplyAsync(() -> { @@ -983,7 +1065,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro // If the rules_cache times out, the permission will be deleted. Presumably that // doesn't affect a // globus task status check - GlobusTask task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); + GlobusTaskState task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); String ruleId = getRuleId(endpoint, task.getOwner_id(), "r"); if (ruleId != null) { logger.fine("Found rule: " + ruleId); @@ -999,6 +1081,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro logger.warning("ruleId not found for taskId: " + taskIdentifier); } task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); + // @todo null check String taskStatus = getTaskStatus(task); // Transfer is done (success or failure) so delete the rule @@ -1033,61 +1116,38 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro Executor executor = Executors.newFixedThreadPool(10); - private GlobusTask globusStatusCheck(GlobusEndpoint endpoint, String taskId, Logger globusLogger) + private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId, Logger globusLogger) throws MalformedURLException { - boolean taskCompletion = false; - String status = ""; - GlobusTask task = null; + boolean taskCompleted = false; + GlobusTaskState task = null; int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 50); do { try { globusLogger.info("checking globus transfer task " + taskId); Thread.sleep(pollingInterval * 1000); + // Call the (centralized) Globus API to check on the task state/status: task = getTask(endpoint.getClientToken(), taskId, globusLogger); - if (task != null) { - status = task.getStatus(); - if (status != null) { - // The task is in progress. - if (status.equalsIgnoreCase("ACTIVE")) { - if (task.getNice_status().equalsIgnoreCase("ok") - || task.getNice_status().equalsIgnoreCase("queued")) { - taskCompletion = false; - } else { - taskCompletion = true; - // status = "FAILED" + "#" + task.getNice_status() + "#" + - // task.getNice_status_short_description(); - } - } else { - // The task is either succeeded, failed or inactive. - taskCompletion = true; - // status = status + "#" + task.getNice_status() + "#" + - // task.getNice_status_short_description(); - } - } else { - // status = "FAILED"; - taskCompletion = true; - } - } else { - // status = "FAILED"; - taskCompletion = true; - } + taskCompleted = GlobusUtil.isTaskCompleted(task); } catch (Exception ex) { ex.printStackTrace(); } - } while (!taskCompletion); + } while (!taskCompleted); globusLogger.info("globus transfer task completed successfully"); return task; } - - private String getTaskStatus(GlobusTask task) { + + private String getTaskStatus(GlobusTaskState task) { String status = null; if (task != null) { status = task.getStatus(); if (status != null) { // The task is in progress but is not ok or queued + // (L.A.) I think the assumption here is that this method is called + // exclusively on tasks that have already completed. So that's why + // it is safe to assume that "ACTIVE" means "FAILED". if (status.equalsIgnoreCase("ACTIVE")) { status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); } else { @@ -1158,7 +1218,16 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger) String fileName = id.split("IDsplit")[2]; // ToDo: what if the file does not exist in s3 + // (L.A.) - good question. maybe it should call .open and .exists() here? + // otherwise, there doesn't seem to be any diagnostics as to which + // files uploaded successfully and which failed (?) + // ... however, any partially successful upload cases should be + // properly handled later, during the .addFiles() call - only + // the files that actually exists in storage remotely will be + // added to the dataset permanently then. // ToDo: what if checksum calculation failed + // (L.A.) - this appears to have been addressed - by using "Not available in Dataverse" + // in place of a checksum. String storageDriverId = DataAccess.getDriverIdAndStorageLocation(fullPath)[0]; @@ -1180,8 +1249,6 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger) logger.info(ex.getMessage()); Thread.sleep(5000); } - - } while (count < 3); } @@ -1311,5 +1378,57 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, } } } + + public List findAllOngoingTasks() { + return em.createQuery("select object(o) from GlobusTaskInProgress as o order by o.startTime", GlobusTaskInProgress.class).getResultList(); + } + + public void deleteTask(GlobusTaskInProgress task) { + GlobusTaskInProgress mergedTask = em.merge(task); + em.remove(mergedTask); + } + + public List findExternalUploadsByTaskId(String taskId) { + return em.createNamedQuery("ExternalFileUploadInProgress.findByTaskId").setParameter("taskId", taskId).getResultList(); + } + + // @todo this may or may not need to be async (?) + public void addFilesOnSuccess(GlobusTaskInProgress globusTask) { + List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); + + if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { + // @todo log error message; do nothing + return; + } + Dataset dataset = globusTask.getDataset(); + AuthenticatedUser authUser = authSvc.lookupUser(globusTask.getApiToken()); + if (authUser == null) { + // @todo log error message; do nothing + return; + } + + JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); + + for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { + String jsonInfoString = pendingFile.getFileInfo(); + JsonObject fileObject = JsonUtil.getJsonObject(jsonInfoString); + filesJsonArrayBuilder.add(fileObject); + } + + JsonArray filesJsonArray = filesJsonArrayBuilder.build(); + + if (filesJsonArray == null || filesJsonArray.size() < 1) { + // @todo log error message; do nothing + return; + } + + try { + processUploadedFiles(filesJsonArray, dataset, authUser, null); + } catch (Exception ex) { + // @todo log error message; make sure the error notification to the + // has been sent (may or may not have already been sent inside the + // method above). + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTask.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java similarity index 93% rename from src/main/java/edu/harvard/iq/dataverse/globus/GlobusTask.java rename to src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java index c2b01779f4a..b5db20d46c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTask.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java @@ -1,6 +1,10 @@ package edu.harvard.iq.dataverse.globus; -public class GlobusTask { +/** + * This class is used to store the state of an ongoing Globus task (transfer) + * as reported by the Globus task API. + */ +public class GlobusTaskState { private String DATA_TYPE; private String type; diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java index 92cf8ac7704..67594ad1a5e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java @@ -30,4 +30,37 @@ public static JsonObject getFilesMap(List dataFiles, Dataset d) { } return filesBuilder.build(); } + + public static boolean isTaskCompleted(GlobusTaskState task) { + if (task != null) { + String status = task.getStatus(); + if (status != null) { + if (status.equalsIgnoreCase("ACTIVE")) { + if (task.getNice_status().equalsIgnoreCase("ok") + || task.getNice_status().equalsIgnoreCase("queued")) { + return false; + } + } + } + } + return true; + } + + public static boolean isTaskSucceeded(GlobusTaskState task) { + String status = null; + if (task != null) { + status = task.getStatus(); + if (status != null) { + status = status.toUpperCase(); + if (status.equals("ACTIVE") || status.startsWith("FAILED") || status.startsWith("INACTIVE")) { + // There are cases where a failed task may still be showing + // as "ACTIVE". But it is definitely safe to assume that it + // has not completed *successfully*. + return false; + } + return true; + } + } + return false; + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 9bacafd173f..3f76a319902 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -345,12 +345,13 @@ public List saveAndAddFilesToDataset(DatasetVersion version, StorageIO dataAccess = DataAccess.getStorageIO(dataFile); //Populate metadata dataAccess.open(DataAccessOption.READ_ACCESS); - + // (this will make a remote call to check if the file exists + // and obtain its size) confirmedFileSize = dataAccess.getSize(); // For directly-uploaded files, we will perform the file size // limit and quota checks here. Perform them *again*, in - // some cases: a directly uploaded files have already been + // some cases: files directly uploaded via the UI have already been // checked (for the sake of being able to reject the upload // before the user clicks "save"). But in case of direct // uploads via API, these checks haven't been performed yet, diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 021977ff8c6..746e6e3b75d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -91,6 +91,11 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ DISABLE_RETURN_TO_AUTHOR_REASON("disable-return-to-author-reason"), + /** + * TEMPORARY feature flag for the new Globus upload framework (will only be + * used for testing). + */ + GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK("globus-use-experimental-async-framework"), ; final String flag; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index f9801419e47..7417a5db4d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -82,6 +82,7 @@ public class SystemConfig { private String buildNumber = null; private static final String JVM_TIMER_SERVER_OPTION = "dataverse.timerServer"; + private static final String JVM_GLOBUS_TASK_MONITORING_OPTION = "dataverse.globus.taskMonitoringServer"; private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L; private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE = 3000000L; // 3 MB @@ -545,6 +546,14 @@ public boolean isTimerServer() { } return false; } + + public boolean isGlobusTaskMonitoringServer() { + String optionValue = System.getProperty(JVM_GLOBUS_TASK_MONITORING_OPTION); + if ("true".equalsIgnoreCase(optionValue)) { + return true; + } + return false; + } public String getFooterCopyrightAndYear() { return BundleUtil.getStringFromBundle("footer.copyright", Arrays.asList(Year.now().getValue() + "")); From dac53023309a17b6761388203a6040ee7199f382 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Aug 2024 13:14:00 -0400 Subject: [PATCH 151/360] new class files that weren't included in the last commit #10623 --- .../ExternalFileUploadInProgress.java | 131 ++++++++++++ .../globus/GlobusTaskInProgress.java | 188 ++++++++++++++++++ .../globus/TaskMonitoringServiceBean.java | 78 ++++++++ 3 files changed, 397 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java new file mode 100644 index 00000000000..dadde64608c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java @@ -0,0 +1,131 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.harvard.iq.dataverse; + +import jakarta.persistence.Column; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.NamedQueries; +import jakarta.persistence.NamedQuery; +import java.io.Serializable; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; + +/** + * + * @author landreev + * + * The name of the class is provisional. I'm open to better-sounding alternatives, + * if anyone can think of any. + * But I wanted to avoid having the word "Globus" in the entity name. I'm adding + * it specifically for the Globus use case. But I'm guessing there's a chance + * this setup may come in handy for other types of datafile uploads that happen + * externally. (?) + */ +@NamedQueries({ + @NamedQuery( name="ExternalFileUploadInProgress.deleteByTaskId", + query="DELETE FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId"), + @NamedQuery(name = "ExternalFileUploadInProgress.findByTaskId", + query = "SELECT f FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId")}) +@Entity +public class ExternalFileUploadInProgress implements Serializable { + + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + /** + * Rather than saving various individual fields defining the datafile, + * which would essentially replicate the DataFile table, we are simply + * storing the full json record as passed to the API here. + */ + @Column(columnDefinition = "TEXT", nullable = false) + private String fileInfo; + + /** + * This is Globus-specific task id associated with the upload in progress + */ + @Column(nullable = false) + private String taskId; + + /** + * The Dataset to which the files are being added. + * (@todo may not be necessary? - since the corresponding task is tied to a specific + * dataset already?) + */ + /*@ManyToOne + private Dataset dataset;*/ + + /*public ExternalFileUploadInProgress(String taskId, Dataset dataset, String fileInfo) { + this.taskId = taskId; + this.fileInfo = fileInfo; + this.dataset = dataset; + }*/ + + public ExternalFileUploadInProgress(String taskId, String fileInfo) { + this.taskId = taskId; + this.fileInfo = fileInfo; + } + + public String getFileInfo() { + return fileInfo; + } + + public void setFileInfo(String fileInfo) { + this.fileInfo = fileInfo; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + /*public Dataset getDataset() { + return dataset; + } + + public void setDataset(Dataset dataset) { + this.dataset = dataset; + }*/ + + @Override + public int hashCode() { + int hash = 0; + hash += (id != null ? id.hashCode() : 0); + return hash; + } + + @Override + public boolean equals(Object object) { + // TODO: Warning - this method won't work in the case the id fields are not set + if (!(object instanceof ExternalFileUploadInProgress)) { + return false; + } + ExternalFileUploadInProgress other = (ExternalFileUploadInProgress) object; + if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { + return false; + } + return true; + } + + @Override + public String toString() { + return "edu.harvard.iq.dataverse.ExternalFileUploadInProgress[ id=" + id + " ]"; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java new file mode 100644 index 00000000000..7b12ec0a3ad --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -0,0 +1,188 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.harvard.iq.dataverse.globus; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import jakarta.persistence.Column; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.ManyToOne; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Arrays; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; + +/** + * + * @author landreev + */ +@Entity +public class GlobusTaskInProgress implements Serializable { + + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + /** + * Globus-side identifier of the task in progress, upload or download + */ + @Column(nullable = false) + private String taskId; + + GlobusTaskInProgress(String taskIdentifier, TaskType taskType, Dataset dataset, String clientToken, ApiToken token, Timestamp timestamp) { + throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody + } + + /** + * I was considering giving this enum type a more specific name "TransferType" + * - but maybe there will be another use case where we need to keep track of + * Globus tasks that are not data transfers (?) + */ + public enum TaskType { + + UPLOAD("UPLOAD"), + DOWNLOAD("DOWNLOAD"); + + private final String text; + + private TaskType(final String text) { + this.text = text; + } + + public static TaskType fromString(String text) { + if (text != null) { + for (TaskType taskType : TaskType.values()) { + if (text.equals(taskType.text)) { + return taskType; + } + } + } + throw new IllegalArgumentException("TaskType must be one of these values: " + Arrays.asList(TaskType.values()) + "."); + } + + @Override + public String toString() { + return text; + } + } + + @Column(nullable = false) + @Enumerated(EnumType.STRING) + private TaskType taskType; + + /** + * Globus API token that should be used to monitor the status of the task + */ + @Column(nullable = false) + private String globusToken; + + /** + * This is the Dataverse API token of the user who initiated the Globus task + */ + private String apiToken; + + @ManyToOne + private Dataset dataset; + + @Column( nullable = false ) + private Timestamp startTime; + + + public GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String clientToken, String apiToken, Timestamp startTime) { + this.taskId = taskId; + this.taskType = taskType; + this.globusToken = clientToken; + this.apiToken = apiToken; + this.dataset = dataset; + this.startTime = startTime; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public TaskType getTaskType() { + return taskType; + } + + public void setTaskType(TaskType taskType) { + this.taskType = taskType; + } + + public String getGlobusToken() { + return globusToken; + } + + public void setGlobusToken(String clientToken) { + this.globusToken = clientToken; + } + + public String getApiToken() { + return apiToken; + } + + public void setApiToken(String apiToken) { + this.apiToken = apiToken; + } + + public Dataset getDataset() { + return dataset; + } + + public void setDataset(Dataset dataset) { + this.dataset = dataset; + } + + public Timestamp getStartTime() { + return startTime; + } + + public void setStartTime(Timestamp startTime) { + this.startTime = startTime; + } + + @Override + public int hashCode() { + int hash = 0; + hash += (id != null ? id.hashCode() : 0); + return hash; + } + + @Override + public boolean equals(Object object) { + // TODO: Warning - this method won't work in the case the id fields are not set + if (!(object instanceof GlobusTaskInProgress)) { + return false; + } + GlobusTaskInProgress other = (GlobusTaskInProgress) object; + if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { + return false; + } + return true; + } + + @Override + public String toString() { + return "edu.harvard.iq.dataverse.globus.GlobusTaskInProgress[ id=" + id + " ]"; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java new file mode 100644 index 00000000000..da31ded90db --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -0,0 +1,78 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package edu.harvard.iq.dataverse.globus; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; +import jakarta.enterprise.concurrent.ManagedScheduledExecutorService; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * + * This Singleton monitors ongoing Globus tasks by checking with the centralized + * Globus API on the status of all the registered ongoing tasks. + * When a successful completion of a task is detected, the service triggers + * the execution of the associated tasks (for example, finalizing adding datafiles + * to the dataset on completion of a remote Globus upload). When a task fails or + * terminates abnormally, a message is logged and the task record is deleted + * from the database. + * + * @author landreev + */ +@Singleton +@Startup +public class TaskMonitoringServiceBean { + @Resource + ManagedScheduledExecutorService scheduler; + + @EJB + SystemConfig systemConfig; + @EJB + SettingsServiceBean settingsSvc; + @EJB + GlobusServiceBean globusService; + + @PostConstruct + public void init() { + if (systemConfig.isGlobusTaskMonitoringServer()) { + int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( + settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 60); + this.scheduler.scheduleAtFixedRate(this::checkOngoingTasks, + 0, pollingInterval, + TimeUnit.SECONDS); + } + } + + /** + * This method will be executed on a timer-like schedule, continuously + * monitoring all the ongoing external Globus tasks (transfers). + * @todo make sure the executions do not overlap/stack up + */ + public void checkOngoingTasks() { + List tasks = globusService.findAllOngoingTasks(); + + tasks.forEach(t -> { + GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), null); + if (GlobusUtil.isTaskCompleted(retrieved)) { + if (GlobusUtil.isTaskSucceeded(retrieved)) { + // Do our thing, finalize adding the files to the dataset + globusService.addFilesOnSuccess(t); + } + // Whether it finished successfully, or failed in the process, + // there's no need to keep monitoring this task, so we can + // delete it. + globusService.deleteTask(t); + // @todo double-check that the locks have been properly handled + } + }); + } + +} From 408034125998e5cd40dda2ff27374cab4bdacd21 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Aug 2024 15:21:45 -0400 Subject: [PATCH 152/360] fixing some bad changes that got committed earlier #10623 --- .../ExternalFileUploadInProgress.java | 41 ++++++------------ .../dataverse/globus/GlobusServiceBean.java | 2 +- .../globus/GlobusTaskInProgress.java | 42 ++++++++++--------- 3 files changed, 35 insertions(+), 50 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java index dadde64608c..ab6a1798307 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java @@ -5,14 +5,15 @@ package edu.harvard.iq.dataverse; import jakarta.persistence.Column; -import jakarta.persistence.ManyToOne; +import jakarta.persistence.Index; import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; +import jakarta.persistence.Table; import java.io.Serializable; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; /** * @@ -31,6 +32,7 @@ @NamedQuery(name = "ExternalFileUploadInProgress.findByTaskId", query = "SELECT f FROM ExternalFileUploadInProgress f WHERE f.taskId=:taskId")}) @Entity +@Table(indexes = {@Index(columnList="taskid")}) public class ExternalFileUploadInProgress implements Serializable { private static final long serialVersionUID = 1L; @@ -51,29 +53,18 @@ public void setId(Long id) { * which would essentially replicate the DataFile table, we are simply * storing the full json record as passed to the API here. */ - @Column(columnDefinition = "TEXT", nullable = false) + @Column(columnDefinition = "TEXT", nullable=false) private String fileInfo; /** * This is Globus-specific task id associated with the upload in progress */ - @Column(nullable = false) + @Column(nullable=false) private String taskId; - /** - * The Dataset to which the files are being added. - * (@todo may not be necessary? - since the corresponding task is tied to a specific - * dataset already?) - */ - /*@ManyToOne - private Dataset dataset;*/ - - /*public ExternalFileUploadInProgress(String taskId, Dataset dataset, String fileInfo) { - this.taskId = taskId; - this.fileInfo = fileInfo; - this.dataset = dataset; - }*/ - + public ExternalFileUploadInProgress() { + } + public ExternalFileUploadInProgress(String taskId, String fileInfo) { this.taskId = taskId; this.fileInfo = fileInfo; @@ -95,14 +86,6 @@ public void setTaskId(String taskId) { this.taskId = taskId; } - /*public Dataset getDataset() { - return dataset; - } - - public void setDataset(Dataset dataset) { - this.dataset = dataset; - }*/ - @Override public int hashCode() { int hash = 0; diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index eb1eb47611a..6b78925beb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -721,7 +721,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // Save the task information in the database so that the Globus monitoring // service can continue checking on its progress. - GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token, new Timestamp(new Date().getTime())); + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), new Timestamp(new Date().getTime())); em.persist(taskInProgress); // Save the metadata entries that define the files that are being uploaded diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java index 7b12ec0a3ad..210f08710dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -5,24 +5,26 @@ package edu.harvard.iq.dataverse.globus; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; import jakarta.persistence.Column; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import jakarta.persistence.UniqueConstraint; import java.io.Serializable; import java.sql.Timestamp; import java.util.Arrays; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; /** * * @author landreev */ @Entity +@Table(uniqueConstraints = {@UniqueConstraint(columnNames = "taskid")}) public class GlobusTaskInProgress implements Serializable { private static final long serialVersionUID = 1L; @@ -33,13 +35,9 @@ public class GlobusTaskInProgress implements Serializable { /** * Globus-side identifier of the task in progress, upload or download */ - @Column(nullable = false) + @Column(nullable=false, unique = true) private String taskId; - GlobusTaskInProgress(String taskIdentifier, TaskType taskType, Dataset dataset, String clientToken, ApiToken token, Timestamp timestamp) { - throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody - } - /** * I was considering giving this enum type a more specific name "TransferType" * - but maybe there will be another use case where we need to keep track of @@ -73,14 +71,14 @@ public String toString() { } } - @Column(nullable = false) + @Column @Enumerated(EnumType.STRING) private TaskType taskType; /** * Globus API token that should be used to monitor the status of the task */ - @Column(nullable = false) + @Column private String globusToken; /** @@ -91,19 +89,23 @@ public String toString() { @ManyToOne private Dataset dataset; - @Column( nullable = false ) + @Column private Timestamp startTime; - - public GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String clientToken, String apiToken, Timestamp startTime) { - this.taskId = taskId; - this.taskType = taskType; - this.globusToken = clientToken; - this.apiToken = apiToken; + public GlobusTaskInProgress() { + } + + GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String globusToken, String apiToken, Timestamp startTime) { + this.taskId = taskId; + this.taskType = taskType; this.dataset = dataset; - this.startTime = startTime; + this.globusToken = globusToken; + this.apiToken = apiToken; + this.startTime = startTime; } + + public Long getId() { return id; } From e086a60aad4a758f3c6c1b1d2797985b7ade380a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 13 Aug 2024 09:25:35 -0400 Subject: [PATCH 153/360] cleanup #10623 --- .../harvard/iq/dataverse/globus/GlobusServiceBean.java | 9 ++++++++- .../iq/dataverse/globus/TaskMonitoringServiceBean.java | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 6b78925beb0..9ab3a2df567 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -966,6 +966,7 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut * @throws ExecutionException * @throws InterruptedException */ + /* public String addFilesAsync(String curlCommand, Logger globusLogger) throws ExecutionException, InterruptedException { CompletableFuture addFilesFuture = CompletableFuture.supplyAsync(() -> { @@ -1018,7 +1019,7 @@ private String addFiles(String curlCommand, Logger globusLogger) { } return status; - } + } */ @Asynchronous public void globusDownload(String jsonData, Dataset dataset, User authUser) throws MalformedURLException { @@ -1431,4 +1432,10 @@ public void addFilesOnSuccess(GlobusTaskInProgress globusTask) { // method above). } } + + public void deleteExternalUploadRecords(String taskId) { + em.createNamedQuery("ExternalFileUploadInProgress.deleteByTaskId") + .setParameter("taskId", taskId) + .executeUpdate(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index da31ded90db..bd274d44e38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -14,6 +14,7 @@ import jakarta.enterprise.concurrent.ManagedScheduledExecutorService; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; /** * @@ -30,6 +31,8 @@ @Singleton @Startup public class TaskMonitoringServiceBean { + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.globus.TaskMonitoringServiceBean"); + @Resource ManagedScheduledExecutorService scheduler; @@ -43,11 +46,14 @@ public class TaskMonitoringServiceBean { @PostConstruct public void init() { if (systemConfig.isGlobusTaskMonitoringServer()) { + logger.info("Starting Globus task monitoring service"); int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 60); this.scheduler.scheduleAtFixedRate(this::checkOngoingTasks, 0, pollingInterval, TimeUnit.SECONDS); + } else { + logger.info("Skipping Globus task monitor initialization"); } } @@ -57,6 +63,7 @@ public void init() { * @todo make sure the executions do not overlap/stack up */ public void checkOngoingTasks() { + logger.info("Performing a scheduled external Globus task check"); List tasks = globusService.findAllOngoingTasks(); tasks.forEach(t -> { @@ -69,6 +76,7 @@ public void checkOngoingTasks() { // Whether it finished successfully, or failed in the process, // there's no need to keep monitoring this task, so we can // delete it. + globusService.deleteExternalUploadRecords(t.getTaskId()); globusService.deleteTask(t); // @todo double-check that the locks have been properly handled } From 35ce7ef1892baa92b36e2350193eec4a63db6237 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 14 Aug 2024 14:37:38 -0400 Subject: [PATCH 154/360] more testing/debugging #10623 --- .../dataverse/globus/GlobusServiceBean.java | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 9ab3a2df567..3f6d0bf7c68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -60,6 +60,7 @@ import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -71,9 +72,12 @@ import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import jakarta.json.JsonReader; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; +import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.core.Response; +import org.apache.http.util.EntityUtils; @Stateless @Named("GlobusServiceBean") @@ -810,7 +814,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S } /** * The code in this method is copy-and-pasted from the previous Borealis - * implemenation + * implemenation. * @todo see if it can be refactored and simplified a bit, the json manipulation * specifically (?) * @param filesJsonArray JsonArray containing files metadata entries as passed to /addGlobusFiles @@ -907,6 +911,11 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut myLogger.info("Successfully generated new JsonData for addFiles call"); + myLogger.info("Files processed: " + countAll); + myLogger.info("Files added successfully: " + countSuccess); + myLogger.info("Files failures: " + countError); + myLogger.info("Finished upload via Globus job."); + /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" + datasetIdentifier + " -F jsonData='" + newjsonData + "'"; @@ -914,8 +923,15 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of // calling API // a quick experimental AddReplaceFileHelper implementation: + + // Passing null for the HttpServletRequest to make a new DataverseRequest. + // The parent method is executed asynchronously, so the real request + // that was associated with the original API call that triggered this upload + // cannot be obtained. + DataverseRequest dataverseRequest = new DataverseRequest(authUser, (HttpServletRequest)null); + AddReplaceFileHelper addFileHelper = new AddReplaceFileHelper( - dataverseRequestSvc.getDataverseRequest(), + dataverseRequest, this.ingestSvc, this.datasetSvc, this.dataFileSvc, @@ -923,10 +939,18 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut this.commandEngine, this.systemConfig ); + + // The old code had 2 sec. of sleep, so ... + Thread.sleep(2000); Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser); - if (Response.Status.OK.equals(addFilesResponse.getStatusInfo())) { + JsonReader jsonReader = Json.createReader(new StringReader((String) addFilesResponse.getEntity().toString())); + JsonObject jsonObject = jsonReader.readObject(); + String addFilesStatus = jsonObject.getString("status"); + String addFilesMessage = jsonObject.getJsonObject("data").getString("message"); + + if ("OK".equalsIgnoreCase(addFilesStatus)) { // if(!taskSkippedFiles) if (countError == 0) { userNotificationService.sendNotification((AuthenticatedUser) authUser, @@ -942,13 +966,14 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut } else { myLogger.log(Level.SEVERE, "******* Error while executing addFiles ", newjsonData); + // @todo send Failure notification + if (addFilesResponse != null) { + myLogger.info("addFilesResponse status: " + addFilesStatus); + myLogger.info("addFilesResponse message" + addFilesMessage); + } } - myLogger.info("Files processed: " + countAll); - myLogger.info("Files added successfully: " + countSuccess); - myLogger.info("Files failures: " + countError); - myLogger.info("Finished upload via Globus job."); - + } /** From 8f00cdb852d8ece974ec8feacbea32d1f938faf1 Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Thu, 15 Aug 2024 14:18:25 +0200 Subject: [PATCH 155/360] null check for name array #10343 --- .../java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java | 2 +- .../java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java index f68957ad060..80e32184731 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java @@ -123,7 +123,7 @@ public static JsonObject getPersonOrOrganization(String name, boolean organizati if (!name.replaceFirst(",", "").contains(",")) { // contributorName=, String[] fullName = name.split(", "); - givenName = fullName[1]; + givenName = fullName.length > 1 ? fullName[1] : null; familyName = fullName[0]; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java index c3d9fd8fcd3..1f1a2d7af00 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java @@ -89,6 +89,7 @@ public void testName() { verifyIsPerson("kcjim11, kcjim11", "kcjim11", "kcjim11"); verifyIsPerson("Bartholomew 3, James", "James", "Bartholomew 3"); + verifyIsPerson("Smith, ", null, "Smith"); } private void verifyIsOrganization(String fullName) { From b335c36fdabf01dd78897e7c5c60cde4151740cf Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Thu, 15 Aug 2024 14:29:04 +0200 Subject: [PATCH 156/360] release notes #10343 --- doc/release-notes/10343-trailing-comma.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/10343-trailing-comma.md diff --git a/doc/release-notes/10343-trailing-comma.md b/doc/release-notes/10343-trailing-comma.md new file mode 100644 index 00000000000..03bd18715d7 --- /dev/null +++ b/doc/release-notes/10343-trailing-comma.md @@ -0,0 +1,5 @@ +### Trailing commas in author name now permitted + +When an author name ends on a comma (e.g. "Smith,") a dataset cannot be properly loaded when using json-ld. A null check fixes this. + +For more information, see #10343. \ No newline at end of file From 91377ba38820f722c89aeaae820dc63e6ae76b63 Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Fri, 16 Aug 2024 09:47:03 +0200 Subject: [PATCH 157/360] added another test case + normalize String for assertion #10343 --- .../edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java index 1f1a2d7af00..d772ba2b9da 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java @@ -90,6 +90,7 @@ public void testName() { verifyIsPerson("Bartholomew 3, James", "James", "Bartholomew 3"); verifyIsPerson("Smith, ", null, "Smith"); + verifyIsPerson("Smith,", null, "Smith"); } private void verifyIsOrganization(String fullName) { @@ -107,7 +108,7 @@ private void verifyIsPerson(String fullName, String givenName, String familyName private void verifyIsPerson(String fullName, String givenName, String familyName, boolean isPerson) { JsonObject obj = PersonOrOrgUtil.getPersonOrOrganization(fullName, false, isPerson); System.out.println(JsonUtil.prettyPrint(obj)); - assertEquals(obj.getString("fullName"),fullName); + assertEquals(obj.getString("fullName"), StringUtil.normalize(fullName)); assertTrue(obj.getBoolean("isPerson")); assertEquals(obj.containsKey("givenName"), givenName != null); if(obj.containsKey("givenName") && givenName != null) { From d4b9bac8366a2b909c5c32cf5e3f4361d6abc7d5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 16 Aug 2024 20:21:44 -0400 Subject: [PATCH 158/360] this is a working, but still work-in-progress state of things - needs some cleanup and refinements. #10623 --- .../iq/dataverse/DatasetServiceBean.java | 11 +- .../harvard/iq/dataverse/api/Datasets.java | 2 + .../datasetutility/AddReplaceFileHelper.java | 4 +- .../impl/UpdateDatasetVersionCommand.java | 26 +- .../dataverse/globus/GlobusServiceBean.java | 232 ++++++++++-------- .../globus/GlobusTaskInProgress.java | 13 +- .../globus/TaskMonitoringServiceBean.java | 14 +- 7 files changed, 182 insertions(+), 120 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index dab0ff43fcf..832d7192965 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -412,12 +412,20 @@ public boolean checkDatasetLock(Long datasetId) { List lock = lockCounter.getResultList(); return lock.size()>0; } - + + public List getLocksByDatasetId(Long datasetId) { + TypedQuery locksQuery = em.createNamedQuery("DatasetLock.getLocksByDatasetId", DatasetLock.class); + locksQuery.setParameter("datasetId", datasetId); + return locksQuery.getResultList(); + } + public List getDatasetLocksByUser( AuthenticatedUser user) { return listLocks(null, user); } + // @todo: we'll be better off getting rid of this method and using the other + // version of addDatasetLock() (that uses datasetId instead of Dataset). @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public DatasetLock addDatasetLock(Dataset dataset, DatasetLock lock) { lock.setDataset(dataset); @@ -467,6 +475,7 @@ public DatasetLock addDatasetLock(Long datasetId, DatasetLock.Reason reason, Lon * is {@code aReason}. * @param dataset the dataset whose locks (for {@code aReason}) will be removed. * @param aReason The reason of the locks that will be removed. + * @todo this should probably take dataset_id, not a dataset */ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void removeDatasetLocks(Dataset dataset, DatasetLock.Reason aReason) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b93257bc0c3..4c547f5295f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -4035,6 +4035,8 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, return wr.getResponse(); } + // @todo check if the dataset is already locked! + JsonObject jsonObject = null; try { jsonObject = JsonUtil.getJsonObject(jsonData); diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 0143fced87c..2b35a4cc783 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -2139,9 +2139,9 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) { logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); } else { datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.log(Level.INFO, "Removed EditInProgress lock "); + logger.log(Level.INFO, "Removed EditInProgress lock "+eipLock.getId()); } - + try { Command cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, clone); ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 994f4c7dfb6..faf8884b08d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -14,6 +14,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DatasetFieldUtil; import edu.harvard.iq.dataverse.util.FileMetadataUtil; @@ -102,7 +103,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } Dataset theDataset = getDataset(); - ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); + //ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); + // this is an experiment (probably temporary) + checkUpdateDatasetVersionLock(ctxt); + Dataset savedDataset = null; DatasetVersion persistedVersion = clone; @@ -297,5 +301,23 @@ public boolean onSuccess(CommandContext ctxt, Object r) { ctxt.index().asyncIndexDataset((Dataset) r, true); return true; } - + + private void checkUpdateDatasetVersionLock(CommandContext ctxt) throws IllegalCommandException { + List locks = ctxt.datasets().getLocksByDatasetId(getDataset().getId()); + //locks.forEach(lock -> { + for (DatasetLock lock : locks) { + // Ingest lock is ok: + if (DatasetLock.Reason.Ingest != lock.getReason()) { + // with Workflow lock *some* users can edit; + // any other kind of lock - nope + if (DatasetLock.Reason.Workflow != lock.getReason() + || !ctxt.permissions().isMatchingWorkflowLock(getDataset(), + getUser().getIdentifier(), + getRequest().getWFInvocationId())) { + throw new IllegalCommandException( + BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), this); + } + } + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 3f6d0bf7c68..03134d811a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -51,6 +51,7 @@ import org.primefaces.PrimeFaces; import com.google.gson.Gson; +import edu.harvard.iq.dataverse.api.ApiConstants; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -68,6 +69,7 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; @@ -148,7 +150,7 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi * @param ruleId - Globus rule id - assumed to be associated with the * dataset's file path (should not be called with a user * specified rule id w/o further checking) - * @param datasetId - the id of the dataset associated with the rule + * @param dataset - the dataset associated with the rule * @param globusLogger - a separate logger instance, may be null */ public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) { @@ -690,7 +692,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S fileHandlerSuceeded = true; } catch (IOException | SecurityException ex) { Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.SEVERE, null, ex); - return; + return; // @todo ? } if (fileHandlerSuceeded) { @@ -706,8 +708,8 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S String taskIdentifier = jsonData.getString("taskIdentifier"); GlobusEndpoint endpoint = getGlobusEndpoint(dataset); - GlobusTaskState task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); - String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw"); + GlobusTaskState taskState = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); + String ruleId = getRuleId(endpoint, taskState.getOwner_id(), "rw"); logger.fine("Found rule: " + ruleId); if (ruleId != null) { Long datasetId = rulesCache.getIfPresent(ruleId); @@ -725,7 +727,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // Save the task information in the database so that the Globus monitoring // service can continue checking on its progress. - GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), new Timestamp(new Date().getTime())); + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), ruleId, new Timestamp(new Date().getTime())); em.persist(taskInProgress); // Save the metadata entries that define the files that are being uploaded @@ -746,17 +748,16 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // sleeping-then-checking the task status repeatedly: // globus task status check - // (the method below performs continuous looped checks of the remote + // (the following method performs continuous looped checks of the remote // Globus API, monitoring it for as long as it takes for the task to // finish one way or another!) - task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); + taskState = globusStatusCheck(endpoint, taskIdentifier, globusLogger); // @todo null check, or make sure it's never null - String taskStatus = getTaskStatus(task); + String taskStatus = getTaskStatus(taskState); if (ruleId != null) { // Transfer is complete, so delete rule deletePermission(ruleId, dataset, globusLogger); - } // If success, switch to an EditInProgress lock - do this before removing the @@ -764,8 +765,17 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // Keeping a lock through the add datafiles API call avoids a conflicting edit // and keeps any open dataset page refreshing until the datafile appears if (!(taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE"))) { - datasetSvc.addDatasetLock(dataset, - new DatasetLock(DatasetLock.Reason.EditInProgress, authUser, "Completing Globus Upload")); + globusLogger.info("Finished upload via Globus job."); + + DatasetLock editLock = datasetSvc.addDatasetLock(dataset.getId(), + DatasetLock.Reason.EditInProgress, + (authUser).getId(), + "Completing Globus Upload"); + if (editLock != null) { + dataset.addLock(editLock); + } else { + globusLogger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); + } } DatasetLock gLock = dataset.getLockFor(DatasetLock.Reason.GlobusUpload); @@ -785,7 +795,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S */ datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.GlobusUpload); } - + if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + taskStatus.split("#")[2]; @@ -911,10 +921,9 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut myLogger.info("Successfully generated new JsonData for addFiles call"); - myLogger.info("Files processed: " + countAll); - myLogger.info("Files added successfully: " + countSuccess); - myLogger.info("Files failures: " + countError); - myLogger.info("Finished upload via Globus job."); + myLogger.info("Files passed to /addGlobusFiles: " + countAll); + myLogger.info("Files processed successfully: " + countSuccess); + myLogger.info("Files failures to process: " + countError); /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" @@ -925,7 +934,7 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut // a quick experimental AddReplaceFileHelper implementation: // Passing null for the HttpServletRequest to make a new DataverseRequest. - // The parent method is executed asynchronously, so the real request + // The parent method is always executed asynchronously, so the real request // that was associated with the original API call that triggered this upload // cannot be obtained. DataverseRequest dataverseRequest = new DataverseRequest(authUser, (HttpServletRequest)null); @@ -945,12 +954,38 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser); - JsonReader jsonReader = Json.createReader(new StringReader((String) addFilesResponse.getEntity().toString())); - JsonObject jsonObject = jsonReader.readObject(); - String addFilesStatus = jsonObject.getString("status"); - String addFilesMessage = jsonObject.getJsonObject("data").getString("message"); + if (addFilesResponse == null) { + logger.info("null response from addFiles call"); + //@todo add this case to the user notification in case of error + return; + } + + JsonObject addFilesJsonObject = JsonUtil.getJsonObject(addFilesResponse.getEntity().toString()); - if ("OK".equalsIgnoreCase(addFilesStatus)) { + // @todo null checks etc. + String addFilesStatus = addFilesJsonObject.getString("status", null); + myLogger.info("addFilesResponse status: " + addFilesStatus); + + + if (ApiConstants.STATUS_OK.equalsIgnoreCase(addFilesStatus)) { + if (addFilesJsonObject.containsKey("data")) { + JsonObject responseFilesData = addFilesJsonObject.getJsonObject("data"); + if (responseFilesData.containsKey("Result")) { + JsonObject addFilesResult = responseFilesData.getJsonObject("Result"); + + Integer addFilesTotal = addFilesResult.getInt("Total number of files", -1); + Integer addFilesSuccess = addFilesResult.getInt("Number of files successfully added", -1); + // @todo handle -1 (missing values) above + // @todo log all this stuff in a task-specific log (??) + myLogger.info("Files processed by addFiles: " + addFilesTotal + ", successfully added: " + addFilesSuccess); + // @todo incorporate this into the user notification + } else { + logger.warning("Malformed addFiles data section: "+ responseFilesData.toString()); + } + } else { + logger.warning("Malformed addFiles response json: " + addFilesJsonObject.toString()); + } + // if(!taskSkippedFiles) if (countError == 0) { userNotificationService.sendNotification((AuthenticatedUser) authUser, @@ -963,89 +998,21 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut countSuccess + " files added out of " + countAll, true); } myLogger.info("Successfully completed addFiles call "); + } else if (ApiConstants.STATUS_ERROR.equalsIgnoreCase(addFilesStatus)) { + String addFilesMessage = addFilesJsonObject.getString("message", null); + + myLogger.log(Level.SEVERE, + "******* Error while executing addFiles ", newjsonData); + myLogger.log(Level.SEVERE, "****** Output from addFiles: ", addFilesMessage); + // @todo send Failure notification + } else { myLogger.log(Level.SEVERE, "******* Error while executing addFiles ", newjsonData); // @todo send Failure notification - if (addFilesResponse != null) { - myLogger.info("addFilesResponse status: " + addFilesStatus); - myLogger.info("addFilesResponse message" + addFilesMessage); - } } - - } - /** - * I don't think this method is needed at all. (I suspect that it's a remnant - * from the times when *multiple* individual /add calls needed to be performed - * for each file being added. So this was part of a framework that attempted - * to run this calls in parallel, potentially speeding things up (similarly to - * how the checksums are being calculated in parallel for multiple files). - * As of now, this method doesn't do anything "asynchronous" - there is one - * /addFiles call, and the method below will wait for it to complete, via the - * CompletableFuture.get(). (L.A.) - * @param curlCommand - * @param globusLogger - * @return - * @throws ExecutionException - * @throws InterruptedException - */ - /* - public String addFilesAsync(String curlCommand, Logger globusLogger) - throws ExecutionException, InterruptedException { - CompletableFuture addFilesFuture = CompletableFuture.supplyAsync(() -> { - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return (addFiles(curlCommand, globusLogger)); - }, executor).exceptionally(ex -> { - globusLogger.fine("Something went wrong : " + ex.getLocalizedMessage()); - ex.printStackTrace(); - return null; - }); - - String result = addFilesFuture.get(); - - return result; - } - - private String addFiles(String curlCommand, Logger globusLogger) { - ProcessBuilder processBuilder = new ProcessBuilder(); - Process process = null; - String line; - String status = ""; - - try { - globusLogger.info("Call to : " + curlCommand); - processBuilder.command("bash", "-c", curlCommand); - process = processBuilder.start(); - process.waitFor(); - - BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); - - StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null) - sb.append(line); - globusLogger.info(" API Output : " + sb.toString()); - JsonObject jsonObject = null; - jsonObject = JsonUtil.getJsonObject(sb.toString()); - - status = jsonObject.getString("status"); - } catch (Exception ex) { - if (ex instanceof JsonParsingException) { - globusLogger.log(Level.SEVERE, "Error parsing dataset json."); - } else { - globusLogger.log(Level.SEVERE, - "******* Unexpected Exception while executing api/datasets/:persistentId/add call ", ex); - } - } - - return status; - } */ - @Asynchronous public void globusDownload(String jsonData, Dataset dataset, User authUser) throws MalformedURLException { @@ -1418,22 +1385,75 @@ public List findExternalUploadsByTaskId(String tas return em.createNamedQuery("ExternalFileUploadInProgress.findByTaskId").setParameter("taskId", taskId).getResultList(); } - // @todo this may or may not need to be async (?) - public void addFilesOnSuccess(GlobusTaskInProgress globusTask) { - List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); - - if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { - // @todo log error message; do nothing - return; - } + // @todo duplicated code, merge with the code handling the "classic" upload workflow + public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess) { + String ruleId = globusTask.getRuleId(); Dataset dataset = globusTask.getDataset(); + if (ruleId != null) { + // Transfer is complete, so delete rule + deletePermission(ruleId, dataset, logger); + } + AuthenticatedUser authUser = authSvc.lookupUser(globusTask.getApiToken()); if (authUser == null) { // @todo log error message; do nothing return; } + + // Switch the locks on the dataset: + // @todo is it necessary? what is wrong exactly with keeping the Globus + // lock on for the duration of the process? + if (taskSuccess) { + DatasetLock editLock = datasetSvc.addDatasetLock(dataset.getId(), + DatasetLock.Reason.EditInProgress, + (authUser).getId(), + "Completing Globus Upload"); + if (editLock != null) { + dataset.addLock(editLock); + } else { + logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); + } + } + + // Remove the Globus lock, regardless of whether this is a success or failure + DatasetLock globusUploadLock = dataset.getLockFor(DatasetLock.Reason.GlobusUpload); + if (globusUploadLock == null) { + logger.log(Level.WARNING, "No GlobusUpload lock found for dataset"); + } else { + logger.log(Level.FINE, "Removing GlobusUpload lock " + globusUploadLock.getId()); + /* + * Note: This call to remove a lock only works immediately because it is in + * another service bean. Despite the removeDatasetLocks method having the + * REQUIRES_NEW transaction annotation, when the globusUpload method and that + * method were in the same bean (globusUpload was in the DatasetServiceBean to + * start), the globus lock was still seen in the API call initiated in the + * addFilesAsync method called within the globusUpload method. I.e. it appeared + * that the lock removal was not committed/visible outside this method until + * globusUpload itself ended. + */ + datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.GlobusUpload); + } + + if (taskSuccess && GlobusTaskInProgress.TaskType.UPLOAD.equals(globusTask.getTaskType())) { + List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); + + if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { + // @todo log error message; do nothing + return; + } + addFilesOnSuccess(dataset, authUser, fileUploadsInProgress); + } + // Handle locks/rules/etc. (?) + if (ruleId != null) { + deletePermission(ruleId, dataset, logger); + logger.info("Removed upload permission: " + ruleId); + } + } + + public void addFilesOnSuccess(Dataset dataset, AuthenticatedUser authUser, List fileUploadsInProgress) { + JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java index 210f08710dc..d3d06d38151 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -86,6 +86,9 @@ public String toString() { */ private String apiToken; + @Column + private String ruleId; + @ManyToOne private Dataset dataset; @@ -95,12 +98,13 @@ public String toString() { public GlobusTaskInProgress() { } - GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String globusToken, String apiToken, Timestamp startTime) { + GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String globusToken, String apiToken, String ruleId, Timestamp startTime) { this.taskId = taskId; this.taskType = taskType; this.dataset = dataset; this.globusToken = globusToken; this.apiToken = apiToken; + this.ruleId = ruleId; this.startTime = startTime; } @@ -146,6 +150,13 @@ public void setApiToken(String apiToken) { this.apiToken = apiToken; } + public String getRuleId() { + return ruleId; + } + + public void setRuleId(String ruleId) { + this.ruleId = ruleId; + } public Dataset getDataset() { return dataset; } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index bd274d44e38..4db25072b6b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -49,6 +49,7 @@ public void init() { logger.info("Starting Globus task monitoring service"); int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 60); + // @todo scheduleAtFixedDelay() this.scheduler.scheduleAtFixedRate(this::checkOngoingTasks, 0, pollingInterval, TimeUnit.SECONDS); @@ -65,20 +66,17 @@ public void init() { public void checkOngoingTasks() { logger.info("Performing a scheduled external Globus task check"); List tasks = globusService.findAllOngoingTasks(); - + tasks.forEach(t -> { GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), null); if (GlobusUtil.isTaskCompleted(retrieved)) { - if (GlobusUtil.isTaskSucceeded(retrieved)) { - // Do our thing, finalize adding the files to the dataset - globusService.addFilesOnSuccess(t); - } + // Do our thing, finalize adding the files to the dataset + globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved)); // Whether it finished successfully, or failed in the process, // there's no need to keep monitoring this task, so we can - // delete it. - globusService.deleteExternalUploadRecords(t.getTaskId()); + // delete it. + //globusService.deleteExternalUploadRecords(t.getTaskId()); globusService.deleteTask(t); - // @todo double-check that the locks have been properly handled } }); } From 9c62b81c484111df3ed9162c8353aa50bd4b295b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 19 Aug 2024 10:47:29 -0400 Subject: [PATCH 159/360] refined logging #10623 --- .../dataverse/globus/GlobusServiceBean.java | 231 ++++++++---------- .../iq/dataverse/globus/GlobusUtil.java | 27 ++ .../globus/TaskMonitoringServiceBean.java | 61 ++++- 3 files changed, 191 insertions(+), 128 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 03134d811a7..5e68128c954 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -679,20 +679,24 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S throw new IllegalArgumentException("No valid json entries supplied for the files being uploaded"); } - String logTimestamp = logFormatter.format(new Date()); + Date startDate = new Date(); + + String logTimestamp = logFormatter.format(startDate); Logger globusLogger = Logger.getLogger( "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimestamp); - String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_id_" + dataset.getId() + "_" + logTimestamp + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + dataset.getId() + "_" + logTimestamp + ".log"; FileHandler fileHandler; - boolean fileHandlerSuceeded; + boolean fileHandlerSuceeded = false; try { fileHandler = new FileHandler(logFileName); globusLogger.setUseParentHandlers(false); fileHandlerSuceeded = true; } catch (IOException | SecurityException ex) { Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.SEVERE, null, ex); - return; // @todo ? + //return; // @todo I don't think we need to return here? + fileHandler = null; + } if (fileHandlerSuceeded) { @@ -727,7 +731,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // Save the task information in the database so that the Globus monitoring // service can continue checking on its progress. - GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), ruleId, new Timestamp(new Date().getTime())); + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), ruleId, new Timestamp(startDate.getTime())); em.persist(taskInProgress); // Save the metadata entries that define the files that are being uploaded @@ -740,6 +744,11 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S em.persist(fileUploadRecord); } + if (fileHandler != null) { + fileHandler.close(); + } + + // return and forget return; } @@ -753,19 +762,62 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // finish one way or another!) taskState = globusStatusCheck(endpoint, taskIdentifier, globusLogger); // @todo null check, or make sure it's never null - String taskStatus = getTaskStatus(taskState); + String taskStatus = GlobusUtil.getTaskStatus(taskState); + boolean taskSuccess = GlobusUtil.isTaskCompleted(taskState); + + processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, globusLogger, fileHandler, taskSuccess, taskStatus); + + if (fileHandler != null) { + fileHandler.close(); + } + } + /** + * As the name suggests, the method completes and finalizes an upload task, + * whether it completed successfully or failed. (In the latter case, it + * simply sends a failure notification and does some cleanup). + * The method is called in both task monitoring scenarios: the old method, + * that relies on continuous looping, and the new, implemented on the basis + * of timer-like monitoring from a dedicated monitoring Singleton service. + * @param dataset the dataset + * @param filesJsonArray JsonArray containing files metadata entries as passed to /addGlobusFiles + * @param authUser the user that should be be performing the addFiles call + * finalizing adding the files to the Dataset. Note that this + * user will need to be obtained from the saved api token, when this + * method is called via the TaskMonitoringService + * @param ruleId Globus rule/permission id associated with the task + * @param myLogger the Logger; if null, the main logger of the service bean will be used + * @param fileHandler FileHandler associated with the Logger, when not null + * @param taskSuccess boolean task status of the completed task + * @param taskState human-readable task status label as reported by the Globus API + * the method should not throw any exceptions; all the exceptions thrown + * by the methods within are expected to be intercepted. + */ + private void processCompletedUploadTask(Dataset dataset, + JsonArray filesJsonArray, + AuthenticatedUser authUser, + String ruleId, + Logger globusLogger, + FileHandler fileHandler, + boolean taskSuccess, + String taskStatus) { + + Logger myLogger = globusLogger == null ? logger : globusLogger; + if (ruleId != null) { // Transfer is complete, so delete rule - deletePermission(ruleId, dataset, globusLogger); + deletePermission(ruleId, dataset, myLogger); } - + // If success, switch to an EditInProgress lock - do this before removing the // GlobusUpload lock // Keeping a lock through the add datafiles API call avoids a conflicting edit - // and keeps any open dataset page refreshing until the datafile appears - if (!(taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE"))) { - globusLogger.info("Finished upload via Globus job."); + // and keeps any open dataset page refreshing until the datafile appears. + + // @todo is it necessary? what is wrong exactly with keeping the Globus + // lock on for the duration of the process? + if (taskSuccess) { + myLogger.info("Finished upload via Globus job."); DatasetLock editLock = datasetSvc.addDatasetLock(dataset.getId(), DatasetLock.Reason.EditInProgress, @@ -774,7 +826,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S if (editLock != null) { dataset.addLock(editLock); } else { - globusLogger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); + myLogger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); } } @@ -792,36 +844,50 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S * addFilesAsync method called within the globusUpload method. I.e. it appeared * that the lock removal was not committed/visible outside this method until * globusUpload itself ended. + * (from @landreev:) If I understand the comment above correctly - annotations + * like "@TransactionAttribute(REQUIRES_NEW) do NOT work when you call a method + * directly within the same service bean. Strictly speaking, it's not the + * "within the same bean" part that is the key, rather, these annotations + * only apply when calling a method via an @EJB-defined service. So it + * is generally possible to call another method within FooServiceBean + * with the REQUIRES_NEW transaction taking effect - but then it would need + * to define *itself* as an @EJB - + * @EJB FooServiceBean fooSvc; + * ... + * fooSvc.doSomethingInNewTransaction(...); + * etc. */ datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.GlobusUpload); } - if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { + if (!taskSuccess) { String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + taskStatus.split("#")[2]; userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - globusLogger.info("Globus task failed "); + myLogger.info("Globus task failed "); } else { try { - processUploadedFiles(filesJsonArray, dataset, authUser, globusLogger); + processUploadedFiles(filesJsonArray, dataset, authUser, myLogger); } catch (Exception e) { logger.info("Exception from processUploadedFiles call "); e.printStackTrace(); - globusLogger.info("Exception from processUploadedFiles call " + e.getMessage()); + myLogger.info("Exception from processUploadedFiles call " + e.getMessage()); datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - // } } if (ruleId != null) { - deletePermission(ruleId, dataset, globusLogger); - globusLogger.info("Removed upload permission: " + ruleId); - } - if (fileHandlerSuceeded) { - fileHandler.close(); + deletePermission(ruleId, dataset, myLogger); + myLogger.info("Removed upload permission: " + ruleId); } + //if (fileHandler != null) { + // fileHandler.close(); + //} + } + + /** * The code in this method is copy-and-pasted from the previous Borealis * implemenation. @@ -1075,7 +1141,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro } task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); // @todo null check - String taskStatus = getTaskStatus(task); + String taskStatus = GlobusUtil.getTaskStatus(task); // Transfer is done (success or failure) so delete the rule if (ruleId != null) { @@ -1132,30 +1198,6 @@ private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId return task; } - private String getTaskStatus(GlobusTaskState task) { - String status = null; - if (task != null) { - status = task.getStatus(); - if (status != null) { - // The task is in progress but is not ok or queued - // (L.A.) I think the assumption here is that this method is called - // exclusively on tasks that have already completed. So that's why - // it is safe to assume that "ACTIVE" means "FAILED". - if (status.equalsIgnoreCase("ACTIVE")) { - status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); - } else { - // The task is either succeeded, failed or inactive. - status = status + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); - } - } else { - status = "FAILED"; - } - } else { - status = "FAILED"; - } - return status; - } - public JsonObject calculateMissingMetadataFields(List inputList, Logger globusLogger) throws InterruptedException, ExecutionException, IOException { @@ -1386,98 +1428,41 @@ public List findExternalUploadsByTaskId(String tas } // @todo duplicated code, merge with the code handling the "classic" upload workflow - public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess) { + public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess, String taskStatus, Logger taskLogger) { String ruleId = globusTask.getRuleId(); Dataset dataset = globusTask.getDataset(); - - if (ruleId != null) { - // Transfer is complete, so delete rule - deletePermission(ruleId, dataset, logger); - } - AuthenticatedUser authUser = authSvc.lookupUser(globusTask.getApiToken()); if (authUser == null) { // @todo log error message; do nothing return; } - // Switch the locks on the dataset: - // @todo is it necessary? what is wrong exactly with keeping the Globus - // lock on for the duration of the process? - if (taskSuccess) { - DatasetLock editLock = datasetSvc.addDatasetLock(dataset.getId(), - DatasetLock.Reason.EditInProgress, - (authUser).getId(), - "Completing Globus Upload"); - if (editLock != null) { - dataset.addLock(editLock); - } else { - logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); - } - } - - // Remove the Globus lock, regardless of whether this is a success or failure - DatasetLock globusUploadLock = dataset.getLockFor(DatasetLock.Reason.GlobusUpload); - if (globusUploadLock == null) { - logger.log(Level.WARNING, "No GlobusUpload lock found for dataset"); - } else { - logger.log(Level.FINE, "Removing GlobusUpload lock " + globusUploadLock.getId()); - /* - * Note: This call to remove a lock only works immediately because it is in - * another service bean. Despite the removeDatasetLocks method having the - * REQUIRES_NEW transaction annotation, when the globusUpload method and that - * method were in the same bean (globusUpload was in the DatasetServiceBean to - * start), the globus lock was still seen in the API call initiated in the - * addFilesAsync method called within the globusUpload method. I.e. it appeared - * that the lock removal was not committed/visible outside this method until - * globusUpload itself ended. - */ - datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.GlobusUpload); - } - - if (taskSuccess && GlobusTaskInProgress.TaskType.UPLOAD.equals(globusTask.getTaskType())) { + if (GlobusTaskInProgress.TaskType.UPLOAD.equals(globusTask.getTaskType())) { List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { // @todo log error message; do nothing return; } - addFilesOnSuccess(dataset, authUser, fileUploadsInProgress); - } - - // Handle locks/rules/etc. (?) - if (ruleId != null) { - deletePermission(ruleId, dataset, logger); - logger.info("Removed upload permission: " + ruleId); - } - } - - public void addFilesOnSuccess(Dataset dataset, AuthenticatedUser authUser, List fileUploadsInProgress) { - - JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); - - for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { - String jsonInfoString = pendingFile.getFileInfo(); - JsonObject fileObject = JsonUtil.getJsonObject(jsonInfoString); - filesJsonArrayBuilder.add(fileObject); - } - - JsonArray filesJsonArray = filesJsonArrayBuilder.build(); - - if (filesJsonArray == null || filesJsonArray.size() < 1) { - // @todo log error message; do nothing - return; - } - - try { - processUploadedFiles(filesJsonArray, dataset, authUser, null); - } catch (Exception ex) { - // @todo log error message; make sure the error notification to the - // has been sent (may or may not have already been sent inside the - // method above). + + JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); + + for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { + String jsonInfoString = pendingFile.getFileInfo(); + JsonObject fileObject = JsonUtil.getJsonObject(jsonInfoString); + filesJsonArrayBuilder.add(fileObject); + } + + JsonArray filesJsonArray = filesJsonArrayBuilder.build(); + + //processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, globusLogger, fileHandler, taskSuccess, taskStatus); + processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, taskLogger, null, taskSuccess, taskStatus); + } else { + // @todo extend this async. framework to handle Glonus downloads as well } + } - + public void deleteExternalUploadRecords(String taskId) { em.createNamedQuery("ExternalFileUploadInProgress.deleteByTaskId") .setParameter("taskId", taskId) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java index 67594ad1a5e..652898591ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java @@ -63,4 +63,31 @@ public static boolean isTaskSucceeded(GlobusTaskState task) { } return false; } + /** + * Produces a human-readable Status label of a completed task + * @param GlobusTaskState task - a looked-up state of a task as reported by Globus API + */ + public static String getTaskStatus(GlobusTaskState task) { + String status = null; + if (task != null) { + status = task.getStatus(); + if (status != null) { + // The task is in progress but is not ok or queued + // (L.A.) I think the assumption here is that this method is called + // exclusively on tasks that have already completed. So that's why + // it is safe to assume that "ACTIVE" means "FAILED". + if (status.equalsIgnoreCase("ACTIVE")) { + status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); + } else { + // The task is either succeeded, failed or inactive. + status = status + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); + } + } else { + status = "FAILED"; + } + } else { + status = "FAILED"; + } + return status; + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index 4db25072b6b..a74d0c3f747 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -12,8 +12,13 @@ import jakarta.ejb.Singleton; import jakarta.ejb.Startup; import jakarta.enterprise.concurrent.ManagedScheduledExecutorService; +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.logging.FileHandler; import java.util.logging.Logger; /** @@ -42,15 +47,16 @@ public class TaskMonitoringServiceBean { SettingsServiceBean settingsSvc; @EJB GlobusServiceBean globusService; - + + private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); + @PostConstruct public void init() { if (systemConfig.isGlobusTaskMonitoringServer()) { logger.info("Starting Globus task monitoring service"); int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 60); - // @todo scheduleAtFixedDelay() - this.scheduler.scheduleAtFixedRate(this::checkOngoingTasks, + this.scheduler.scheduleWithFixedDelay(this::checkOngoingTasks, 0, pollingInterval, TimeUnit.SECONDS); } else { @@ -68,17 +74,62 @@ public void checkOngoingTasks() { List tasks = globusService.findAllOngoingTasks(); tasks.forEach(t -> { - GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), null); + FileHandler taskLogHandler = getTaskLogHandler(t); + Logger taskLogger = getTaskLogger(t, taskLogHandler); + + GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), taskLogger); if (GlobusUtil.isTaskCompleted(retrieved)) { // Do our thing, finalize adding the files to the dataset - globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved)); + globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getTaskStatus(retrieved), taskLogger); // Whether it finished successfully, or failed in the process, // there's no need to keep monitoring this task, so we can // delete it. //globusService.deleteExternalUploadRecords(t.getTaskId()); globusService.deleteTask(t); } + + if (taskLogHandler != null) { + // @todo it should be prudent to cache these loggers and handlers + // between monitoring runs + taskLogHandler.close(); + } }); } + private FileHandler getTaskLogHandler(GlobusTaskInProgress task) { + if (task == null) { + return null; + } + + Date startDate = new Date(task.getStartTime().getTime()); + String logTimeStamp = logFormatter.format(startDate); + + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + task.getDataset().getId() + "_" + logTimeStamp + + ".log"; + FileHandler fileHandler; + try { + fileHandler = new FileHandler(logFileName); + } catch (IOException | SecurityException ex) { + // @todo log this error somehow? + fileHandler = null; + } + return fileHandler; + } + + private Logger getTaskLogger(GlobusTaskInProgress task, FileHandler logFileHandler) { + if (logFileHandler == null) { + return null; + } + Date startDate = new Date(task.getStartTime().getTime()); + String logTimeStamp = logFormatter.format(startDate); + + Logger taskLogger = Logger.getLogger( + "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimeStamp); + taskLogger.setUseParentHandlers(false); + + taskLogger.addHandler(logFileHandler); + + return taskLogger; + } + } From 8cdff8d66eb655e8def3e35b21f1a3d438f5608a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 19 Aug 2024 19:51:43 -0400 Subject: [PATCH 160/360] Added notifications for various failure cases. #10623 --- .../harvard/iq/dataverse/MailServiceBean.java | 27 +++- .../iq/dataverse/UserNotification.java | 3 +- .../iq/dataverse/api/ApiConstants.java | 4 + .../providers/builtin/DataverseUserPage.java | 2 + .../datasetutility/AddReplaceFileHelper.java | 4 +- .../dataverse/globus/GlobusServiceBean.java | 120 ++++++++++-------- .../globus/TaskMonitoringServiceBean.java | 3 +- .../harvard/iq/dataverse/util/MailUtil.java | 17 +++ src/main/java/propertyFiles/Bundle.properties | 9 +- src/main/webapp/dataverseuser.xhtml | 14 ++ 10 files changed, 144 insertions(+), 59 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 7359ef8eb33..d29649ad3a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -623,6 +623,7 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio comment )) ; return downloadCompletedMessage; + case GLOBUSUPLOADCOMPLETEDWITHERRORS: dataset = (Dataset) targetObject; messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); @@ -633,8 +634,30 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio comment )) ; return uploadCompletedWithErrorsMessage; + + case GLOBUSUPLOADREMOTEFAILURE: + dataset = (Dataset) targetObject; + messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); + String uploadFailedRemotelyMessage = messageText + BundleUtil.getStringFromBundle("notification.mail.globus.upload.failedRemotely", Arrays.asList( + systemConfig.getDataverseSiteUrl(), + dataset.getGlobalId().asString(), + dataset.getDisplayName(), + comment + )) ; + return uploadFailedRemotelyMessage; - case GLOBUSDOWNLOADCOMPLETEDWITHERRORS: + case GLOBUSUPLOADLOCALFAILURE: + dataset = (Dataset) targetObject; + messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); + String uploadFailedLocallyMessage = messageText + BundleUtil.getStringFromBundle("notification.mail.globus.upload.failedLocally", Arrays.asList( + systemConfig.getDataverseSiteUrl(), + dataset.getGlobalId().asString(), + dataset.getDisplayName(), + comment + )) ; + return uploadFailedLocallyMessage; + + case GLOBUSDOWNLOADCOMPLETEDWITHERRORS: dataset = (Dataset) targetObject; messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html"); String downloadCompletedWithErrorsMessage = messageText + BundleUtil.getStringFromBundle("notification.mail.globus.download.completedWithErrors", Arrays.asList( @@ -763,6 +786,8 @@ public Object getObjectOfNotification (UserNotification userNotification){ return versionService.find(userNotification.getObjectId()); case GLOBUSUPLOADCOMPLETED: case GLOBUSUPLOADCOMPLETEDWITHERRORS: + case GLOBUSUPLOADREMOTEFAILURE: + case GLOBUSUPLOADLOCALFAILURE: case GLOBUSDOWNLOADCOMPLETED: case GLOBUSDOWNLOADCOMPLETEDWITHERRORS: return datasetService.find(userNotification.getObjectId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index 280c2075494..2d37540fab3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -39,7 +39,8 @@ public enum Type { CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE, STATUSUPDATED, DATASETCREATED, DATASETMENTIONED, GLOBUSUPLOADCOMPLETED, GLOBUSUPLOADCOMPLETEDWITHERRORS, - GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS, REQUESTEDFILEACCESS; + GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS, REQUESTEDFILEACCESS, + GLOBUSUPLOADREMOTEFAILURE, GLOBUSUPLOADLOCALFAILURE; public String getDescription() { return BundleUtil.getStringFromBundle("notification.typeDescription." + this.name()); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java index 347a8946a46..a2faaf3637c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java @@ -17,4 +17,8 @@ private ApiConstants() { public static final String DS_VERSION_LATEST = ":latest"; public static final String DS_VERSION_DRAFT = ":draft"; public static final String DS_VERSION_LATEST_PUBLISHED = ":latest-published"; + + // addFiles call + public static final String API_ADD_FILES_COUNT_PROCESSED = "Total number of files"; + public static final String API_ADD_FILES_COUNT_SUCCESSFULL = "Number of files successfully added"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index a0e3f899443..48afb2b830a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -528,6 +528,8 @@ public void displayNotification() { case GLOBUSUPLOADCOMPLETEDWITHERRORS: case GLOBUSDOWNLOADCOMPLETED: case GLOBUSDOWNLOADCOMPLETEDWITHERRORS: + case GLOBUSUPLOADREMOTEFAILURE: + case GLOBUSUPLOADLOCALFAILURE: userNotification.setTheObject(datasetService.find(userNotification.getObjectId())); break; diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 2b35a4cc783..336fa9b5b7a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -2167,8 +2167,8 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) { } JsonObjectBuilder result = Json.createObjectBuilder() - .add("Total number of files", totalNumberofFiles) - .add("Number of files successfully added", successNumberofFiles); + .add(ApiConstants.API_ADD_FILES_COUNT_PROCESSED, totalNumberofFiles) + .add(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFULL, successNumberofFiles); return Response.ok().entity(Json.createObjectBuilder() diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 5e68128c954..4bb478d26ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -766,7 +766,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S boolean taskSuccess = GlobusUtil.isTaskCompleted(taskState); - processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, globusLogger, fileHandler, taskSuccess, taskStatus); + processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, globusLogger, taskSuccess, taskStatus); if (fileHandler != null) { fileHandler.close(); @@ -798,7 +798,6 @@ private void processCompletedUploadTask(Dataset dataset, AuthenticatedUser authUser, String ruleId, Logger globusLogger, - FileHandler fileHandler, boolean taskSuccess, String taskStatus) { @@ -861,11 +860,17 @@ private void processCompletedUploadTask(Dataset dataset, } if (!taskSuccess) { - String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " - + taskStatus.split("#")[2]; + String comment; + if (taskStatus != null) { + comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + + taskStatus.split("#")[2]; + } else { + comment = "No further information available"; + } + + myLogger.info("Globus Upload task failed "); userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - myLogger.info("Globus task failed "); + UserNotification.Type.GLOBUSUPLOADREMOTEFAILURE, dataset.getId(), comment, true); } else { try { @@ -908,6 +913,8 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut Integer countAll = 0; Integer countSuccess = 0; Integer countError = 0; + Integer countAddFilesSuccess = 0; + String notificationErrorMessage = ""; List inputList = new ArrayList(); @@ -991,13 +998,22 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut myLogger.info("Files processed successfully: " + countSuccess); myLogger.info("Files failures to process: " + countError); - /*String command = "curl -H \"X-Dataverse-key:" + token.getTokenString() + "\" -X POST " - + httpRequestUrl + "/api/datasets/:persistentId/addFiles?persistentId=doi:" - + datasetIdentifier + " -F jsonData='" + newjsonData + "'"; - System.out.println("*******====command ==== " + command);*/ - // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of - // calling API - // a quick experimental AddReplaceFileHelper implementation: + if (countSuccess < 1) { + // We don't have any valid entries to call addFiles() for; so, no + // need to proceed. + notificationErrorMessage = "Failed to successfully process any of the file entries, " + + "out of the " + countAll + " total as submitted to Dataverse"; + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADREMOTEFAILURE, + dataset.getId(), notificationErrorMessage, true); + return; + } else if (countSuccess < countAll) { + notificationErrorMessage = "Out of the " + countAll + " file entries submitted to /addGlobusFiles " + + "only " + countSuccess + " could be successfully parsed and processed. "; + } + + // A new AddReplaceFileHelper implementation, replacing the old one that + // was relying on calling /addFiles api via curl: // Passing null for the HttpServletRequest to make a new DataverseRequest. // The parent method is always executed asynchronously, so the real request @@ -1028,55 +1044,57 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut JsonObject addFilesJsonObject = JsonUtil.getJsonObject(addFilesResponse.getEntity().toString()); - // @todo null checks etc. + // @todo null check? String addFilesStatus = addFilesJsonObject.getString("status", null); myLogger.info("addFilesResponse status: " + addFilesStatus); - if (ApiConstants.STATUS_OK.equalsIgnoreCase(addFilesStatus)) { - if (addFilesJsonObject.containsKey("data")) { - JsonObject responseFilesData = addFilesJsonObject.getJsonObject("data"); - if (responseFilesData.containsKey("Result")) { - JsonObject addFilesResult = responseFilesData.getJsonObject("Result"); - - Integer addFilesTotal = addFilesResult.getInt("Total number of files", -1); - Integer addFilesSuccess = addFilesResult.getInt("Number of files successfully added", -1); - // @todo handle -1 (missing values) above - // @todo log all this stuff in a task-specific log (??) - myLogger.info("Files processed by addFiles: " + addFilesTotal + ", successfully added: " + addFilesSuccess); - // @todo incorporate this into the user notification - } else { - logger.warning("Malformed addFiles data section: "+ responseFilesData.toString()); - } - } else { - logger.warning("Malformed addFiles response json: " + addFilesJsonObject.toString()); - } - - // if(!taskSkippedFiles) - if (countError == 0) { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADCOMPLETED, - dataset.getId(), countSuccess + " files added out of " + countAll, true); + if (addFilesJsonObject.containsKey("data") && addFilesJsonObject.getJsonObject("data").containsKey("Result")) { + + //Integer countAddFilesTotal = addFilesJsonObject.getJsonObject("data").getJsonObject("Result").getInt(ApiConstants.API_ADD_FILES_COUNT_PROCESSED, -1); + countAddFilesSuccess = addFilesJsonObject.getJsonObject("data").getJsonObject("Result").getInt(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFULL, -1); + myLogger.info("Files successfully added by addFiles(): " + countAddFilesSuccess); + } else { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), - countSuccess + " files added out of " + countAll, true); + myLogger.warning("Malformed addFiles response json: " + addFilesJsonObject.toString()); + notificationErrorMessage = "Malformed response received when attempting to add the files to the dataset. "; } - myLogger.info("Successfully completed addFiles call "); + + myLogger.info("Completed addFiles call "); } else if (ApiConstants.STATUS_ERROR.equalsIgnoreCase(addFilesStatus)) { String addFilesMessage = addFilesJsonObject.getString("message", null); - + myLogger.log(Level.SEVERE, "******* Error while executing addFiles ", newjsonData); myLogger.log(Level.SEVERE, "****** Output from addFiles: ", addFilesMessage); - // @todo send Failure notification + notificationErrorMessage += "Error response received when attempting to add the files to the dataset: " + addFilesMessage + " "; } else { myLogger.log(Level.SEVERE, "******* Error while executing addFiles ", newjsonData); - // @todo send Failure notification + notificationErrorMessage += "Unexpected error encountered when attemptingh to add the files to the dataset."; + } + + // if(!taskSkippedFiles) + if (countAddFilesSuccess == countAll) { + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSUPLOADCOMPLETED, + dataset.getId(), countSuccess + " files added out of " + countAll, true); + } else if (countAddFilesSuccess > 0) { + // success, but partial: + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSUPLOADCOMPLETEDWITHERRORS, dataset.getId(), + countSuccess + " files added out of " + countAll + notificationErrorMessage, true); + } else { + notificationErrorMessage = "".equals(notificationErrorMessage) + ? " No additional information is available." : notificationErrorMessage; + userNotificationService.sendNotification((AuthenticatedUser) authUser, + new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSUPLOADLOCALFAILURE, dataset.getId(), + notificationErrorMessage, true); } + } @Asynchronous @@ -1140,7 +1158,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro logger.warning("ruleId not found for taskId: " + taskIdentifier); } task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); - // @todo null check + // @todo null check? String taskStatus = GlobusUtil.getTaskStatus(task); // Transfer is done (success or failure) so delete the rule @@ -1427,13 +1445,13 @@ public List findExternalUploadsByTaskId(String tas return em.createNamedQuery("ExternalFileUploadInProgress.findByTaskId").setParameter("taskId", taskId).getResultList(); } - // @todo duplicated code, merge with the code handling the "classic" upload workflow public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess, String taskStatus, Logger taskLogger) { String ruleId = globusTask.getRuleId(); Dataset dataset = globusTask.getDataset(); AuthenticatedUser authUser = authSvc.lookupUser(globusTask.getApiToken()); if (authUser == null) { // @todo log error message; do nothing + // (the fields in GlobusTaskInProgress are not nullable though - ?) return; } @@ -1442,6 +1460,7 @@ public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSu if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { // @todo log error message; do nothing + // (will this ever happen though?) return; } @@ -1455,10 +1474,9 @@ public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSu JsonArray filesJsonArray = filesJsonArrayBuilder.build(); - //processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, globusLogger, fileHandler, taskSuccess, taskStatus); - processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, taskLogger, null, taskSuccess, taskStatus); + processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, taskLogger, taskSuccess, taskStatus); } else { - // @todo extend this async. framework to handle Glonus downloads as well + // @todo eventually, extend this async. framework to handle Glonus downloads as well } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index a74d0c3f747..c956831317c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -67,7 +67,6 @@ public void init() { /** * This method will be executed on a timer-like schedule, continuously * monitoring all the ongoing external Globus tasks (transfers). - * @todo make sure the executions do not overlap/stack up */ public void checkOngoingTasks() { logger.info("Performing a scheduled external Globus task check"); @@ -90,7 +89,7 @@ public void checkOngoingTasks() { if (taskLogHandler != null) { // @todo it should be prudent to cache these loggers and handlers - // between monitoring runs + // between monitoring runs (should be fairly easy to do) taskLogHandler.close(); } }); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 36c249de834..f81ce093815 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -99,6 +99,23 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti } catch (Exception e) { return BundleUtil.getStringFromBundle("notification.email.globus.uploadCompletedWithErrors.subject", rootDvNameAsList); } + case GLOBUSUPLOADREMOTEFAILURE: + try { + DatasetVersion version = (DatasetVersion)objectOfNotification; + List dsNameAsList = Arrays.asList(version.getDataset().getDisplayName()); + return BundleUtil.getStringFromBundle("notification.email.globus.uploadFailedRemotely.subject", dsNameAsList); + + } catch (Exception e) { + return BundleUtil.getStringFromBundle("notification.email.globus.uploadFailedRemotely.subject", rootDvNameAsList); + } + case GLOBUSUPLOADLOCALFAILURE: + try { + DatasetVersion version = (DatasetVersion)objectOfNotification; + List dsNameAsList = Arrays.asList(version.getDataset().getDisplayName()); + return BundleUtil.getStringFromBundle("notification.email.globus.uploadFailedLocally.subject", dsNameAsList); + } catch (Exception e) { + return BundleUtil.getStringFromBundle("notification.email.globus.uploadFailedLocally.subject", rootDvNameAsList); + } case GLOBUSDOWNLOADCOMPLETEDWITHERRORS: try { DatasetVersion version = (DatasetVersion)objectOfNotification; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 4b366522966..e0488f95e2c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -253,11 +253,16 @@ notification.mail.import.filesystem=Dataset {2} ({0}/dataset.xhtml?persistentId= notification.mail.globus.upload.completed=Globus transfer to Dataset {2} was successful. File(s) have been uploaded and verified.

{3}
notification.mail.globus.download.completed=Globus transfer of file(s) from the dataset {2} was successful.

{3}
notification.mail.globus.upload.completedWithErrors=Globus transfer to Dataset {2} is complete with errors.

{3}
+notification.mail.globus.upload.failedRemotely=Remote data transfer between Globus collections for Dataset {2} failed, reported via Globus API.

{3}
+notification.mail.globus.upload.failedLocally=Dataverse received a confirmation of a successful Globus data transfer for Dataset {2}, but failed to add the files to the dataset locally.

{3}
notification.mail.globus.download.completedWithErrors=Globus transfer from the dataset {2} is complete with errors.

{3}
notification.import.filesystem=Dataset {1} has been successfully uploaded and verified. notification.globus.upload.completed=Globus transfer to Dataset {1} was successful. File(s) have been uploaded and verified. notification.globus.download.completed=Globus transfer from the dataset {1} was successful. notification.globus.upload.completedWithErrors=Globus transfer to Dataset {1} is complete with errors. +notification.globus.upload.failedRemotely=Remote data transfer between Globus collections for Dataset {2} failed, reported via Globus API.

{3}
+notification.globus.upload.failedLocally=Dataverse received a confirmation of a successful Globus data transfer for Dataset {2}, but failed to add the files to the dataset locally.

{3}
+ notification.globus.download.completedWithErrors=Globus transfer from the dataset {1} is complete with errors. notification.import.checksum={1}, dataset had file checksums added via a batch job. removeNotification=Remove Notification @@ -823,8 +828,8 @@ notification.email.datasetWasMentioned.subject={0}: A Dataset Relationship has b notification.email.globus.uploadCompleted.subject={0}: Files uploaded successfully via Globus and verified notification.email.globus.downloadCompleted.subject={0}: Files downloaded successfully via Globus notification.email.globus.uploadCompletedWithErrors.subject={0}: Uploaded files via Globus with errors -notification.email.globus.downloadCompletedWithErrors.subject={0}: Downloaded files via Globus with errors - +notification.email.globus.uploadFailedRemotely.subject={0}: Failed to upload files via Globus +notification.email.globus.uploadFailedLocally.subject={0}: Failed to add files uploaded via Globus to dataset # dataverse.xhtml dataverse.name=Dataverse Name dataverse.name.title=The project, department, university, professor, or journal this dataverse will contain data for. diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 9ed8b5209b6..d061348ad87 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -367,6 +367,20 @@ + + + + + + + + + + + + + + From 531e25c05df0ec507b5e3c0650033b0691c96df2 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 20 Aug 2024 09:23:36 -0400 Subject: [PATCH 161/360] Config guide entry. #10623 --- doc/sphinx-guides/source/installation/config.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0038c188ea5..01bf1419a52 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3259,6 +3259,13 @@ The email for your institution that you'd like to appear in bag-info.txt. See :r Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_EMAIL``. +.. _dataverse.bagit.sourceorg.email: + +dataverse.globus.taskMonitoringServer ++++++++++++++++++++++++++++++++++++++ + +This setting is required in conjunction with the ``globus-use-experimental-async-framework`` feature flag. Setting it to true designates the Dataverse instance to serve as the dedicated polling server. It is needed so that the new framework can be used in a multi-node installation. + .. _feature-flags: Feature Flags @@ -3294,6 +3301,9 @@ please find all known feature flags below. Any of these flags can be activated u * - disable-return-to-author-reason - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. - ``Off`` + * - globus-use-experimental-async-framework + - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Note that the JVM option ``dataverse.globus.taskMonitoringServer`` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable From 007d7150763b55db95c35d3007900cfe46d0f50d Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 20 Aug 2024 09:47:41 -0400 Subject: [PATCH 162/360] Added a few more doc notes. #10623 --- doc/release-notes/10623-globus-improvements.md | 1 + doc/sphinx-guides/source/developers/big-data-support.rst | 2 ++ doc/sphinx-guides/source/developers/globus-api.rst | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 doc/release-notes/10623-globus-improvements.md diff --git a/doc/release-notes/10623-globus-improvements.md b/doc/release-notes/10623-globus-improvements.md new file mode 100644 index 00000000000..03579b59631 --- /dev/null +++ b/doc/release-notes/10623-globus-improvements.md @@ -0,0 +1 @@ +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See the `globus-use-experimental-async-framework` feature flag in the Configuration guide. \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 4aaed10512e..99ea4d2e0ba 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -187,3 +187,5 @@ As described in that document, Globus transfers can be initiated by choosing the An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark. See also :ref:`Globus settings <:GlobusSettings>`. + +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst index 902fc9db2ee..0f16dc704ef 100644 --- a/doc/sphinx-guides/source/developers/globus-api.rst +++ b/doc/sphinx-guides/source/developers/globus-api.rst @@ -185,6 +185,8 @@ As the transfer can take significant time and the API call is asynchronous, the Once the transfer completes, Dataverse will remove the write permission for the principal. +An alternative, experimental implementation of Globus polling of ongoing upload transfers has been added in v6.4. This new framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. Due to its experimental nature it is not enabled by default. See the ``globus-use-experimental-async-framework`` feature flag and the JVM option ``dataverse.globus.taskMonitoringServer`` described in the Configuration guide. + Note that when using a managed endpoint that uses the Globus S3 Connector, the checksum should be correct as Dataverse can validate it. For file-based endpoints, the checksum should be included if available but Dataverse cannot verify it. In the remote/reference case, where there is no transfer to monitor, the standard /addFiles API call (see :ref:`direct-add-to-dataset-api`) is used instead. There are no changes for the Globus case. From 6fcb2856f15ff88b9b0ae4f82dd3caf40b5cb2c3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 20 Aug 2024 10:47:55 -0400 Subject: [PATCH 163/360] typo #10623 --- src/main/webapp/dataverseuser.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index d061348ad87..bcb71733716 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -374,7 +374,7 @@ - + From f6882df1ea49acb322cd5091a6bf1f6d8e075cfd Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 20 Aug 2024 11:17:14 -0400 Subject: [PATCH 164/360] cut-and-paste error #10623 --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 01bf1419a52..731eef71c57 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3259,7 +3259,7 @@ The email for your institution that you'd like to appear in bag-info.txt. See :r Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_EMAIL``. -.. _dataverse.bagit.sourceorg.email: +.. _dataverse.globus.taskMonitoringServer: dataverse.globus.taskMonitoringServer +++++++++++++++++++++++++++++++++++++ From 4ae3ee6e4c268fa5911bff8e05377d85f45e5a94 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 20 Aug 2024 11:31:10 -0400 Subject: [PATCH 165/360] (#10623) --- .../java/edu/harvard/iq/dataverse/settings/FeatureFlags.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 746e6e3b75d..c8983b80df1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -92,8 +92,7 @@ public enum FeatureFlags { */ DISABLE_RETURN_TO_AUTHOR_REASON("disable-return-to-author-reason"), /** - * TEMPORARY feature flag for the new Globus upload framework (will only be - * used for testing). + * Feature flag for the new Globus upload framework. */ GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK("globus-use-experimental-async-framework"), ; From 9cf4e1b90888cf0d78f3659c2098bf142b5192a0 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 18:26:24 -0400 Subject: [PATCH 166/360] some minor cleanup changes #10623 --- .../ExternalFileUploadInProgress.java | 4 ---- .../datasetutility/AddReplaceFileHelper.java | 4 ++-- .../impl/UpdateDatasetVersionCommand.java | 22 +------------------ .../globus/GlobusTaskInProgress.java | 4 ---- 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java index ab6a1798307..96bfd3f63f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalFileUploadInProgress.java @@ -1,7 +1,3 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package edu.harvard.iq.dataverse; import jakarta.persistence.Column; diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 336fa9b5b7a..8e7c07b80bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -2139,7 +2139,7 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) { logger.log(Level.WARNING, "Dataset not locked for EditInProgress "); } else { datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.log(Level.INFO, "Removed EditInProgress lock "+eipLock.getId()); + logger.log(Level.FINE, "Removed EditInProgress lock"); } try { @@ -2306,7 +2306,7 @@ public Response replaceFiles(String jsonData, Dataset ds, User authUser) { logger.warning("Dataset not locked for EditInProgress "); } else { datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); - logger.info("Removed EditInProgress lock "); + logger.fine("Removed EditInProgress lock "); } try { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index faf8884b08d..cad61000e69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -103,9 +103,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { } Dataset theDataset = getDataset(); - //ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); - // this is an experiment (probably temporary) - checkUpdateDatasetVersionLock(ctxt); + ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); Dataset savedDataset = null; @@ -302,22 +300,4 @@ public boolean onSuccess(CommandContext ctxt, Object r) { return true; } - private void checkUpdateDatasetVersionLock(CommandContext ctxt) throws IllegalCommandException { - List locks = ctxt.datasets().getLocksByDatasetId(getDataset().getId()); - //locks.forEach(lock -> { - for (DatasetLock lock : locks) { - // Ingest lock is ok: - if (DatasetLock.Reason.Ingest != lock.getReason()) { - // with Workflow lock *some* users can edit; - // any other kind of lock - nope - if (DatasetLock.Reason.Workflow != lock.getReason() - || !ctxt.permissions().isMatchingWorkflowLock(getDataset(), - getUser().getIdentifier(), - getRequest().getWFInvocationId())) { - throw new IllegalCommandException( - BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), this); - } - } - } - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java index d3d06d38151..0a56b4933a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -1,7 +1,3 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package edu.harvard.iq.dataverse.globus; import edu.harvard.iq.dataverse.Dataset; From 45fb938acb3fcd027920d8f5308a026713dc942c Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 18:30:49 -0400 Subject: [PATCH 167/360] cosmetic #10623 --- .../engine/command/impl/UpdateDatasetVersionCommand.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index cad61000e69..e04cae13b35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -14,7 +14,6 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DatasetFieldUtil; import edu.harvard.iq.dataverse.util.FileMetadataUtil; @@ -104,7 +103,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { Dataset theDataset = getDataset(); ctxt.permissions().checkUpdateDatasetVersionLock(theDataset, getRequest(), this); - Dataset savedDataset = null; DatasetVersion persistedVersion = clone; From b3f79fe3102a308df1c8354465adb1f3b9bd2244 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 18:40:39 -0400 Subject: [PATCH 168/360] extra L in SUCCESSFUL (#10623) --- src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java | 2 +- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 2 +- .../java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java index a2faaf3637c..15114085c21 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java @@ -20,5 +20,5 @@ private ApiConstants() { // addFiles call public static final String API_ADD_FILES_COUNT_PROCESSED = "Total number of files"; - public static final String API_ADD_FILES_COUNT_SUCCESSFULL = "Number of files successfully added"; + public static final String API_ADD_FILES_COUNT_SUCCESSFUL = "Number of files successfully added"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 8e7c07b80bb..a470f08f736 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -2168,7 +2168,7 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) { JsonObjectBuilder result = Json.createObjectBuilder() .add(ApiConstants.API_ADD_FILES_COUNT_PROCESSED, totalNumberofFiles) - .add(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFULL, successNumberofFiles); + .add(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFUL, successNumberofFiles); return Response.ok().entity(Json.createObjectBuilder() diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 4bb478d26ed..ae84cad545c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -1052,7 +1052,7 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut if (addFilesJsonObject.containsKey("data") && addFilesJsonObject.getJsonObject("data").containsKey("Result")) { //Integer countAddFilesTotal = addFilesJsonObject.getJsonObject("data").getJsonObject("Result").getInt(ApiConstants.API_ADD_FILES_COUNT_PROCESSED, -1); - countAddFilesSuccess = addFilesJsonObject.getJsonObject("data").getJsonObject("Result").getInt(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFULL, -1); + countAddFilesSuccess = addFilesJsonObject.getJsonObject("data").getJsonObject("Result").getInt(ApiConstants.API_ADD_FILES_COUNT_SUCCESSFUL, -1); myLogger.info("Files successfully added by addFiles(): " + countAddFilesSuccess); } else { From 1acae684c31542591ba9e407e8c50cb80de1993a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 18:58:08 -0400 Subject: [PATCH 169/360] better Globus service availability checks #10623 --- .../harvard/iq/dataverse/api/Datasets.java | 22 ++++++++++++++----- src/main/java/propertyFiles/Bundle.properties | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4c547f5295f..aa6989d365a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3908,7 +3908,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP if (!systemConfig.isGlobusUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, - BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled")); + BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled")); } // ------------------------------------- @@ -4008,11 +4008,6 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, logger.info(" ==== (api addGlobusFilesToDataset) jsonData ====== " + jsonData); - if (!systemConfig.isHTTPUpload()) { - // @todo why isHTTPUpload()? - shouldn't it be checking isGlobusUpload() here? - return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); - } - // ------------------------------------- // (1) Get the user from the API key // ------------------------------------- @@ -4035,6 +4030,21 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, return wr.getResponse(); } + // Is Globus upload service available? + + // ... on this Dataverse instance? + if (!systemConfig.isGlobusUpload()) { + return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.globusUploadDisabled")); + } + + // ... and on this specific Dataset? + String storeId = dataset.getEffectiveStorageDriverId(); + // acceptsGlobusTransfers should only be true for an S3 or globus store + if (!GlobusAccessibleStore.acceptsGlobusTransfers(storeId) + && !GlobusAccessibleStore.allowsGlobusReferences(storeId)) { + return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled")); + } + // @todo check if the dataset is already locked! JsonObject jsonObject = null; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index e0488f95e2c..25ae8ad00c0 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1765,6 +1765,7 @@ file.fromWebloaderAfterCreate.tip=An option to upload a folder of files will be file.fromWebloader=Upload a Folder file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse. +file.api.globusUploadDisabled=File upload via Globus is not available for this installation of Dataverse. file.api.alreadyHasPackageFile=File upload via HTTP disabled since this dataset already contains a package file. file.replace.original=Original File file.editFiles=Edit Files From 5ba28883e2cdce99da768ebaa07ddd055f1b1229 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 19:03:18 -0400 Subject: [PATCH 170/360] better Globus service availability checks #10623 --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index aa6989d365a..ac5b9147dff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3908,7 +3908,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP if (!systemConfig.isGlobusUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, - BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled")); + BundleUtil.getStringFromBundle("file.api.globusUploadDisabled")); } // ------------------------------------- From 2512eab74e82284a4eae6fe29c172a869dbb64a8 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 19:12:43 -0400 Subject: [PATCH 171/360] removed an unnecessary @todo (#10623) --- .../edu/harvard/iq/dataverse/globus/GlobusServiceBean.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index ae84cad545c..6d3138856f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -687,19 +687,16 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + dataset.getId() + "_" + logTimestamp + ".log"; FileHandler fileHandler; - boolean fileHandlerSuceeded = false; + try { fileHandler = new FileHandler(logFileName); globusLogger.setUseParentHandlers(false); - fileHandlerSuceeded = true; } catch (IOException | SecurityException ex) { Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.SEVERE, null, ex); - //return; // @todo I don't think we need to return here? fileHandler = null; - } - if (fileHandlerSuceeded) { + if (fileHandler != null) { globusLogger.addHandler(fileHandler); } else { globusLogger = logger; From 6b06d9472326b28c3d47e549e14e5ee2af7e6bd3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 21 Aug 2024 19:20:22 -0400 Subject: [PATCH 172/360] cosmetic #10623 --- .../edu/harvard/iq/dataverse/globus/GlobusServiceBean.java | 2 -- .../iq/dataverse/globus/TaskMonitoringServiceBean.java | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 6d3138856f2..133da0a1cab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -810,8 +810,6 @@ private void processCompletedUploadTask(Dataset dataset, // Keeping a lock through the add datafiles API call avoids a conflicting edit // and keeps any open dataset page refreshing until the datafile appears. - // @todo is it necessary? what is wrong exactly with keeping the Globus - // lock on for the duration of the process? if (taskSuccess) { myLogger.info("Finished upload via Globus job."); diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index c956831317c..5dd4ce312b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -1,7 +1,3 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package edu.harvard.iq.dataverse.globus; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; From 6d06927169f0c7780ba02791ce3b77d90b9121b3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 22 Aug 2024 10:54:06 -0400 Subject: [PATCH 173/360] more changes per feedback. (saving the api token in the GlobusTask entity was a BAD idea!) #10623 --- .../harvard/iq/dataverse/api/Datasets.java | 6 +--- .../dataverse/globus/GlobusServiceBean.java | 9 +++--- .../globus/GlobusTaskInProgress.java | 29 +++++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ac5b9147dff..960cc408ee5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -4077,19 +4077,15 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId()); } - - ApiToken token = authSvc.findApiTokenByUser(authUser); - if(uriInfo != null) { logger.info(" ==== (api uriInfo.getRequestUri()) jsonData ====== " + uriInfo.getRequestUri().toString()); } - String requestUrl = SystemConfig.getDataverseSiteUrlStatic(); // Async Call try { - globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser); + globusService.globusUpload(jsonObject, dataset, requestUrl, authUser); } catch (IllegalArgumentException ex) { return badRequest("Invalid parameters: "+ex.getMessage()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 133da0a1cab..ac3c81622fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -667,7 +667,7 @@ private String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List< @Asynchronous @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, String httpRequestUrl, + public void globusUpload(JsonObject jsonData, Dataset dataset, String httpRequestUrl, AuthenticatedUser authUser) throws IllegalArgumentException, ExecutionException, InterruptedException, MalformedURLException { // Before we do anything else, let's do some basic validation of what @@ -728,7 +728,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S // Save the task information in the database so that the Globus monitoring // service can continue checking on its progress. - GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), token.getTokenString(), ruleId, new Timestamp(startDate.getTime())); + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, GlobusTaskInProgress.TaskType.UPLOAD, dataset, endpoint.getClientToken(), authUser, ruleId, new Timestamp(startDate.getTime())); em.persist(taskInProgress); // Save the metadata entries that define the files that are being uploaded @@ -1443,10 +1443,9 @@ public List findExternalUploadsByTaskId(String tas public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess, String taskStatus, Logger taskLogger) { String ruleId = globusTask.getRuleId(); Dataset dataset = globusTask.getDataset(); - AuthenticatedUser authUser = authSvc.lookupUser(globusTask.getApiToken()); + AuthenticatedUser authUser = globusTask.getLocalUser(); if (authUser == null) { - // @todo log error message; do nothing - // (the fields in GlobusTaskInProgress are not nullable though - ?) + // @todo log error message; do nothing return; } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java index 0a56b4933a1..8644bca6143 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.globus; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import jakarta.persistence.Column; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; @@ -14,6 +15,7 @@ import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; /** * @@ -67,24 +69,27 @@ public String toString() { } } - @Column + @Column(nullable=false) @Enumerated(EnumType.STRING) private TaskType taskType; /** * Globus API token that should be used to monitor the status of the task */ - @Column + @Column(nullable=false) private String globusToken; /** - * This is the Dataverse API token of the user who initiated the Globus task - */ - private String apiToken; + * This is the the user who initiated the Globus task + */ + @ManyToOne + @JoinColumn + private AuthenticatedUser user; - @Column + @Column(nullable=false) private String ruleId; + @JoinColumn(nullable = false) @ManyToOne private Dataset dataset; @@ -94,12 +99,12 @@ public String toString() { public GlobusTaskInProgress() { } - GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String globusToken, String apiToken, String ruleId, Timestamp startTime) { + GlobusTaskInProgress(String taskId, TaskType taskType, Dataset dataset, String globusToken, AuthenticatedUser authUser, String ruleId, Timestamp startTime) { this.taskId = taskId; this.taskType = taskType; this.dataset = dataset; this.globusToken = globusToken; - this.apiToken = apiToken; + this.user = authUser; this.ruleId = ruleId; this.startTime = startTime; } @@ -138,12 +143,12 @@ public void setGlobusToken(String clientToken) { this.globusToken = clientToken; } - public String getApiToken() { - return apiToken; + public AuthenticatedUser getLocalUser() { + return user; } - public void setApiToken(String apiToken) { - this.apiToken = apiToken; + public void setLocalUser(AuthenticatedUser authUser) { + this.user = authUser; } public String getRuleId() { From 69cfe2909deb099a201442d36101d6686204ad70 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 23 Aug 2024 12:40:58 -0400 Subject: [PATCH 174/360] changed the polling interval default in the new TaskMonitoringServiceBean #10623 --- doc/sphinx-guides/source/installation/config.rst | 2 +- .../iq/dataverse/globus/TaskMonitoringServiceBean.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 731eef71c57..fe438504f06 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4784,7 +4784,7 @@ The URL where the `dataverse-globus tasks = globusService.findAllOngoingTasks(); tasks.forEach(t -> { From d223a8f69166778b5015d7d9ad4f8e733ce22394 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 23 Aug 2024 13:31:02 -0400 Subject: [PATCH 175/360] more changes/refinements per review feedback (#10623) --- .../iq/dataverse/globus/TaskMonitoringServiceBean.java | 3 ++- .../harvard/iq/dataverse/ingest/IngestServiceBean.java | 4 ++-- .../edu/harvard/iq/dataverse/settings/JvmSettings.java | 1 + .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 9 --------- src/main/java/propertyFiles/Bundle.properties | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index 341714539cf..fdb2b222804 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.globus; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; import jakarta.annotation.PostConstruct; @@ -48,7 +49,7 @@ public class TaskMonitoringServiceBean { @PostConstruct public void init() { - if (systemConfig.isGlobusTaskMonitoringServer()) { + if (JvmSettings.GLOBUS_TASK_MONITORING_SERVER.lookupOptional(Boolean.class).orElse(false)) { logger.info("Starting Globus task monitoring service"); int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 600); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 3f76a319902..b42fd950528 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -345,8 +345,8 @@ public List saveAndAddFilesToDataset(DatasetVersion version, StorageIO dataAccess = DataAccess.getStorageIO(dataFile); //Populate metadata dataAccess.open(DataAccessOption.READ_ACCESS); - // (this will make a remote call to check if the file exists - // and obtain its size) + // (the .open() above makes a remote call to check if + // the file exists and obtains its size) confirmedFileSize = dataAccess.getSize(); // For directly-uploaded files, we will perform the file size diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 9d13be005c9..262d488acab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -51,6 +51,7 @@ public enum JvmSettings { DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"), + GLOBUS_TASK_MONITORING_SERVER(SCOPE_FILES, "globus-monitoring-server"), //STORAGE DRIVER SETTINGS SCOPE_DRIVER(SCOPE_FILES), diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 7417a5db4d4..c664d7d6730 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -82,7 +82,6 @@ public class SystemConfig { private String buildNumber = null; private static final String JVM_TIMER_SERVER_OPTION = "dataverse.timerServer"; - private static final String JVM_GLOBUS_TASK_MONITORING_OPTION = "dataverse.globus.taskMonitoringServer"; private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L; private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE = 3000000L; // 3 MB @@ -547,14 +546,6 @@ public boolean isTimerServer() { return false; } - public boolean isGlobusTaskMonitoringServer() { - String optionValue = System.getProperty(JVM_GLOBUS_TASK_MONITORING_OPTION); - if ("true".equalsIgnoreCase(optionValue)) { - return true; - } - return false; - } - public String getFooterCopyrightAndYear() { return BundleUtil.getStringFromBundle("footer.copyright", Arrays.asList(Year.now().getValue() + "")); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 25ae8ad00c0..d7e30c5e0e3 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -253,7 +253,7 @@ notification.mail.import.filesystem=Dataset {2} ({0}/dataset.xhtml?persistentId= notification.mail.globus.upload.completed=Globus transfer to Dataset {2} was successful. File(s) have been uploaded and verified.

{3}
notification.mail.globus.download.completed=Globus transfer of file(s) from the dataset {2} was successful.

{3}
notification.mail.globus.upload.completedWithErrors=Globus transfer to Dataset {2} is complete with errors.

{3}
-notification.mail.globus.upload.failedRemotely=Remote data transfer between Globus collections for Dataset {2} failed, reported via Globus API.

{3}
+notification.mail.globus.upload.failedRemotely=Remote data transfer between Globus endpoints for Dataset {2} failed, as reported via Globus API.

{3}
notification.mail.globus.upload.failedLocally=Dataverse received a confirmation of a successful Globus data transfer for Dataset {2}, but failed to add the files to the dataset locally.

{3}
notification.mail.globus.download.completedWithErrors=Globus transfer from the dataset {2} is complete with errors.

{3}
notification.import.filesystem=Dataset {1} has been successfully uploaded and verified. From 0ca5e621002711befaba23ba21208e2c19f08a92 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 23 Aug 2024 14:32:19 -0400 Subject: [PATCH 176/360] added an upfront locks check to the /addGlobusFiles api #10623 --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 960cc408ee5..47505ef3879 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -99,6 +99,7 @@ import java.util.stream.Collectors; import static edu.harvard.iq.dataverse.api.ApiConstants.*; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; @@ -4045,7 +4046,16 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled")); } - // @todo check if the dataset is already locked! + // Check if the dataset is already locked + // We are reusing the code and logic used by various command to determine + // if there are any locks on the dataset that would prevent the current + // users from modifying it: + try { + DataverseRequest dataverseRequest = createDataverseRequest(authUser); + permissionService.checkEditDatasetLock(dataset, dataverseRequest, new UpdateDatasetVersionCommand(dataset, dataverseRequest)); + } catch (IllegalCommandException icex) { + return error(Response.Status.FORBIDDEN, "Dataset " + datasetId + " is locked: " + icex.getLocalizedMessage()); + } JsonObject jsonObject = null; try { From 23d0f6c8db678c2f87e7fc2ff4bffabbc3b44d94 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 23 Aug 2024 14:44:26 -0400 Subject: [PATCH 177/360] added an upfront locks check to the /addGlobusFiles api #10623 --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 47505ef3879..d8e010940cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -4052,7 +4052,7 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, // users from modifying it: try { DataverseRequest dataverseRequest = createDataverseRequest(authUser); - permissionService.checkEditDatasetLock(dataset, dataverseRequest, new UpdateDatasetVersionCommand(dataset, dataverseRequest)); + permissionService.checkEditDatasetLock(dataset, dataverseRequest, null); } catch (IllegalCommandException icex) { return error(Response.Status.FORBIDDEN, "Dataset " + datasetId + " is locked: " + icex.getLocalizedMessage()); } From 50a9ede0037ae255b1f363881a8a0cb42e236746 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:12:40 +0200 Subject: [PATCH 178/360] ci(ct): temporarily allow other repos to run the base image workflow Needs to be reverted when done with testing --- .github/workflows/container_base_push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 660f9bdd861..4843800a3d1 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -26,8 +26,9 @@ jobs: permissions: contents: read packages: read + # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - if: ${{ github.repository_owner == 'IQSS' }} + #if: ${{ github.repository_owner == 'IQSS' }} outputs: branches: ${{ steps.matrix.outputs.branches }} steps: @@ -56,8 +57,9 @@ jobs: fail-fast: false matrix: branch: ${{ fromJson(needs.discover.outputs.branches) }} + # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' }} + #if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout repository From 19de76d77e72b8da9558365ee0510b714a465cb5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:33:29 +0200 Subject: [PATCH 179/360] ci(ct): temporarily disable remote docker builder in fork --- .github/workflows/container_base_push.yml | 55 ++++++++++++----------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 4843800a3d1..ca05a30c95f 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -102,33 +102,34 @@ jobs: platforms: ${{ env.PLATFORMS }} # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - - name: Setup SSH agent - if: ${{ github.event_name != 'schedule' }} - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - - name: Provide the known hosts key and the builder config - if: ${{ github.event_name != 'schedule' }} - run: | - echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts - mkdir -p modules/container-base/target/buildx-state/buildx/instances - cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF - { "Name": "maven", - "Driver": "docker-container", - "Dynamic": false, - "Nodes": [{"Name": "maven0", - "Endpoint": "unix:///var/run/docker.sock", - "Platforms": [{"os": "linux", "architecture": "amd64"}], - "DriverOpts": null, - "Flags": ["--allow-insecure-entitlement=network.host"], - "Files": null}, - {"Name": "maven1", - "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", - "Platforms": [{"os": "linux", "architecture": "arm64"}], - "DriverOpts": null, - "Flags": ["--allow-insecure-entitlement=network.host"], - "Files": null}]} - EOF + # TODO: re-enable for final PR + #- name: Setup SSH agent + # if: ${{ github.event_name != 'schedule' }} + # uses: webfactory/ssh-agent@v0.9.0 + # with: + # ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + #- name: Provide the known hosts key and the builder config + # if: ${{ github.event_name != 'schedule' }} + # run: | + # echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + # mkdir -p modules/container-base/target/buildx-state/buildx/instances + # cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + # { "Name": "maven", + # "Driver": "docker-container", + # "Dynamic": false, + # "Nodes": [{"Name": "maven0", + # "Endpoint": "unix:///var/run/docker.sock", + # "Platforms": [{"os": "linux", "architecture": "amd64"}], + # "DriverOpts": null, + # "Flags": ["--allow-insecure-entitlement=network.host"], + # "Files": null}, + # {"Name": "maven1", + # "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + # "Platforms": [{"os": "linux", "architecture": "arm64"}], + # "DriverOpts": null, + # "Flags": ["--allow-insecure-entitlement=network.host"], + # "Files": null}]} + # EOF - name: Add additional tags as options # TODO: remove the feature branch From a9a5709412b9640e1e595ca989a926b62090d3a9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 21:14:11 +0200 Subject: [PATCH 180/360] feat(ct): introduce backports for older base images This way we can apply important changes if need be. --- .github/workflows/container_base_push.yml | 17 ++++++++-- .../src/backports/v6.1/001-pom.xml.patch | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/001-pom.xml.patch diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ca05a30c95f..147e5e28d12 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -142,10 +142,23 @@ jobs: echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi + # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) + # and try to apply them if present + - name: Get backport patches + continue-on-error: true + uses: actions/checkout@v4 + with: + path: backports + sparse-checkout: | + modules/container-base/src/backports/${{ matrix.branch }} + - name: Apply backport patches + id: patch + run: | + find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ + xargs -0 patch -p1 -s -i + - name: Deploy multi-arch base container image to Docker Hub id: build - # Do not build for v6.0 and v6.1. We can simply reuse the one from v6.2. - if: ${{ matrix.branch != 'v6.0' && matrix.branch != 'v6.1' }} run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch new file mode 100644 index 00000000000..590eeae962a --- /dev/null +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.6.1 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-pyr${payara.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + From 0691d4d707c60818b3c285d398973ab35ac9b102 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:18:38 +0200 Subject: [PATCH 181/360] feat(ct): add patches for v6.1, 6.2 and 6.3 base images - Patching versions into the POM and it's tags. - Adding security fixes for 6.2 and 6.3 base image (missing user statement) - Adding labels to keep information about origins --- .../src/backports/v6.1/001-pom.xml.patch | 4 +-- .../src/backports/v6.1/002-Dockerfile.patch | 10 ++++++ .../src/backports/v6.2/001-pom.xml.patch | 33 +++++++++++++++++++ .../v6.2/002-Dockerfile.labels.patch | 10 ++++++ .../v6.2/003-Dockerfile.security.patch | 10 ++++++ .../src/backports/v6.3/001-pom.xml.patch | 33 +++++++++++++++++++ .../v6.3/002-Dockerfile.labels.patch | 18 ++++++++++ .../v6.3/003-Dockerfile.security.patch | 7 ++++ 8 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/002-Dockerfile.patch create mode 100644 modules/container-base/src/backports/v6.2/001-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch create mode 100644 modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch create mode 100644 modules/container-base/src/backports/v6.3/001-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch create mode 100644 modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 590eeae962a..4e1c672b521 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -4,7 +4,7 @@ io.gdcc container-base -+ 0.6.1 ++ 0.8.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -14,7 +14,7 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-pyr${payara.version} ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} + 0 + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy diff --git a/modules/container-base/src/backports/v6.1/002-Dockerfile.patch b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch new file mode 100644 index 00000000000..4bb7a1eac91 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -233,4 +233,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch new file mode 100644 index 00000000000..5a0fe404148 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.9.0 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..fbb7f80c4ce --- /dev/null +++ b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -242,4 +242,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch new file mode 100644 index 00000000000..1ab4c3a980a --- /dev/null +++ b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -226,6 +226,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara + + # Set the entrypoint to tini (as a process supervisor) + ENTRYPOINT ["/usr/bin/dumb-init", "--"] diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch new file mode 100644 index 00000000000..595a43beed4 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -0,0 +1,33 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -12,6 +12,7 @@ + + io.gdcc + container-base ++ 0.9.1 + ${packaging.type} + Container Base Image + This module provides an application server base image to be decorated with the Dataverse app. +@@ -40,8 +42,10 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} ++ 0 ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag}-r${base.image.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..c0bbe7698d8 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch @@ -0,0 +1,18 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -224,6 +224,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara + + # Set the entrypoint to tini (as a process supervisor) + ENTRYPOINT ["/usr/bin/dumb-init", "--"] +@@ -240,4 +241,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch new file mode 100644 index 00000000000..d8487b3aacc --- /dev/null +++ b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch @@ -0,0 +1,7 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -224,6 +224,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara From ae8ced2f3040b4c29648a3f9da321c25c189a1fa Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:25:18 +0200 Subject: [PATCH 182/360] ci,fix(ct): only apply 1 patch file at a time for base images Otherwise the patch is applied to some other patchfile, which makes no sense. --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 147e5e28d12..b0be1b72c89 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -155,7 +155,7 @@ jobs: id: patch run: | find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ - xargs -0 patch -p1 -s -i + xargs -0 -n1 patch -p1 -s -i - name: Deploy multi-arch base container image to Docker Hub id: build From 37fb155739096f5e4a178ab3a851db8033eb4d34 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:27:31 +0200 Subject: [PATCH 183/360] ci,fix(ct): correction of v6.3 Dockerfile patches Double patching makes CI trip over. --- .../src/backports/v6.3/002-Dockerfile.labels.patch | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch index c0bbe7698d8..c7744882153 100644 --- a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch +++ b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch @@ -1,13 +1,5 @@ --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile -@@ -224,6 +224,7 @@ USER root - RUN true && \ - chgrp -R 0 "${DOMAIN_DIR}" && \ - chmod -R g=u "${DOMAIN_DIR}" -+USER payara - - # Set the entrypoint to tini (as a process supervisor) - ENTRYPOINT ["/usr/bin/dumb-init", "--"] @@ -240,4 +241,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ From c177d9ab7e281e5b0accbc94767c4630ac87cf92 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 26 Aug 2024 23:42:01 +0200 Subject: [PATCH 184/360] ci,fix(ct): make v6.3 get alpha tag Was 6.2 - but we are at v6.3 already! --- .github/workflows/container_base_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b0be1b72c89..ee7cb85a42e 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -138,7 +138,7 @@ jobs: if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" - elif [[ "${{ matrix.branch }}" == "v6.2" ]]; then + elif [[ "${{ matrix.branch }}" == "v6.3" ]]; then echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" fi From fffcfe05374ea932fcf76112c328b77a9dcb2e95 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 Aug 2024 16:23:45 +0200 Subject: [PATCH 185/360] refactor(ct): align base POM tags with suitable policy across releases and develop --- modules/container-base/pom.xml | 13 ++++++++++--- .../src/backports/v6.1/001-pom.xml.patch | 10 ++++++---- .../src/backports/v6.2/001-pom.xml.patch | 10 ++++++---- .../src/backports/v6.3/001-pom.xml.patch | 10 ++++++---- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 60be420bd9a..5de94bfc941 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -13,7 +13,7 @@ io.gdcc container-base - 1 + 1.0.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -42,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - rev${project.version} - eclipse-temurin:${target.java.version}-jre + ${project.version}-${java.image.flavor} + + SNAPSHOT + ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + noble 1000 1000 linux/amd64,linux/arm64 @@ -111,6 +115,9 @@ assembly.xml + + ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 4e1c672b521..508b496af14 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 5a0fe404148..256680c2595 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index 595a43beed4..b1ab9bfe589 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -8,14 +8,16 @@ ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,10 @@ +@@ -40,8 +42,12 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-pyr${payara.version}-tmrn${target.java.version} -+ 0 ++ ${project.version}-${java.image.flavor} ++ ++ SNAPSHOT ++ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 @@ -26,7 +28,7 @@ assembly.xml + -+ ${base.image.tag}-r${base.image.revision} ++ ${base.image.tag.revision} + From c5b9ccc3e370fa7c7d3e83be538a5c359de4b2e9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 15:53:51 +0200 Subject: [PATCH 186/360] docs(ct): document new tagging and release policy for base image --- .../source/container/base-image.rst | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 0005265fb1c..e364579b568 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -26,15 +26,26 @@ Supported Image Tags This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: - -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) - - +happens there (again, by the community). + +In our tagging, we `follow the Bitname example `_: + +- ``latest``: Rolling tag on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) + Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. + Usage is recommended for development purposes only. +- ``..--p-j``: + Rolling tag corresponding to functionality provided by the ``container-base`` module at a certain version. + Auto-released for changes on ``develop`` branch as well as scheduled maintenance for the *three* most recent Dataverse releases. + Semantic versioning applies to functionality of the base image only. + (`Dockerfile `__ or same location within tagged Dataverse releases) +- ``..--p-j-r``: + Immutable tag, otherwise same as above. + Revision is incremented for rebuilds of the base image. + Recommended for production builds and usage. + +**Note:** It is not possible to provide shorter rolling tags, as the source for a tag like ``x.y.z-jammy`` would not be unambiguous. +If the version number does not change between releases and/or the ``develop`` branch, one is going to win over another image. +If at the same time Payara or Java version change (which is independent from the module version!), it becomes unclear which version people end up using. Image Contents ++++++++++++++ From c8f95a0e008d8e004dd2b25f3f450083968b73ef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 15:59:49 +0200 Subject: [PATCH 187/360] refactor(ct): align base image tagging with documented policy --- modules/container-base/pom.xml | 4 ++-- modules/container-base/src/backports/v6.1/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.2/001-pom.xml.patch | 4 ++-- modules/container-base/src/backports/v6.3/001-pom.xml.patch | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 5de94bfc941..8e5c476c64a 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -42,10 +42,10 @@ docker-build gdcc/base:${base.image.tag} - ${project.version}-${java.image.flavor} + ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} SNAPSHOT - ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} + ${base.image.tag}-r${base.image.revision} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index 508b496af14..a61f6b6fb14 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 256680c2595..90fccffe53d 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index b1ab9bfe589..83e151b6656 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -14,10 +14,10 @@ gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor} ++ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + + SNAPSHOT -+ ${base.image.tag}-pyr${payara.version}-tmrn${target.java.version}-r${base.image.revision} ++ ${base.image.tag}-r${base.image.revision} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 From 136021316726bcfe2ff3c7ba48ad008f24973854 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 28 Aug 2024 18:03:25 +0200 Subject: [PATCH 188/360] ci(ct): split up push releases and maintenance workflows --- .github/workflows/container_base_push.yml | 107 +++----------- .github/workflows/container_maintenance.yml | 152 ++++++++++++++++++++ 2 files changed, 171 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/container_maintenance.yml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index ee7cb85a42e..f0e1b9d9f14 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,5 +1,5 @@ --- -name: Base Container Image +name: Container Images Releasing on: push: @@ -7,56 +7,23 @@ on: - 'v[6-9].**' branches: - 'develop' - - '10478-version-base-img' + # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' - schedule: - - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC env: PLATFORMS: linux/amd64,linux/arm64 - NUM_PAST_RELEASES: 3 + DEVELOPMENT_BRANCH: develop jobs: - discover: - name: Discover Release Matrix - runs-on: ubuntu-latest - permissions: - contents: read - packages: read - # TODO: re-enable for final PR - # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled - #if: ${{ github.repository_owner == 'IQSS' }} - outputs: - branches: ${{ steps.matrix.outputs.branches }} - steps: - - name: Build branch matrix options - id: matrix - # TODO: remove the feature branch and re-enable the if/else! - run: | - # Get last three releases and include develop branch as matrix elements - #if [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ - jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "develop", "10478-version-base-img" ]')" | tr -d "\n" | tr -s " " | \ - tee -a "$GITHUB_OUTPUT" - #else - # # Note: github.ref_name will be the name of the branch or the tag pushed - # echo "branches=['${{ github.ref_name }}']" | tee -a "$GITHUB_OUTPUT" - #fi - build: name: Build image runs-on: ubuntu-latest permissions: contents: read packages: read - needs: discover - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks #if: ${{ github.repository_owner == 'IQSS' }} @@ -64,8 +31,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - with: - ref: ${{ matrix.branch }} - name: Determine Java version from Parent POM run: | echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} @@ -94,12 +59,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # In case of scheduled maintenance, we don't care about buildtime: use QEMU for AMD64 + ARM64 - - name: Set up QEMU for multi-arch builds - if: ${{ github.event_name == 'schedule' }} - uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ env.PLATFORMS }} # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. # TODO: re-enable for final PR @@ -131,52 +90,17 @@ jobs: # "Files": null}]} # EOF - - name: Add additional tags as options - # TODO: remove the feature branch - run: | - # For the development branch, update the latest tag in addition - if [[ "${{ matrix.branch }}" == "develop" || "${{ matrix.branch }}" == "10478-version-base-img" ]]; then - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - # In case of releases <=6.2, we still need to provide backward compatible names "alpha" and "unstable" - elif [[ "${{ matrix.branch }}" == "v6.3" ]]; then - echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.additional=alpha" | tee -a "${GITHUB_ENV}" - fi + # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) - # and try to apply them if present - - name: Get backport patches - continue-on-error: true - uses: actions/checkout@v4 - with: - path: backports - sparse-checkout: | - modules/container-base/src/backports/${{ matrix.branch }} - - name: Apply backport patches - id: patch + - name: Configure update of "latest" tag for development branch + if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | - find ${GITHUB_WORKSPACE}/backports/modules/container-base/src/backports/${{ matrix.branch }} -type f -name '*.patch' -print0 | \ - xargs -0 -n1 patch -p1 -s -i + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} - - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} - name: Push description to DockerHub - uses: peter-evans/dockerhub-description@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/base - short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md - - # - if: always() - # name: Save status (workaround for matrix outputs) - # run: | - # # steps.build.outcome is the status BEFORE continue-on-error - # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - push-app-img: name: "Rebase & Publish App Image" permissions: @@ -187,11 +111,18 @@ jobs: needs: - discover - build - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} uses: ./.github/workflows/container_app_push.yml with: - branch: ${{ matrix.branch }} + branch: ${{ github.ref_name }} + + # TODO: job to update the docker hub description with supported tags and all + #- if: ${{ github.event_name == 'push' && github.ref_name == env.DEVELOPMENT_BRANCH }} + # name: Push description to DockerHub + # uses: peter-evans/dockerhub-description@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + # repository: gdcc/base + # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + # readme-filepath: ./modules/container-base/README.md diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml new file mode 100644 index 00000000000..8f7efa4300d --- /dev/null +++ b/.github/workflows/container_maintenance.yml @@ -0,0 +1,152 @@ +--- +name: Container Images Scheduled Maintenance + +on: + # TODO: remove "push" in final PR + push: + branches: + - '10478-version-base-img' + paths: + - 'modules/container-base/**' + - 'modules/dataverse-parent/pom.xml' + - '.github/workflows/container_maintenance.yml' + # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) + workflow_dispatch: + schedule: + - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + +env: + PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 + # TODO: change to "develop" in final PR + DEVELOP_BRANCH: 10478-version-base-img + +jobs: + discover: + name: Discover Release Matrix + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # TODO: re-enable for final PR + # Only run in upstream repo - avoid unnecessary runs in forks and only for scheduled + #if: ${{ github.repository_owner == 'IQSS' }} + outputs: + branches: ${{ steps.matrix.outputs.branches }} + current_release: ${{ steps.matrix.outputs.current_release }} + steps: + - name: Build branch matrix options + id: matrix + run: | + echo "branches=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | \ + jq '[ .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, "${{ env.DEVELOP_BRANCH }}" ]')" | tr -d "\n" | tr -s " " | \ + tee -a "$GITHUB_OUTPUT" + echo "current_release=$(curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq '.[0].tag_name' )" | tee -a "$GITHUB_OUTPUT" + + build: + name: Build image + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: discover + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} + # TODO: re-enable for final PR + # Only run in upstream repo - avoid unnecessary runs in forks + #if: ${{ github.repository_owner == 'IQSS' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} + - name: Determine Java version from Parent POM + run: | + echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} + + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: | + modules/container-base/pom.xml + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + + # Try to retrieve backport patches for this git ref (but don't fail if there aren't any) + # and try to apply them if present + - name: Get and apply backported patches + # There might be no patches - ignore errors + continue-on-error: true + run: | + mkdir -p "${GITHUB_WORKSPACE}/patches" + curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ + tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 + find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + - name: Configure update of "latest" tag for development branch + if: ${{ matrix.branch == env.DEVELOP_BRANCH }} + run: | + echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" + + # TODO: figure out if rebuild is necessary + # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below + + - name: Deploy multi-arch base container image to Docker Hub + id: build + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + + # - if: always() + # name: Save status (workaround for matrix outputs) + # run: | + # # steps.build.outcome is the status BEFORE continue-on-error + # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + + push-app-img: + name: "Rebase & Publish App Image" + permissions: + contents: read + packages: write + pull-requests: write + secrets: inherit + needs: + - discover + - build + strategy: + fail-fast: false + matrix: + branch: ${{ fromJson(needs.discover.outputs.branches) }} + uses: ./.github/workflows/container_app_push.yml + with: + branch: ${{ matrix.branch }} + + # TODO: job to update the docker hub description with supported tags and all + # - name: Push description to DockerHub + # uses: peter-evans/dockerhub-description@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + # repository: gdcc/base + # short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + # readme-filepath: ./modules/container-base/README.md From 716e2899656f03b64395057b8e193f40b8cc485b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 11:14:13 +0200 Subject: [PATCH 189/360] ci(ct): exclude app image rebuilds for easier testing --- .github/workflows/container_maintenance.yml | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 8f7efa4300d..4e329cb9fe7 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -123,23 +123,23 @@ jobs: # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" - push-app-img: - name: "Rebase & Publish App Image" - permissions: - contents: read - packages: write - pull-requests: write - secrets: inherit - needs: - - discover - - build - strategy: - fail-fast: false - matrix: - branch: ${{ fromJson(needs.discover.outputs.branches) }} - uses: ./.github/workflows/container_app_push.yml - with: - branch: ${{ matrix.branch }} + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - discover + # - build + # strategy: + # fail-fast: false + # matrix: + # branch: ${{ fromJson(needs.discover.outputs.branches) }} + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ matrix.branch }} # TODO: job to update the docker hub description with supported tags and all # - name: Push description to DockerHub From d6339a9ae26594b215aad1a6b6e236b354e71f34 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 11:08:03 +0200 Subject: [PATCH 190/360] ci(ct): detect necessary rebuilds and calculate revision number --- .github/workflows/container_maintenance.yml | 76 +++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 4e329cb9fe7..0323c1a7227 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -105,17 +105,83 @@ jobs: curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + + # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates + - name: Check for recent Temurin image updates + id: temurin-check + run: | + JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" + JAVA_IMAGE_NS="library" + JAVA_IMAGE_REPO="$( echo "$JAVA_IMAGE" | cut -f1 -d: )" + JAVA_IMAGE_TAG="$( echo "$JAVA_IMAGE" | cut -f2 -d: )" + JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" + exit 1 + fi + + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" + BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" + BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" + BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then + echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" + echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" + else + echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" + echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" + fi + # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates + - name: Check for package updates in base image + id: package-check + if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + echo "Base image $BASE_IMAGE needs package updates" + echo "newer_packages=true" >> "${GITHUB_OUTPUT}" + else + echo "Base image $BASE_IMAGE has no package updates" + echo "newer_packages=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Calculate revision number for immutable tag + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + BASE_IMAGE_NS_REPO="$( echo "$BASE_IMAGE" | cut -d: -f1 )" + BASE_IMAGE_TAG=""$( echo "$BASE_IMAGE" | cut -d: -f2 )"" + + function get_all_tags() { + ref="$1" + case "$ref" in + */*) :;; # namespace/repository syntax, leave as is + *) ref="library/$ref";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ref}:pull" ) + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + curl -sS -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/${ref}/tags/?page=$i&page_size=100" | jq -r '."results"[]["name"]' 2>/dev/null + done + } + CURRENT=$( get_all_tags "${BASE_IMAGE_NS_REPO}" | grep "${BASE_IMAGE_TAG}-r" | sed -e "s#${BASE_IMAGE_TAG}-r##" | sort -h | tail -n1 ) + + # If there is a current number, increment it - otherwise this is the initial version, set to 0 + if [[ "$CURRENT" ]]; then + echo "REVISION_OPTION=-Dbase.image.revision=$((CURRENT+1))" | tee -a "${GITHUB_ENV}" + else + echo "REVISION_OPTION=-Dbase.image.revision=0" | tee -a "${GITHUB_ENV}" + fi - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - - # TODO: figure out if rebuild is necessary - # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - - name: Deploy multi-arch base container image to Docker Hub + if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} # - if: always() # name: Save status (workaround for matrix outputs) From 289f352ed11db894ecb52f5f3f34592e5a700177 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:36:25 +0200 Subject: [PATCH 191/360] build(ct): update base image with wait4x 2.14.1 Hopefully will clear lots of reported security vulnerabilities --- modules/container-base/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 00deb085ec9..e3717928285 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.0" +ARG WAIT4X_VERSION="v2.14.1" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching From 3a771f96726c98b66ada0c5eeb8020b56be36364 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:40:24 +0200 Subject: [PATCH 192/360] ci(ct): add option to manual trigger of maintenance workflow to force build --- .github/workflows/container_maintenance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 0323c1a7227..f4a91aea562 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -12,6 +12,12 @@ on: - '.github/workflows/container_maintenance.yml' # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: + inputs: + force_build: + type: boolean + required: false + default: false + description: "Build and deploy even if no newer Java images or package updates are found." schedule: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC @@ -179,7 +185,7 @@ jobs: run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' }} + if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} From 899bba62bb5aa753e4be876079b69b7c03d64e7b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 12:43:51 +0200 Subject: [PATCH 193/360] ci(ct): no longer react to pushes in maintenance workflow Only use either scheduled or manual events to run the workflow. --- .github/workflows/container_maintenance.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index f4a91aea562..6c7b9d3e9ca 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -2,14 +2,6 @@ name: Container Images Scheduled Maintenance on: - # TODO: remove "push" in final PR - push: - branches: - - '10478-version-base-img' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_maintenance.yml' # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: inputs: From ed17240da719d8a8862be57299ec60cb34a73d04 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:33:16 +0200 Subject: [PATCH 194/360] ci(ct): extract revision logic into separate script Easier to test and reuse for different workflows --- .../workflows/scripts/get_next_revision.sh | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 .github/workflows/scripts/get_next_revision.sh diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh new file mode 100755 index 00000000000..5ec10ee2fb9 --- /dev/null +++ b/.github/workflows/scripts/get_next_revision.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# This script is used to retrieve the next revision number for a given image reference. +# The image reference must be of the form "[/]:", where the namespace will default to "library" if omitted. + +set -eu + +IMAGE=${1} +IMAGE_NS_REPO="$( echo "$IMAGE" | cut -d: -f1 )" +IMAGE_TAG="$( echo "$IMAGE" | cut -d: -f2 )" + +if [[ "$IMAGE_TAG" == "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 +fi + +case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax +esac + +# Without such a token we run into rate limits +token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + +ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done +)" + +# If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. +CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" ) +echo "$((CURRENT+1))" From 454fbe6c63cd9ea84c20970abcc2727c7fc401dc Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:09 +0200 Subject: [PATCH 195/360] ci(ct): add revision logic to base image release workflow on push --- .github/workflows/container_base_push.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index f0e1b9d9f14..fad436f0dda 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -90,8 +90,10 @@ jobs: # "Files": null}]} # EOF - # TODO: logic of retrieving the current reversion number, adding the property to the mvn call below - + - name: Calculate revision number for immutable tag + run: | + BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" + echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | @@ -99,7 +101,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} push-app-img: name: "Rebase & Publish App Image" From f1e3a3f0890a44e51f0417bd25433f8cbadbf7b1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:32 +0200 Subject: [PATCH 196/360] ci(ct): temporarily disable release of app image when rebuilding base --- .github/workflows/container_base_push.yml | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index fad436f0dda..5ce05c6f8e6 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -103,19 +103,19 @@ jobs: id: build run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} - push-app-img: - name: "Rebase & Publish App Image" - permissions: - contents: read - packages: write - pull-requests: write - secrets: inherit - needs: - - discover - - build - uses: ./.github/workflows/container_app_push.yml - with: - branch: ${{ github.ref_name }} + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - discover + # - build + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ github.ref_name }} # TODO: job to update the docker hub description with supported tags and all From 266c7bc29993086a6918be710baaa6f3975f7af2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:34:48 +0200 Subject: [PATCH 197/360] ci(ct): temporarily enable triggering release workflow from feature branch --- .github/workflows/container_base_push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 5ce05c6f8e6..532c64bd781 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -7,6 +7,8 @@ on: - 'v[6-9].**' branches: - 'develop' + # TODO: delete for final PR + - '10478-version-base-img' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' From c04d9e406716b02265894f2b9b5b0f3f47242046 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:35:25 +0200 Subject: [PATCH 198/360] ci(ct): use script for revision logic in maintenance flow --- .github/workflows/container_maintenance.yml | 29 ++++----------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 6c7b9d3e9ca..2e668af5162 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -147,31 +147,12 @@ jobs: - name: Calculate revision number for immutable tag run: | + # Get the revision logic script (not present on older releases) + curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ + tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 + # Now get the new revision number BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" - BASE_IMAGE_NS_REPO="$( echo "$BASE_IMAGE" | cut -d: -f1 )" - BASE_IMAGE_TAG=""$( echo "$BASE_IMAGE" | cut -d: -f2 )"" - - function get_all_tags() { - ref="$1" - case "$ref" in - */*) :;; # namespace/repository syntax, leave as is - *) ref="library/$ref";; # bare repository name (docker official image); must convert to namespace/repository syntax - esac - token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ref}:pull" ) - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - curl -sS -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/${ref}/tags/?page=$i&page_size=100" | jq -r '."results"[]["name"]' 2>/dev/null - done - } - CURRENT=$( get_all_tags "${BASE_IMAGE_NS_REPO}" | grep "${BASE_IMAGE_TAG}-r" | sed -e "s#${BASE_IMAGE_TAG}-r##" | sort -h | tail -n1 ) - - # If there is a current number, increment it - otherwise this is the initial version, set to 0 - if [[ "$CURRENT" ]]; then - echo "REVISION_OPTION=-Dbase.image.revision=$((CURRENT+1))" | tee -a "${GITHUB_ENV}" - else - echo "REVISION_OPTION=-Dbase.image.revision=0" | tee -a "${GITHUB_ENV}" - fi + echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | From 2efd1b534803bb04f7e171548ba3f5561db0886d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 29 Aug 2024 13:52:46 +0200 Subject: [PATCH 199/360] ci(ct): fix revision logic to return 0 when rolling tag not present before --- .github/workflows/scripts/get_next_revision.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh index 5ec10ee2fb9..631f8b6f513 100755 --- a/.github/workflows/scripts/get_next_revision.sh +++ b/.github/workflows/scripts/get_next_revision.sh @@ -45,5 +45,9 @@ ALL_TAGS="$( )" # If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. -CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" ) -echo "$((CURRENT+1))" +CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 ) +if [[ "$CURRENT" ]]; then + echo "$((CURRENT+1))" +else + echo "0" +fi From b61395ae57f02587ce6cc4de15ad46cf6ed72a42 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:11:20 +0200 Subject: [PATCH 200/360] build,doc(parent): add special base image versioning As long as we are developing the next version of Dataverse, we don't want the develop branch to fight the container images updated by the maintenance workflow for the past releases. To avoid this, we simply use the next release version for the base image we use during development. It requires updating a property during the release process twice, as we need to set the version to the "main" version property ($revision) once we aim for creating the tag. This way, when using the release tag in forks or other ways as a foundation, people end up with the right base image. After the tag is done, we need to prepare using the virtual next version within the develop branch again. --- .../source/developers/making-releases.rst | 10 ++++++++ modules/dataverse-parent/pom.xml | 24 ++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index e436ba9e9d2..a403f249beb 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -46,6 +46,7 @@ Bump Version Numbers Increment the version number to the milestone (e.g. 5.10.1) in the following two files: - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${revision}`` - doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) Add the version being released to the lists in the following file: @@ -213,6 +214,15 @@ Now that we've published the release, close the milestone and create a new one. Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). +Update the Container Base Image Version Property +------------------------------------------------ + +Create a new branch ``prepare-next-iteration`` and update the following files to prepare for the next development cycle: + +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` as ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` + +Now create a pull request and merge it - there is no review or QA necessary, as it is a predefined necessity. + Add the Release to the Dataverse Roadmap ---------------------------------------- diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 281aa274132..3df06281547 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -449,10 +449,13 @@ ct - - + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + @@ -478,6 +481,21 @@ false + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + parse-version + + parse-version + + initialize + + + From f95476a4d9baa1c3386ac068583732a037c4fd58 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:17:48 +0200 Subject: [PATCH 201/360] build(ct,parent): adapt base image POM to new tagging policy We will use a long tag with Payara and Java version in it during development. This way we auto-update images on dev's PCs on these breaking changes. For any releases, we will use short tags using only the version and the OS flavor. These rolling tags will be accompanied by the revisioned tags. These extend the base name with the suffix as before, but only for releases, not the development branch. As the development branch is a rolling target, immutable tags make little sense. This commit also adds the necessary backporting patches to older releases for the parent module to define the base.image.version as $revision, as described in the extended release process. --- modules/container-base/pom.xml | 12 ++++-------- .../src/backports/v6.1/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.1/003-parent-pom.xml.patch | 11 +++++++++++ .../src/backports/v6.2/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.2/004-parent-pom.xml.patch | 11 +++++++++++ .../src/backports/v6.3/001-pom.xml.patch | 17 ++++------------- .../src/backports/v6.3/004-parent-pom.xml.patch | 11 +++++++++++ 7 files changed, 49 insertions(+), 47 deletions(-) create mode 100644 modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch create mode 100644 modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index 8e5c476c64a..e707c6a69c8 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -13,7 +13,6 @@ io.gdcc container-base - 1.0.0 ${packaging.type} Container Base Image This module provides an application server base image to be decorated with the Dataverse app. @@ -41,11 +40,11 @@ ct docker-build + gdcc/base:${base.image.tag} - ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} - - SNAPSHOT - ${base.image.tag}-r${base.image.revision} + + ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} + ${base.image.version}-${java.image.flavor} eclipse-temurin:${target.java.version}-jre-${java.image.flavor} noble 1000 @@ -115,9 +114,6 @@ assembly.xml - - ${base.image.tag.revision} - diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch index a61f6b6fb14..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.1/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.8.0 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch index 90fccffe53d..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.2/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.9.0 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch index 83e151b6656..ef0fbdda9fc 100644 --- a/modules/container-base/src/backports/v6.3/001-pom.xml.patch +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -1,23 +1,14 @@ --- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 +++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 -@@ -12,6 +12,7 @@ - - io.gdcc - container-base -+ 0.9.1 - ${packaging.type} - Container Base Image - This module provides an application server base image to be decorated with the Dataverse app. -@@ -40,8 +42,12 @@ +@@ -40,8 +42,11 @@ docker-build gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre -+ ${project.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} -+ -+ SNAPSHOT -+ ${base.image.tag}-r${base.image.revision} ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${java.image.flavor} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + jammy 1000 diff --git a/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + From a4354c0e05c4cc09057e924519fb70f313b65cee Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:20:24 +0200 Subject: [PATCH 202/360] ci(ct): determine base image name once in maintenance workflow Instead of retrieving the base image name again in every step, add it to the environment as variable. In the same go, enforce the shorter tags for release branches and the full length ones for the development branch. --- .github/workflows/container_maintenance.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 2e668af5162..976f01f920f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -104,6 +104,16 @@ jobs: tar -zxf - -C "${GITHUB_WORKSPACE}/patches" --wildcards "*/modules/container-base/src/backports/${{ matrix.branch }}" --strip-components=6 find "${GITHUB_WORKSPACE}/patches" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + # Determine the base image name we are going to use from here on + - name: Determine base image name + run: | + if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then + NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag='${base.image.tag.release}' -q -DforceStdout ) + fi + echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" + # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates id: temurin-check @@ -118,7 +128,6 @@ jobs: exit 1 fi - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" @@ -135,7 +144,6 @@ jobs: id: package-check if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} run: | - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then echo "Base image $BASE_IMAGE needs package updates" @@ -151,7 +159,6 @@ jobs: curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 # Now get the new revision number - BASE_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} From f15c9d60bcaed58fbe667595beafe099cf1e910f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:21:06 +0200 Subject: [PATCH 203/360] style,ci(ct): use shell parameter extension and less cut Little faster, little more portable. --- .github/workflows/container_maintenance.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 976f01f920f..fdf40f81750 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -120,17 +120,17 @@ jobs: run: | JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" JAVA_IMAGE_NS="library" - JAVA_IMAGE_REPO="$( echo "$JAVA_IMAGE" | cut -f1 -d: )" - JAVA_IMAGE_TAG="$( echo "$JAVA_IMAGE" | cut -f2 -d: )" + JAVA_IMAGE_REPO="${JAVA_IMAGE%:*}" + JAVA_IMAGE_TAG="${JAVA_IMAGE#*:}" JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" exit 1 fi - BASE_IMAGE_NS="$( echo "$BASE_IMAGE" | cut -f1 -d/ )" - BASE_IMAGE_REPO="$( echo "$BASE_IMAGE" | cut -f1 -d: | cut -f2 -d/ )" - BASE_IMAGE_TAG="$( echo "$BASE_IMAGE" | cut -f2 -d: )" + BASE_IMAGE_NS="${BASE_IMAGE%/*}" + BASE_IMAGE_REPO="$( echo "${BASE_IMAGE%:*}" | cut -f2 -d/ )" + BASE_IMAGE_TAG="${BASE_IMAGE#*:}" BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" From e3de4a178de418934d8036ac7a720523920959bf Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 30 Aug 2024 16:28:12 +0200 Subject: [PATCH 204/360] ci(ct): on release branches make the base image tags short and add additional revision tag --- .github/workflows/container_maintenance.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index fdf40f81750..da7d3028c3f 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -153,13 +153,15 @@ jobs: echo "newer_packages=false" >> "${GITHUB_OUTPUT}" fi - - name: Calculate revision number for immutable tag + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ matrix.branch != env.DEVELOP_BRANCH }} run: | # Get the revision logic script (not present on older releases) curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 # Now get the new revision number - echo "REVISION_OPTION=-Dbase.image.revision=$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" | tee -a "${GITHUB_ENV}" + REVISION_TAG="${BASE_IMAGE#*:}-r$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" + echo "DOCKER_TAGS=-Dbase.image.tag=\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=${REVISION_TAG}" | tee -a "${GITHUB_ENV}" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | @@ -167,7 +169,7 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${REVISION_OPTION} -Ddocker.platforms=${{ env.PLATFORMS }} + run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} # - if: always() # name: Save status (workaround for matrix outputs) From 59b676be7567aa68b1351df2b7b0bf5f16823512 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Sep 2024 14:52:42 +0200 Subject: [PATCH 205/360] docs(ct): update tagging policy for base image --- .../source/container/base-image.rst | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index e364579b568..13cd393a7e2 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -28,24 +28,29 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -In our tagging, we `follow the Bitname example `_: - -- ``latest``: Rolling tag on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) - Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - Usage is recommended for development purposes only. -- ``..--p-j``: - Rolling tag corresponding to functionality provided by the ``container-base`` module at a certain version. - Auto-released for changes on ``develop`` branch as well as scheduled maintenance for the *three* most recent Dataverse releases. - Semantic versioning applies to functionality of the base image only. - (`Dockerfile `__ or same location within tagged Dataverse releases) -- ``..--p-j-r``: - Immutable tag, otherwise same as above. - Revision is incremented for rebuilds of the base image. - Recommended for production builds and usage. - -**Note:** It is not possible to provide shorter rolling tags, as the source for a tag like ``x.y.z-jammy`` would not be unambiguous. -If the version number does not change between releases and/or the ``develop`` branch, one is going to win over another image. -If at the same time Payara or Java version change (which is independent from the module version!), it becomes unclear which version people end up using. +In our tagging, we `follow the Bitname example `_. +For the time being, the only operating system *flavors* we use and support are ``noble`` (6.4+) and ``jammy`` (pre 6.4). + +- | ``latest``: + | **Rolling tag** on par with whatever happens on the ``develop`` branch. (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. + Usage is recommended for **development purposes only**. +- | ``.--p-j``: + | **Rolling tag** during a development cycle of the Dataverse software (`Dockerfile `__). + | *Note 1:* No updates for tags after a development cycle ends. + Usage is recommended for **development purposes only**. + | *Note 2:* The current tag will always be 1 minor version ahead of Datavere releases. + Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. + The rolling tag in use during the cycle will be ``6.(x+1)-ffffff-p6.202P.P-jJJ``. + See also: :doc:`release process <../developers/making-releases>`. +- | ``.-``: + | **Rolling tag** for released versions of Dataverse, will always point to latest revision (see below) of a release image. + | The **three latest releases** of the Dataverse project are supported by regular, scheduled maintenance. + Content will be fairly stable, as disruptive changes like Payara or Java version switches are extremly unlikely once released. + Expect disruptive changes in case of high risk security threats. +- | ``.--r``: + | **Immutable tag**; revision is incremented for rebuilds of the base image. + Recommended for **production builds and usage.** Image Contents ++++++++++++++ From 6567e029d996478c2e8c660bd561f4ba6827b97f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 09:43:00 +0200 Subject: [PATCH 206/360] ci: add setup-maven action Instead of repeating the same steps everywhere, use a common action under our version control. --- .github/actions/setup-maven/action.yml | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/actions/setup-maven/action.yml diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml new file mode 100644 index 00000000000..6edbba1b1a9 --- /dev/null +++ b/.github/actions/setup-maven/action.yml @@ -0,0 +1,37 @@ +--- +name: "Setup Maven and Caches" +description: "Determine Java version and setup Maven, including necessary caches." +inputs: + git-reference: + description: 'The git reference (branch/tag) to check out' + required: true + default: 'develop' + pom-paths: + description: "List of paths to Maven POM(s) for cache dependency setup" + required: false + default: 'pom.xml' +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git-reference }} + - name: Determine Java version by reading the Maven property + shell: bash + run: | + echo "JAVA_VERSION=$(grep '' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: ${{ inputs.pom-paths }} + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository From 0966edaad306e123f294effc2b7be45f4491ba60 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 09:43:29 +0200 Subject: [PATCH 207/360] ci(ct): run maintenance with the new setup-maven action Testing here - might not yet work. --- .github/workflows/container_maintenance.yml | 45 ++++++++++----------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index da7d3028c3f..3baae428180 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -2,6 +2,8 @@ name: Container Images Scheduled Maintenance on: + # TODO: think about adding a (filtered) push event trigger here in case we change the patches + # --- # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) workflow_dispatch: inputs: @@ -57,29 +59,13 @@ jobs: #if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout and Setup Maven + # TODO: change to upstream location in final PR + # Necessary as the checked out release branch might not contain the action as files + uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img with: - ref: ${{ matrix.branch }} - - name: Determine Java version from Parent POM - run: | - echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - - - name: Set up JDK ${{ env.JAVA_VERSION }} - id: setup-java - uses: actions/setup-java@v4 - with: - java-version: ${{ env.JAVA_VERSION }} - distribution: 'temurin' - cache: 'maven' - cache-dependency-path: | - modules/container-base/pom.xml - - name: Download common cache on branch cache miss - if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} - uses: actions/cache/restore@v4 - with: - key: dataverse-maven-cache - path: ~/.m2/repository + git-reference: ${{ matrix.branch }} + pom-paths: modules/container-base/pom.xml # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and # on events in context of upstream because secrets. PRs run in context of forks by default! @@ -169,7 +155,9 @@ jobs: - name: Deploy multi-arch base container image to Docker Hub if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build - run: mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # - if: always() # name: Save status (workaround for matrix outputs) @@ -177,6 +165,17 @@ jobs: # # steps.build.outcome is the status BEFORE continue-on-error # echo "STATUS_$( echo "${{ matrix.branch }}" | tr ".:;,-/ " "_" )=${{ steps.build.outcome }}" | tee -a "${GITHUB_ENV}" + - name: Rebuild application container + if: ${{ steps.build.outputs.rebuild }} + uses: ./.github/actions/deploy-app-container + with: + registry: "" + registry_token: "" + ref: "" + base_image: "" + base_image_tag: "" + + #push-app-img: # name: "Rebase & Publish App Image" # permissions: From 7177c32a9642ec99e1df1f802d1b567778a8c821 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:05:57 +0200 Subject: [PATCH 208/360] ci: add action to check for newer releases of base images Reusable action to determine if a base image has more recent updates than the creation of the derived image. --- .../actions/check-newer-base-image/action.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/actions/check-newer-base-image/action.yml diff --git a/.github/actions/check-newer-base-image/action.yml b/.github/actions/check-newer-base-image/action.yml new file mode 100644 index 00000000000..d0c2e92d60a --- /dev/null +++ b/.github/actions/check-newer-base-image/action.yml @@ -0,0 +1,53 @@ +--- +name: "Check Newer Base Image" +description: "Determine if a base image is more recent than the derived." +inputs: + base: + description: 'The name and (rolling) tag of the base image to check - or - a shell command to get it' + required: true + derived: + description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' + required: true +outputs: + is-more-recent: + description: "True if base image has a more recent update, false if not." + value: "${{ steps.determine.outputs.is_more_recent }}" + +runs: + using: composite + steps: + - shell: bash + id: determine + run: | + BASE_IMAGE="${{ inputs.base }}" + # Get namespace, default to "library" if not found + BASE_IMAGE_NS="${BASE_IMAGE%/*}" + if [[ "$BASE_IMAGE_NS" = "${BASE_IMAGE}" ]]; then + BASE_IMAGE_NS="library" + fi + BASE_IMAGE_REPO="${BASE_IMAGE%:*}" + BASE_IMAGE_TAG="${BASE_IMAGE#*:}" + + BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid Base Image'::Could not find ${BASE_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="${{ inputs.derived }}" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$BASE_IMAGE_LAST_UPDATE" ]]; then + echo "Base image $BASE_IMAGE has a newer release ($BASE_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=true" >> $GITHUB_OUTPUT + else + echo "Base image $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=false" >> $GITHUB_OUTPUT + fi From 8bc6bceff3fc365800440bedad1471f1ff382c68 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:05:57 +0200 Subject: [PATCH 209/360] ci: add action to check for newer releases of base images Reusable action to determine if a base image has more recent updates than the creation of the derived image. --- .../check-newer-parent-image/action.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/actions/check-newer-parent-image/action.yml diff --git a/.github/actions/check-newer-parent-image/action.yml b/.github/actions/check-newer-parent-image/action.yml new file mode 100644 index 00000000000..6f8ce950cec --- /dev/null +++ b/.github/actions/check-newer-parent-image/action.yml @@ -0,0 +1,53 @@ +--- +name: "Check Newer Parent Image" +description: "Determine if a base image is more recent than the derived." +inputs: + parent: + description: 'The name and (rolling) tag of the parent image to check - or - a shell command to get it' + required: true + derived: + description: 'The name and (rolling) tag of the derived image - or - a shell command to get it' + required: true +outputs: + is-more-recent: + description: "True if base image has a more recent update, false if not." + value: "${{ steps.determine.outputs.is_more_recent }}" + +runs: + using: composite + steps: + - shell: bash + id: determine + run: | + PARENT_IMAGE="${{ inputs.parent }}" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="${{ inputs.derived }}" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=true" >> $GITHUB_OUTPUT + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + echo "is_more_recent=false" >> $GITHUB_OUTPUT + fi From 19fcf89540120d8dbb5d01ceb993868e41f2c6e6 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 11:06:26 +0200 Subject: [PATCH 210/360] ci(ct): use newer image action in maintenance workflow --- .github/workflows/container_maintenance.yml | 31 +++++---------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 3baae428180..aabfe093301 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -103,32 +103,15 @@ jobs: # Figure out if a rebuild is necessary because either there is an updated Java image or our installed packages need updates - name: Check for recent Temurin image updates id: temurin-check - run: | - JAVA_IMAGE="$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" - JAVA_IMAGE_NS="library" - JAVA_IMAGE_REPO="${JAVA_IMAGE%:*}" - JAVA_IMAGE_TAG="${JAVA_IMAGE#*:}" - JAVA_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${JAVA_IMAGE_NS}/repositories/${JAVA_IMAGE_REPO}/tags/${JAVA_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$JAVA_IMAGE_LAST_UPDATE" = "null" ]]; then - echo "::error title='Invalid Java Image'::Could not find ${JAVA_IMAGE} in the registry" - exit 1 - fi - - BASE_IMAGE_NS="${BASE_IMAGE%/*}" - BASE_IMAGE_REPO="$( echo "${BASE_IMAGE%:*}" | cut -f2 -d/ )" - BASE_IMAGE_TAG="${BASE_IMAGE#*:}" - BASE_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${BASE_IMAGE_NS}/repositories/${BASE_IMAGE_REPO}/tags/${BASE_IMAGE_TAG}" | jq -r .last_updated )" - if [[ "$BASE_IMAGE_LAST_UPDATE" = "null" || "$BASE_IMAGE_LAST_UPDATE" < "$JAVA_IMAGE_LAST_UPDATE" ]]; then - echo "Java image $JAVA_IMAGE has a newer release ($JAVA_IMAGE_LAST_UPDATE), which is more recent than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" - echo "newer_java_image=true" >> "${GITHUB_OUTPUT}" - else - echo "Java image $JAVA_IMAGE ($JAVA_IMAGE_LAST_UPDATE) is older than $BASE_IMAGE ($BASE_IMAGE_LAST_UPDATE)" - echo "newer_java_image=false" >> "${GITHUB_OUTPUT}" - fi + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/check-newer-parent-image@10478-version-base-img + with: + parent: "$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout )" + derived: "${{ env.BASE_IMAGE }}" # TODO: if we introduce more flavors as a matrix, we need to adapt the install command to check for updates - name: Check for package updates in base image id: package-check - if: ${{ steps.temurin-check.outputs.newer_java_image == 'false' }} + if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then @@ -153,7 +136,7 @@ jobs: run: | echo "DOCKER_TAGS=-Ddocker.imagePropertyConfiguration=override -Ddocker.tags.develop=latest" | tee -a "${GITHUB_ENV}" - name: Deploy multi-arch base container image to Docker Hub - if: ${{ steps.temurin-check.outputs.newer_java_image == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} + if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} From dbf2372da3e6a624c1e957da09551dd38563e95a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 12:59:40 +0200 Subject: [PATCH 211/360] ci: add action to determine next image revision --- .github/actions/get-image-revision/action.yml | 78 +++++++++++++++++++ .../workflows/scripts/get_next_revision.sh | 53 ------------- 2 files changed, 78 insertions(+), 53 deletions(-) create mode 100644 .github/actions/get-image-revision/action.yml delete mode 100755 .github/workflows/scripts/get_next_revision.sh diff --git a/.github/actions/get-image-revision/action.yml b/.github/actions/get-image-revision/action.yml new file mode 100644 index 00000000000..ca9eedeabe1 --- /dev/null +++ b/.github/actions/get-image-revision/action.yml @@ -0,0 +1,78 @@ +name: "Get Incremented Revision Tag" +description: "For a pre-existing rolling tag lookup the latest immutable revision tag, increment and return" +inputs: + image-ref: + description: "The full image reference including namespace, repo and tag" + required: true + revision-separator: + description: "The separator string to use between tag and revision number. Defaults to '-r'" + required: false + default: "-r" + tag-options-prefix: + description: "A string that the revision tag will be appended to and sent to output tag-options" + required: false + default: "" +outputs: + revision-tag: + description: "The updated immutable tag, ready to use" + value: ${{ steps.extract.outputs.revision_tag }} + tag-options: + description: "A string with some command line options (may be empty)" + value: ${{ steps.extract.outputs.tag_options }} +runs: + using: composite + steps: + - shell: bash + id: extract + run: | + IMAGE="${{ inputs.image-ref }}" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we run into rate limits + token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}${{ inputs.revision-separator }}" | sed -e "s#${IMAGE_TAG}${{ inputs.revision-separator }}##" | sort -h | tail -n1 ) + if [[ "$CURRENT" ]]; then + REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}$((CURRENT+1))" + else + REVISION_TAG="${IMAGE_TAG}${{ inputs.revision-separator }}0" + fi + echo "revision_tag=${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" + + if [[ -n "${{ inputs.tag-options-prefix }}" ]]; then + echo "tag_options=${{ inputs.tag-options-prefix }}${REVISION_TAG}" | tee -a "$GITHUB_OUTPUT" + fi \ No newline at end of file diff --git a/.github/workflows/scripts/get_next_revision.sh b/.github/workflows/scripts/get_next_revision.sh deleted file mode 100755 index 631f8b6f513..00000000000 --- a/.github/workflows/scripts/get_next_revision.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# This script is used to retrieve the next revision number for a given image reference. -# The image reference must be of the form "[/]:", where the namespace will default to "library" if omitted. - -set -eu - -IMAGE=${1} -IMAGE_NS_REPO="$( echo "$IMAGE" | cut -d: -f1 )" -IMAGE_TAG="$( echo "$IMAGE" | cut -d: -f2 )" - -if [[ "$IMAGE_TAG" == "$IMAGE_NS_REPO" ]]; then - >&2 echo "You must provide an image reference in the format [/]:" - exit 1 -fi - -case "$IMAGE_NS_REPO" in - */*) :;; # namespace/repository syntax, leave as is - *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax -esac - -# Without such a token we run into rate limits -token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) - -ALL_TAGS="$( - i=0 - while [ $? == 0 ]; do - i=$((i+1)) - RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) - if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then - # If we run into an error on the first attempt, that means we have a problem. - if [[ "$i" == "1" ]]; then - >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" - exit 2 - # Otherwise it will just mean we reached the last page already - else - break - fi - else - echo "$RESULT" | jq -r '."results"[]["name"]' - # DEBUG: - #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' - fi - done -)" - -# If a former tag could not be found, it just might not exist already. Setting to -1, will be incremented to 0 to start a new series. -CURRENT=$( echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 ) -if [[ "$CURRENT" ]]; then - echo "$((CURRENT+1))" -else - echo "0" -fi From 54e21cc63e526826cfcf29776ba2279d9d7dc4d8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 13:00:02 +0200 Subject: [PATCH 212/360] ci(ct): use image revision action in maintenance workflow --- .github/workflows/container_maintenance.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index aabfe093301..692e934e9d2 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -96,7 +96,7 @@ jobs: if [[ "${{ matrix.branch }}" = "${{ env.DEVELOP_BRANCH }}" ]]; then NAME=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) else - NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag='${base.image.tag.release}' -q -DforceStdout ) + NAME=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag="\${base.image.tag.release}" -q -DforceStdout ) fi echo "BASE_IMAGE=${NAME}" | tee -a "${GITHUB_ENV}" @@ -114,7 +114,7 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep -q "0 upgraded" ) ]]; then echo "Base image $BASE_IMAGE needs package updates" echo "newer_packages=true" >> "${GITHUB_OUTPUT}" else @@ -124,13 +124,12 @@ jobs: - name: Calculate revision number for immutable tag (on release branches only) if: ${{ matrix.branch != env.DEVELOP_BRANCH }} - run: | - # Get the revision logic script (not present on older releases) - curl -sSL "https://github.com/${GITHUB_REPOSITORY}/archive/${DEVELOP_BRANCH}.tar.gz" | \ - tar -zxf - -C "." --wildcards "*/.github/workflows/scripts/get_next_revision.sh" --strip-components=1 - # Now get the new revision number - REVISION_TAG="${BASE_IMAGE#*:}-r$( .github/workflows/scripts/get_next_revision.sh "${BASE_IMAGE}" )" - echo "DOCKER_TAGS=-Dbase.image.tag=\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=${REVISION_TAG}" | tee -a "${GITHUB_ENV}" + id: revision-tag + # TODO: change to upstream location in final PR + uses: gdcc/wip-dataverse-base-image/.github/actions/get-image-revision@10478-version-base-img + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag=\\${base.image.tag.release} -Ddocker.imagePropertyConfiguration=override -Ddocker.tags.revision=" - name: Configure update of "latest" tag for development branch if: ${{ matrix.branch == env.DEVELOP_BRANCH }} run: | @@ -139,7 +138,7 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'true' || steps.package-check.outputs.newer_packages == 'true' || inputs.force_build }} id: build run: | - mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} -Ddocker.platforms=${{ env.PLATFORMS }} + mvn -f modules/container-base -Pct deploy -Ddocker.noCache ${DOCKER_TAGS} ${{ steps.revision-tag.outputs.tag-options }} -Ddocker.platforms=${{ env.PLATFORMS }} echo "rebuild=true" | tee -a "${GITHUB_OUTPUT}" # - if: always() From db64fdfe05e7cbc89966ca3a6744a16fc423b97e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 3 Sep 2024 13:25:35 +0200 Subject: [PATCH 213/360] fix,ci(ct): don't grep for package updates from apt update output --- .github/workflows/container_maintenance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 692e934e9d2..c09aad22916 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -114,11 +114,11 @@ jobs: if: ${{ steps.temurin-check.outputs.is-more-recent == 'false' }} run: | PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" - if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update && apt install -s ${PKGS}" | grep -q "0 upgraded" ) ]]; then - echo "Base image $BASE_IMAGE needs package updates" + if [[ ! $( docker run --rm -u 0 "${BASE_IMAGE}" sh -c "apt update >&2 && apt install -s ${PKGS}" | grep "0 upgraded" ) ]]; then + echo "Base image $BASE_IMAGE needs updates for our custom installed packages" echo "newer_packages=true" >> "${GITHUB_OUTPUT}" else - echo "Base image $BASE_IMAGE has no package updates" + echo "Base image $BASE_IMAGE has no updates for our custom installed packages" echo "newer_packages=false" >> "${GITHUB_OUTPUT}" fi From d19e7f2cc192acdcd9bf4ef7abc5e194292a2cbe Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 3 Sep 2024 16:20:41 -0400 Subject: [PATCH 214/360] missing empty watermark entry --- src/main/java/propertyFiles/citation.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index 1b65b5d391e..ad8ec326e1e 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -191,6 +191,7 @@ datasetfieldtype.topicClassValue.watermark= datasetfieldtype.topicClassVocab.watermark= datasetfieldtype.topicClassVocabURI.watermark=https:// datasetfieldtype.publication.watermark= +datasetfieldtype.publicationRelationType.watermark= datasetfieldtype.publicationCitation.watermark= datasetfieldtype.publicationIDType.watermark= datasetfieldtype.publicationIDNumber.watermark= From 5c041e448eedd5f24fe45f9d3138efe5780d975e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 3 Sep 2024 16:44:48 -0400 Subject: [PATCH 215/360] fix capitalization --- src/main/java/propertyFiles/citation.properties | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index ad8ec326e1e..c5c40ac66b7 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -274,12 +274,12 @@ controlledvocabulary.publicationIDType.upc=upc controlledvocabulary.publicationIDType.url=url controlledvocabulary.publicationIDType.urn=urn controlledvocabulary.publicationIDType.dash-nrs=DASH-NRS -controlledVocabulary.publicationRelationType.iscitedby=Is Cited By -controlledVocabulary.publicationRelationType.cites=Cites -controlledVocabulary.publicationRelationType.issupplementto=Is Supplement To -controlledVocabulary.publicationRelationType.issupplementedby=Is Supplemented By -controlledVocabulary.publicationRelationType.isreferencedby=Is Referenced By -controlledVocabulary.publicationRelationType.references=References +controlledvocabulary.publicationRelationType.iscitedby=Is Cited By +controlledvocabulary.publicationRelationType.cites=Cites +controlledvocabulary.publicationRelationType.issupplementto=Is Supplement To +controlledvocabulary.publicationRelationType.issupplementedby=Is Supplemented By +controlledvocabulary.publicationRelationType.isreferencedby=Is Referenced By +controlledvocabulary.publicationRelationType.references=References controlledvocabulary.contributorType.data_collector=Data Collector controlledvocabulary.contributorType.data_curator=Data Curator controlledvocabulary.contributorType.data_manager=Data Manager From b4327789e0c14813417cb1e75c2bc618dd9651e1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 6 Sep 2024 13:50:53 -0400 Subject: [PATCH 216/360] fix test --- .../pidproviders/doi/datacite/XmlMetadataTemplateTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java index c1bbc3bebc1..c03146904de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.pidproviders.doi.DoiMetadata; @@ -129,6 +130,9 @@ public void testDataCiteXMLCreation() throws IOException { ArrayList dsvs = new ArrayList<>(); dsvs.add(0, dv); d.setVersions(dsvs); + DatasetType dType = new DatasetType(); + dType.setName(DatasetType.DATASET_TYPE_DATASET); + d.setDatasetType(dType); String xml = template.generateXML(d); System.out.println("Output is " + xml); From b8878b00f3ece6b9b595fa4367e559fe7db2e471 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 6 Sep 2024 10:32:05 -0400 Subject: [PATCH 217/360] fix labels when cvoc is used --- src/main/webapp/metadataFragment.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 0da2953a0d0..723f95148cd 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -251,7 +251,7 @@
- + #{dsf.datasetFieldType.localeTitle}

-