Merge branch 'master' into 1537-get-list-of-validators-that-couldnt-r…

…un-because-of-a-parsing-problem

.github/workflows/package_installers.yml

@@ -13,6 +13,19 @@ on:
     types: [ prereleased, released ]
   workflow_dispatch:
 
+env:
+  MACOS_APP_NAME: "GTFS Validator"
+  MACOS_NOTARIZATION_NAME: "notarization.zip"
+  MACOS_TARGET_PATH: "app/pkg/build/jpackage/GTFS Validator.app"
+  MACOS_TARGET_DEST_PATH: "app/pkg/build/jpackage"
+  MACOS_CERTIFICATE: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_P12_BASE64 }}
+  MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_PASSWORD }}
+  MACOS_CERTIFICATE_NAME: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_NAME }}
+  MACOS_CI_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+  MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.MACOS_NOTARIZATION_USERNAME }}
+  MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.MACOS_NOTARIZATION_TEAM_ID }}
+  MACOS_NOTARIZATION_PWD: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+
 jobs:
   validate_gradle_wrapper:
     runs-on: ubuntu-latest
@@ -25,7 +38,7 @@ jobs:
     name: Build and upload packaged app
     runs-on: ${{ matrix.os }}
     strategy:
-#     Adding fail-fast: false so at least some artifacts gets uploaded if others fail
+      # Adding fail-fast: false so at least some artifacts gets uploaded if others fail
       fail-fast: false
       matrix:
         os: [ macos-latest, windows-latest, ubuntu-latest ]
@@ -47,17 +60,18 @@ jobs:
       # We create a code-signing keychain on MacOS before building and packaging the app, as the
       # app will be signed as part of the jpackage build phase.
       - name: "MacOS - Import Certificate: Developer ID Application"
+        id: codesign
         if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/import-signing-certificate@v1
-        with:
-          # This should be a certificate + private key, exported as a p12 file and base64 encoded as
-          # a GitHub secret.  The certificate should be the:
-          # 'Developer ID Application: The International Data Organization For Transport (BF2U75HN4D)'
-          # certificate, locked with the specified passphrase.
-          certificate-data: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_P12_BASE64 }}
-          certificate-passphrase: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_PASSWORD }}
-          # The resulting keychain will be locked with this separate password.
-          keychain-password: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+        run: |
+          # Turn our base64-encoded certificate back to a regular .p12 file
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+
+          # We need to create a new keychain, otherwise using the certificate will prompt with a UI dialog asking for the certificate password, which we can't use in a headless CI environment
+          security create-keychain -p "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
+          security import certificate.p12 -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
 
       - name: "Package GUI app installer with Gradle"
         uses: gradle/gradle-build-action@v2
@@ -69,24 +83,35 @@ jobs:
       # On MacOS, we now submit the app for "notarization", where Apple will scan the app for
       # malware and other issues.  This step can take a few minutes or more; the action will wait
       # until the report is available.
-      - name: "MacOS - Notarize Release Build"
+      - name: "MacOS - Notarize & Staple Release Build"
         if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/xcode-notarize@v1
-        with:
-          product-path: "app/pkg/build/jpackage/GTFS Validator.app"
-          # The Apple developer account used to run notarization.  This account will receive an
-          # email every time notarization is run.
-          appstore-connect-username: ${{ secrets.MACOS_NOTARIZATION_USERNAME }}
-          # The app-specific password configured for the Apple developer account that will be used
-          # for authentication (different from the main password for the dev account).
-          appstore-connect-password: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+        id: notarize_staple_macos_app
+        run: |
+          # Store the notarization credentials so that we can prevent a UI password dialog from blocking the CI
+          
+          echo "Create keychain profile"
+          xcrun notarytool store-credentials "notarytool-profile" --apple-id "${MACOS_NOTARIZATION_APPLE_ID}" --team-id "${MACOS_NOTARIZATION_TEAM_ID}" --password "${MACOS_NOTARIZATION_PWD}"
+          
+          # We can't notarize an app bundle directly, but we need to compress it as an archive.
+          # Therefore, we create a zip file containing our app bundle, so that we can send it to the
+          # notarization service
+          
+          echo "Creating temp notarization archive"
+          ditto -c -k --keepParent "${{ env.MACOS_TARGET_PATH }}" ${{ env.MACOS_NOTARIZATION_NAME }}
+          
+          # Here we send the notarization request to the Apple's Notarization service, waiting for the result.
+          # This typically takes a few seconds inside a CI environment, but it might take more depending on the App
+          # characteristics. Visit the Notarization docs for more information and strategies on how to optimize it if
+          # you're curious
+          
+          echo "Notarize app"
+          xcrun notarytool submit "${{ env.MACOS_NOTARIZATION_NAME }}" --keychain-profile "notarytool-profile" --wait
+          
+          # Finally, we need to "attach the staple" to our executable, which will allow our app to be
+          # validated by macOS even when an internet connection is not available.
+          echo "Attach staple"
+          xcrun stapler staple "${{ env.MACOS_TARGET_PATH }}"
 
-      # Now that we have a notarization report, we attach it to the app binary.
-      - name: "Mac OS - Staple Release Build"
-        if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/xcode-staple@v1
-        with:
-          product-path: "app/pkg/build/jpackage/GTFS Validator.app"
 
       # Now that we have a notarized app, we can package it.
       - name: "Mac OS - Package the app"
@@ -97,26 +122,26 @@ jobs:
           appVersion=${appVersion//-SNAPSHOT/}
           jpackage \
            --type dmg \
-           --name 'GTFS Validator' \
+           --name "${{ env.MACOS_APP_NAME }}" \
            --app-version ${appVersion} \
-           --app-image app/pkg/build/jpackage/GTFS\ Validator.app \
-           --dest app/pkg/build/jpackage
+           --app-image "${{ env.MACOS_TARGET_PATH }}" \
+           --dest ${{ env.MACOS_TARGET_DEST_PATH }}
           jpackage \
            --type pkg \
            --name 'GTFS Validator' \
            --app-version ${appVersion} \
-           --app-image app/pkg/build/jpackage/GTFS\ Validator.app \
-           --dest app/pkg/build/jpackage
+           --app-image "${{ env.MACOS_TARGET_PATH }}" \
+           --dest ${{ env.MACOS_TARGET_DEST_PATH }}
 
       - name: "Upload Installer"
         uses: actions/upload-artifact@v3
         with:
           name: Installer - ${{matrix.os}}
           path: |
-            app/pkg/build/jpackage/*.msi
-            app/pkg/build/jpackage/*.dmg
-            app/pkg/build/jpackage/*.pkg
-            app/pkg/build/jpackage/*.deb
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.msi
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.dmg
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.pkg
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.deb
 
       - name: "Upload assets to release"
         if: github.event_name == 'prerelease' || github.event_name == 'release'

.github/workflows/stg_web_client_merge.yml

@@ -0,0 +1,51 @@
+name: Web client staging CD
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+
+  web_cd:
+    name: Deploy web client
+    runs-on: ubuntu-latest
+
+    env:
+      ENV_NAME              : stg
+      CLOUDSDK_CORE_PROJECT : web-based-gtfs-validator
+
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          # We need to download all tags so that the axion-release-plugin
+          # can resolve the most recent version tag.
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          # We need a recent version of Java with jpackage included.
+          java-version: '17'
+          # We use the zulu distribution, which is an OpenJDK distro.
+          distribution: 'zulu'
+
+      # for npm
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_WEB_VALIDATOR_SA_KEY }}
+
+      - uses: google-github-actions/setup-gcloud@v1
+        with:
+          version: '>= 390.0.0'
+
+      - name: run gradle tasks
+        shell: bash
+        run: |
+          ENV_FILE=web/pipeline/${ENV_NAME}.env source web/pipeline/env-file.sh
+          ./gradlew ':web:client:webDeploy'

.github/workflows/stg_web_svc_merge.yml

@@ -0,0 +1,51 @@
+name: Web service staging CD
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+
+  web_cd:
+    name: Deploy web service
+    runs-on: ubuntu-latest
+
+    env:
+      ENV_NAME              : stg
+      CLOUDSDK_CORE_PROJECT : web-based-gtfs-validator
+
+    steps:
+
+      - uses: actions/checkout@v3
+        with:
+          # We need to download all tags so that the axion-release-plugin
+          # can resolve the most recent version tag.
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          # We need a recent version of Java with jpackage included.
+          java-version: '17'
+          # We use the zulu distribution, which is an OpenJDK distro.
+          distribution: 'zulu'
+
+      # for npm
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_WEB_VALIDATOR_SA_KEY }}
+
+      - uses: google-github-actions/setup-gcloud@v1
+        with:
+          version: '>= 390.0.0'
+
+      - name: run gradle tasks
+        shell: bash
+        run: |
+          ENV_FILE=web/pipeline/${ENV_NAME}.env source web/pipeline/env-file.sh
+          ./gradlew ':web:service:webDeploy'

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java

@@ -80,7 +80,10 @@ public Collection<GtfsTableContainer<?>> getTables() {
   public String tableTotalsText() {
     List<String> totalList = new ArrayList<>();
     for (GtfsTableContainer<?> table : tables.values()) {
-      if (table.getTableStatus() == TableStatus.MISSING_FILE && !table.isRequired()) continue;
+      if (table.getTableStatus() == TableStatus.MISSING_FILE
+          && !table.getDescriptor().isRequired()) {
+        continue;
+      }
       totalList.add(
           table.gtfsFilename()
               + "\t"

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java

@@ -33,15 +33,23 @@
  */
 public abstract class GtfsTableContainer<T extends GtfsEntity> {
 
+  private final GtfsTableDescriptor<T> descriptor;
+
   private final TableStatus tableStatus;
 
   private final CsvHeader header;
 
-  public GtfsTableContainer(TableStatus tableStatus, CsvHeader header) {
+  public GtfsTableContainer(
+      GtfsTableDescriptor<T> descriptor, TableStatus tableStatus, CsvHeader header) {
+    this.descriptor = descriptor;
     this.tableStatus = tableStatus;
     this.header = header;
   }
 
+  public GtfsTableDescriptor<T> getDescriptor() {
+    return descriptor;
+  }
+
   public TableStatus getTableStatus() {
     return tableStatus;
   }
@@ -96,24 +104,6 @@ public boolean isMissingFile() {
     return tableStatus == TableStatus.MISSING_FILE;
   }
 
-  /**
-   * Tells if the file is recommended according to GTFS.
-   *
-   * <p>Note that a recommended file may be empty.
-   *
-   * @return true if the file is recommended, false otherwise
-   */
-  public abstract boolean isRecommended();
-
-  /**
-   * Tells if the file is required according to GTFS.
-   *
-   * <p>Note that a required file may be empty.
-   *
-   * @return true if the file is required, false otherwise
-   */
-  public abstract boolean isRequired();
-
   /**
    * Tells if the file was successfully parsed.
    *
@@ -135,7 +125,7 @@ public boolean isParsedSuccessfully() {
       case PARSABLE_HEADERS_AND_ROWS:
         return true;
       case MISSING_FILE:
-        return !isRequired();
+        return !descriptor.isRequired();
       default:
         return false;
     }

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java

@@ -8,6 +8,10 @@
 import org.mobilitydata.gtfsvalidator.parsing.CsvHeader;
 
 public abstract class GtfsTableDescriptor<T extends GtfsEntity> {
+
+  // True if the specified file is required in a feed.
+  private boolean required;
+
   public abstract GtfsTableContainer createContainerForInvalidStatus(
       GtfsTableContainer.TableStatus tableStatus);
 
@@ -24,7 +28,13 @@ public abstract GtfsTableContainer createContainerForHeaderAndEntities(
 
   public abstract boolean isRecommended();
 
-  public abstract boolean isRequired();
+  public boolean isRequired() {
+    return this.required;
+  }
+
+  public void setRequired(boolean required) {
+    this.required = required;
+  }
 
   public abstract Optional<Integer> maxCharsPerColumn();