feat: multiple skips

.github/actions/test/action.yml

@@ -25,6 +25,10 @@ inputs:
     description: "A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec)."
     required: false
     default: ""
+  skips:
+    description: "A json array of regexp to skip tests."
+    required: false
+    default: ""
   args:
     description: "[DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests."
     required: false
@@ -33,19 +37,30 @@ runs:
   steps:
     - id: github
       uses: pl-strflt/docker-container-action/.github/actions/github@v1
+    - id: prepare-args
+      shell: bash
+      env:
+        SKIPS: ${{ inputs.skips }}
+      run: |
+        SKIPS_ARGS=""
+        if [ -n "$SKIPS" ]; then
+          SKIPS_ARGS=$(echo "$SKIPS" | jq -r '.[]' | sed -e 's/^/--skip="/' -e 's/$/"/' | paste -s -d ' ' -)
+        fi
+        echo "SKIPS_ARGS=$SKIPS_ARGS" >> $GITHUB_OUTPUT
     - name: Run the test
       uses: pl-strflt/docker-container-action@v1
       env:
         URL: ${{ inputs.gateway-url }}
         SUBDOMAIN: ${{ inputs.subdomain-url }}
         JSON: ${{ inputs.json }}
         SPECS: ${{ inputs.specs }}
+        SKIPS: ${{ steps.prepare-args.outputs.SKIPS_ARGS }}
       with:
         repository: ${{ steps.github.outputs.action_repository }}
         ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
         opts: --network=host
-        args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }}
+        args: test --url="$URL" --json="$JSON" --specs="$SPECS" ${SKIPS} --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }}
         build-args: |
           VERSION:${{ steps.github.outputs.action_ref }}
     - name: Create the XML

.github/workflows/test-kubo-skipped-e2e.yml

@@ -0,0 +1,81 @@
+name: Test Kubo Skipped (e2e)
+
+on:
+  workflow_dispatch:
+  push:
+    branches: 
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ['latest', 'master']
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.4
+      - uses: actions/checkout@v3
+        with:
+          path: 'gateway-conformance'
+      - name: Extract fixtures
+        uses: ./gateway-conformance/.github/actions/extract-fixtures
+        with:
+          output: fixtures
+      - uses: protocol/cache-go-action@v1
+      - run: go install github.com/ipfs/kubo/cmd/ipfs@${{ matrix.target }}
+        shell: bash
+        env:
+          GOPROXY: direct
+      - name: Configure Kubo Gateway
+        run: |
+          ipfs init;
+          source ./gateway-conformance/kubo-config.example.sh "$(pwd)/fixtures"
+          echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV
+        # note: the IPFS_NS_MAP set above will be passed the daemon
+      - uses: ipfs/start-ipfs-daemon-action@v1
+        with:
+          args: '--offline'
+          wait-for-addrs: false
+      - name: Provision Kubo Gateway
+        run: |
+          # Import car files
+          cars=$(find ./fixtures -name '*.car')
+          for car in $cars
+          do
+              ipfs dag import --pin-roots=false --stats "$car"
+          done
+
+          # Import ipns records
+          records=$(find ./fixtures -name '*.ipns-record')
+          for record in $records
+          do
+              key=$(basename -s .ipns-record "$record" | cut -d'_' -f1)
+              ipfs routing put --allow-offline "/ipns/$key" "$record"
+          done
+      - name: Run the tests
+        uses: ./gateway-conformance/.github/actions/test
+        with:
+          gateway-url: http://127.0.0.1:8080
+          subdomain-url: http://example.com
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          skips: '["TestTrustlessCarPathing", "TestNativeDag/GET_plain_JSON_codec_from_.*"]'
+      - name: Set summary
+        if: (failure() || success())
+        run: cat ./output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload one-page HTML report
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ matrix.target }}.html
+          path: ./output.html

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 ### Added
 - `--version` flag shows the current version
+- `--skip` parameter to skip one or more tests. [PR](https://github.com/ipfs/gateway-conformance/pull/148)
 - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
 
 ## [0.3.0] - 2023-07-31

README.md

@@ -37,6 +37,7 @@ The `test` command is the main command of the tool. It is used to test a given I
 | html | GitHub Action | The path where the one-page HTML test report should be generated. | `./report.html` |
 | markdown | GitHub Action | The path where the summary Markdown test report should be generated. | `./report.md` |
 | specs | Both | A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec). | Mature specs only |
+| skip | Both | Run only the those tests that do not match the regular expression. Similar to golang's skip, the expression is split by slash (/) into a sequence and each part must match the corresponding part in the test name, if any | empty |
 | args | Both | [DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests. | N/A |
 
 ##### Specs
@@ -79,6 +80,7 @@ A few examples:
     markdown: report.md
     html: report.html
     args: -timeout 30m
+    skips: '["TestGatewaySubdomains", "TestGatewayCar/GET_response_for_application/.*/Header_Content-Length"]'
 ```
 
 ##### Docker
@@ -235,10 +237,11 @@ gateway-conformance test --specs trustless-gateway,-trustless-gateway-ipns
 
 ### Skip a specific test
 
-Tests are skipped using Go's standard syntax:
+Tests are skipped using the `--skip` parameter and Go's standard syntax:
 
 ```bash
-gateway-conformance test -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+gateway-conformance test --skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+gateway-conformance test --skip 'TestGatewayCar/.*/vnd.ipld.car' --skip 'TestGatewayCar/GET_response_for_application/vnd.*'
 ```
 
 ### Extracting the test fixtures

cmd/gateway-conformance/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"log"
@@ -76,6 +77,7 @@ func main() {
 	var directory string
 	var merged bool
 	var verbose bool
+	var skips cli.StringSlice
 
 	app := &cli.App{
 		Name:    "gateway-conformance",
@@ -113,6 +115,13 @@ func main() {
 						Value:       "",
 						Destination: &specs,
 					},
+					&cli.StringSliceFlag{
+						Name: "skip",
+						Usage: "Accepts a test path to skip.\nCan be used multiple times.\n" +
+							"Example: --skip \"TestTar/GET_TAR_with_format=.*/Body\" --skip \"TestGatewayBlock\" --skip \"TestTrustlessCarPathing\"\n" +
+							"It uses the same syntax as the -skip flag of go test.",
+						Destination: &skips,
+					},
 					&cli.BoolFlag{
 						Name:        "verbose",
 						Usage:       "Prints all the output to the console.",
@@ -134,10 +143,17 @@ func main() {
 
 					fmt.Println("go " + strings.Join(args, " "))
 
+					// json encode the string array for parameter
+					skipsList := skips.Value()
+					skipsJSON, err := json.Marshal(skipsList)
+					if err != nil {
+						return fmt.Errorf("failed to marshal skips: %w", err)
+					}
+
 					output := &bytes.Buffer{}
 					cmd := exec.Command("go", args...)
 					cmd.Dir = tooling.Home()
-					cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL))
+					cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL), fmt.Sprintf("TEST_SKIPS=%s", skipsJSON))
 
 					if subdomainGatewayURL != "" {
 						cmd.Env = append(cmd.Env, fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL))

tooling/test/skips.go

@@ -0,0 +1,115 @@
+package test
+
+import (
+	"encoding/json"
+	"os"
+	"regexp"
+	"testing"
+)
+
+const SKIPS_ENV_VAR = "TEST_SKIPS"
+
+var skips = []string{}
+
+func GetSkips() []string {
+	skips := os.Getenv(SKIPS_ENV_VAR)
+	if skips == "" {
+		return []string{}
+	}
+
+	var skipsList []string
+	err := json.Unmarshal([]byte(skips), &skipsList)
+	if err != nil {
+		panic(err)
+	}
+
+	return skipsList
+}
+
+func split(name string) []string {
+	// split name by "/" not prefixed with "\"
+	parts := []string{}
+	current := ""
+	hasSlash := false
+
+	for _, c := range name {
+		if hasSlash {
+			if c == '/' || c == '\\' {
+				current += string(c)
+				hasSlash = false
+			} else {
+				current += "\\"
+				current += string(c)
+				hasSlash = false
+			}
+			continue
+		}
+
+		if c == '/' {
+			parts = append(parts, current)
+			current = ""
+			continue
+		}
+		if c == '\\' {
+			hasSlash = true
+			continue
+		}
+
+		current += string(c)
+	}
+
+	if current != "" {
+		parts = append(parts, current)
+	}
+
+	return parts
+}
+
+func isSkipped(name string, skips []string) bool {
+	for _, skip := range skips {
+		if name == skip {
+			return true
+		}
+
+		skipParts := split(skip)
+		nameParts := split(name)
+
+		if len(skipParts) > len(nameParts) {
+			continue
+		}
+
+		matches := true
+		for i := range skipParts {
+			skipPart := skipParts[i]
+			skipPart = "^" + skipPart + "$"
+
+			matched, err := regexp.MatchString(skipPart, nameParts[i])
+			if err != nil {
+				panic(err)
+			}
+
+			if !matched {
+				matches = false
+				break
+			}
+		}
+
+		return matches
+	}
+
+	return false
+}
+
+func MustNotBeSkipped(t *testing.T) {
+	t.Helper()
+	skipped := isSkipped(t.Name(), skips)
+
+	if skipped {
+		t.Skipf("skipped")
+	}
+}
+
+// init will load the skips
+func init() {
+	skips = GetSkips()
+}