diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 8a28e771e..6f5aac798 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -25,6 +25,10 @@ inputs: description: "A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec)." required: false default: "" + skips: + description: "A json array of regexp to skip tests." + required: false + default: "" args: description: "[DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests." required: false @@ -33,6 +37,16 @@ runs: steps: - id: github uses: pl-strflt/docker-container-action/.github/actions/github@v1 + - id: prepare-args + shell: bash + env: + SKIPS: ${{ inputs.skips }} + run: | + SKIPS_ARGS="" + if [ -n "$SKIPS" ]; then + SKIPS_ARGS=$(echo "$SKIPS" | jq -r '.[]' | sed -e 's/^/--skip="/' -e 's/$/"/' | paste -s -d ' ' -) + fi + echo "SKIPS_ARGS=$SKIPS_ARGS" >> $GITHUB_OUTPUT - name: Run the test uses: pl-strflt/docker-container-action@v1 env: @@ -40,12 +54,13 @@ runs: SUBDOMAIN: ${{ inputs.subdomain-url }} JSON: ${{ inputs.json }} SPECS: ${{ inputs.specs }} + SKIPS: ${{ steps.prepare-args.outputs.SKIPS_ARGS }} with: repository: ${{ steps.github.outputs.action_repository }} ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }} dockerfile: Dockerfile opts: --network=host - args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }} + args: test --url="$URL" --json="$JSON" --specs="$SPECS" ${SKIPS} --subdomain-url="$SUBDOMAIN" -- ${{ inputs.args }} build-args: | VERSION:${{ steps.github.outputs.action_ref }} - name: Create the XML diff --git a/.github/workflows/test-kubo-skipped-e2e.yml b/.github/workflows/test-kubo-skipped-e2e.yml new file mode 100644 index 000000000..7036b940c --- /dev/null +++ b/.github/workflows/test-kubo-skipped-e2e.yml @@ -0,0 +1,81 @@ +name: Test Kubo Skipped (e2e) + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: 'ubuntu-latest' + strategy: + fail-fast: false + matrix: + target: ['latest', 'master'] + defaults: + run: + shell: bash + steps: + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.20.4 + - uses: actions/checkout@v3 + with: + path: 'gateway-conformance' + - name: Extract fixtures + uses: ./gateway-conformance/.github/actions/extract-fixtures + with: + output: fixtures + - uses: protocol/cache-go-action@v1 + - run: go install github.com/ipfs/kubo/cmd/ipfs@${{ matrix.target }} + shell: bash + env: + GOPROXY: direct + - name: Configure Kubo Gateway + run: | + ipfs init; + source ./gateway-conformance/kubo-config.example.sh "$(pwd)/fixtures" + echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + # note: the IPFS_NS_MAP set above will be passed the daemon + - uses: ipfs/start-ipfs-daemon-action@v1 + with: + args: '--offline' + wait-for-addrs: false + - name: Provision Kubo Gateway + run: | + # Import car files + cars=$(find ./fixtures -name '*.car') + for car in $cars + do + ipfs dag import --pin-roots=false --stats "$car" + done + + # Import ipns records + records=$(find ./fixtures -name '*.ipns-record') + for record in $records + do + key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) + ipfs routing put --allow-offline "/ipns/$key" "$record" + done + - name: Run the tests + uses: ./gateway-conformance/.github/actions/test + with: + gateway-url: http://127.0.0.1:8080 + subdomain-url: http://example.com + json: output.json + xml: output.xml + html: output.html + markdown: output.md + skips: '["TestTrustlessCarPathing", "TestNativeDag/GET_plain_JSON_codec_from_.*"]' + - name: Set summary + if: (failure() || success()) + run: cat ./output.md >> $GITHUB_STEP_SUMMARY + - name: Upload one-page HTML report + if: (failure() || success()) + uses: actions/upload-artifact@v3 + with: + name: conformance-${{ matrix.target }}.html + path: ./output.html diff --git a/CHANGELOG.md b/CHANGELOG.md index fc2788da3..6fe4fd4ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added - `--version` flag shows the current version +- `--skip` parameter to skip one or more tests. [PR](https://github.com/ipfs/gateway-conformance/pull/148) - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc. ## [0.3.0] - 2023-07-31 diff --git a/README.md b/README.md index 97aaea155..e87ab1526 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The `test` command is the main command of the tool. It is used to test a given I | html | GitHub Action | The path where the one-page HTML test report should be generated. | `./report.html` | | markdown | GitHub Action | The path where the summary Markdown test report should be generated. | `./report.md` | | specs | Both | A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec). | Mature specs only | +| skip | Both | Run only the those tests that do not match the regular expression. Similar to golang's skip, the expression is split by slash (/) into a sequence and each part must match the corresponding part in the test name, if any | empty | | args | Both | [DANGER] The `args` input allows you to pass custom, free-text arguments directly to the Go test command that the tool employs to execute tests. | N/A | ##### Specs @@ -79,6 +80,7 @@ A few examples: markdown: report.md html: report.html args: -timeout 30m + skips: '["TestGatewaySubdomains", "TestGatewayCar/GET_response_for_application/.*/Header_Content-Length"]' ``` ##### Docker @@ -235,10 +237,11 @@ gateway-conformance test --specs trustless-gateway,-trustless-gateway-ipns ### Skip a specific test -Tests are skipped using Go's standard syntax: +Tests are skipped using the `--skip` parameter and Go's standard syntax: ```bash -gateway-conformance test -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' +gateway-conformance test --skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' +gateway-conformance test --skip 'TestGatewayCar/.*/vnd.ipld.car' --skip 'TestGatewayCar/GET_response_for_application/vnd.*' ``` ### Extracting the test fixtures diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index dc0bd90f7..ae0e51f10 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io" "log" @@ -76,6 +77,7 @@ func main() { var directory string var merged bool var verbose bool + var skips cli.StringSlice app := &cli.App{ Name: "gateway-conformance", @@ -113,6 +115,13 @@ func main() { Value: "", Destination: &specs, }, + &cli.StringSliceFlag{ + Name: "skip", + Usage: "Accepts a test path to skip.\nCan be used multiple times.\n" + + "Example: --skip \"TestTar/GET_TAR_with_format=.*/Body\" --skip \"TestGatewayBlock\" --skip \"TestTrustlessCarPathing\"\n" + + "It uses the same syntax as the -skip flag of go test.", + Destination: &skips, + }, &cli.BoolFlag{ Name: "verbose", Usage: "Prints all the output to the console.", @@ -134,10 +143,17 @@ func main() { fmt.Println("go " + strings.Join(args, " ")) + // json encode the string array for parameter + skipsList := skips.Value() + skipsJSON, err := json.Marshal(skipsList) + if err != nil { + return fmt.Errorf("failed to marshal skips: %w", err) + } + output := &bytes.Buffer{} cmd := exec.Command("go", args...) cmd.Dir = tooling.Home() - cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL)) + cmd.Env = append(os.Environ(), fmt.Sprintf("GATEWAY_URL=%s", gatewayURL), fmt.Sprintf("TEST_SKIPS=%s", skipsJSON)) if subdomainGatewayURL != "" { cmd.Env = append(cmd.Env, fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL)) diff --git a/tooling/test/skips.go b/tooling/test/skips.go new file mode 100644 index 000000000..03b6c86c6 --- /dev/null +++ b/tooling/test/skips.go @@ -0,0 +1,115 @@ +package test + +import ( + "encoding/json" + "os" + "regexp" + "testing" +) + +const SKIPS_ENV_VAR = "TEST_SKIPS" + +var skips = []string{} + +func GetSkips() []string { + skips := os.Getenv(SKIPS_ENV_VAR) + if skips == "" { + return []string{} + } + + var skipsList []string + err := json.Unmarshal([]byte(skips), &skipsList) + if err != nil { + panic(err) + } + + return skipsList +} + +func split(name string) []string { + // split name by "/" not prefixed with "\" + parts := []string{} + current := "" + hasSlash := false + + for _, c := range name { + if hasSlash { + if c == '/' || c == '\\' { + current += string(c) + hasSlash = false + } else { + current += "\\" + current += string(c) + hasSlash = false + } + continue + } + + if c == '/' { + parts = append(parts, current) + current = "" + continue + } + if c == '\\' { + hasSlash = true + continue + } + + current += string(c) + } + + if current != "" { + parts = append(parts, current) + } + + return parts +} + +func isSkipped(name string, skips []string) bool { + for _, skip := range skips { + if name == skip { + return true + } + + skipParts := split(skip) + nameParts := split(name) + + if len(skipParts) > len(nameParts) { + continue + } + + matches := true + for i := range skipParts { + skipPart := skipParts[i] + skipPart = "^" + skipPart + "$" + + matched, err := regexp.MatchString(skipPart, nameParts[i]) + if err != nil { + panic(err) + } + + if !matched { + matches = false + break + } + } + + return matches + } + + return false +} + +func MustNotBeSkipped(t *testing.T) { + t.Helper() + skipped := isSkipped(t.Name(), skips) + + if skipped { + t.Skipf("skipped") + } +} + +// init will load the skips +func init() { + skips = GetSkips() +} diff --git a/tooling/test/skips_test.go b/tooling/test/skips_test.go new file mode 100644 index 000000000..c0c67f21a --- /dev/null +++ b/tooling/test/skips_test.go @@ -0,0 +1,142 @@ +package test + +import "testing" + +func TestSplit(t *testing.T) { + type test struct { + name string + expected []string + } + + tests := []test{ + { + name: `TestA`, + expected: []string{ + "TestA", + }, + }, + { + name: `TestA/With/Path`, + expected: []string{ + "TestA", + "With", + "Path", + }, + }, + { + name: `TestA/With/Path/And/Slash/\\`, + expected: []string{ + "TestA", + "With", + "Path", + "And", + "Slash", + `\`, + }, + }, + { + name: `TestA/With/Pa\th\/An\\d/Slash/\\\\`, + expected: []string{ + "TestA", + "With", + `Pa\th/An\d`, + "Slash", + `\\`, + }, + }, + } + + for _, test := range tests { + got := split(test.name) + if len(got) != len(test.expected) { + t.Errorf("split(%s) = %v, want %v", test.name, got, test.expected) + continue + } + + for i := range got { + if got[i] != test.expected[i] { + t.Errorf("split(%s) = %v, want %v (%s != %s)", test.name, got, test.expected, got[i], test.expected[i]) + break + } + } + } +} + +func TestSkips(t *testing.T) { + type test struct { + name string + skips []string + expected bool + } + + tests := []test{ + { + name: "TestNeverSkipped", + skips: []string{}, + expected: false, + }, + { + name: "TestAlwaysSkipped", + skips: []string{ + "TestAlwaysSkipped", + }, + expected: true, + }, + { + name: "TestA", + skips: []string{ + "TestA/With/Path", + }, + expected: false, + }, + { + name: "TestA/With/Path", + skips: []string{ + "TestA", + }, + expected: true, + }, + { + name: "TestA/With/Path", + skips: []string{ + "TestA/With.*out/Path", + }, + expected: false, + }, + { + name: "TestA/With/Path", + skips: []string{ + "Test.*", + }, + expected: true, + }, + { + name: "TestA/With/Path", + skips: []string{ + "Test.*/With/.*", + }, + expected: true, + }, + { + name: "TestA/Without/Path", + skips: []string{ + "Test.*/With/.*", + }, + expected: false, + }, + { + name: "TestA/With/Path", + skips: []string{ + "Test.*/With", + }, + expected: true, + }, + } + + for _, test := range tests { + got := isSkipped(test.name, test.skips) + if got != test.expected { + t.Errorf("isSkipped(%s, %v) = %v, want %v", test.name, test.skips, got, test.expected) + } + } +} diff --git a/tooling/test/test.go b/tooling/test/test.go index 33245cc07..2190b038f 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -44,6 +44,7 @@ func RunWithSpecs( func run(t *testing.T, tests SugarTests) { t.Helper() + MustNotBeSkipped(t) for _, test := range tests { timeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute) @@ -51,6 +52,7 @@ func run(t *testing.T, tests SugarTests) { if len(test.Requests) > 0 { t.Run(test.Name, func(t *testing.T) { + MustNotBeSkipped(t) responses := make([]*http.Response, 0, len(test.Requests)) for _, req := range test.Requests { @@ -63,6 +65,7 @@ func run(t *testing.T, tests SugarTests) { }) } else { t.Run(test.Name, func(t *testing.T) { + MustNotBeSkipped(t) _, res, localReport := runRequest(timeout, t, test, test.Request) validateResponse(t, test.Response, res, localReport) }) diff --git a/tooling/test/validate.go b/tooling/test/validate.go index 2187d06d4..6e6c4f71e 100644 --- a/tooling/test/validate.go +++ b/tooling/test/validate.go @@ -25,6 +25,7 @@ func validateResponse( for _, header := range expected.Headers_ { t.Run(fmt.Sprintf("Header %s", header.Key_), func(t *testing.T) { + MustNotBeSkipped(t) actual := res.Header.Values(header.Key_) c := header.Check_ @@ -45,6 +46,7 @@ func validateResponse( if expected.Body_ != nil { t.Run("Body", func(t *testing.T) { + MustNotBeSkipped(t) defer res.Body.Close() resBody, err := io.ReadAll(res.Body) if err != nil {