diff --git a/tooling/helpers/car.go b/tooling/helpers/car.go index 0c745055d..b0d1f784e 100644 --- a/tooling/helpers/car.go +++ b/tooling/helpers/car.go @@ -18,7 +18,11 @@ func StandardCARTestTransforms(t *testing.T, sts test.SugarTests) test.SugarTest } func applyStandardCarResponseHeaders(t *testing.T, st test.SugarTest) test.SugarTest { - st.Response = st.Response.Headers( + resp, ok := st.Response.(test.ExpectBuilder) + if !ok { + t.Fatal("can only apply test transformation on an ExpectBuilder") + } + st.Response = resp.Headers( // TODO: Go always sends Content-Length and it's not possible to explicitly disable the behavior. // For now, we ignore this check. It should be able to be resolved soon: https://github.com/ipfs/boxo/pull/177 // test.Header("Content-Length"). diff --git a/tooling/test/sugar.go b/tooling/test/sugar.go index 966c96859..93030adfb 100644 --- a/tooling/test/sugar.go +++ b/tooling/test/sugar.go @@ -1,7 +1,9 @@ package test import ( + "net/http" "net/url" + "testing" "github.com/ipfs/gateway-conformance/tooling/check" "github.com/ipfs/gateway-conformance/tooling/tmpl" @@ -133,6 +135,10 @@ func (r RequestBuilder) Clone() RequestBuilder { } } +type ExpectValidator interface { + Validate(t *testing.T, res *http.Response, localReport Reporter) +} + type ExpectBuilder struct { StatusCode_ int `json:"statusCode,omitempty"` Headers_ []HeaderBuilder `json:"headers,omitempty"` @@ -208,6 +214,47 @@ func (e ExpectBuilder) BodyWithHint(hint string, body interface{}) ExpectBuilder return e } +func (e ExpectBuilder) Validate(t *testing.T, res *http.Response, localReport Reporter) { + t.Helper() + + checks := validateResponse(t, e, res) + for _, c := range checks { + t.Run(c.testName, func(t *testing.T) { + if !c.checkOutput.Success { + localReport(t, c.checkOutput.Reason) + } + }) + } +} + +type AnyOfExpectBuilder struct { + Expect_ []ExpectBuilder `json:"expect,omitempty"` +} + +func AnyOf(expect ...ExpectBuilder) AnyOfExpectBuilder { + return AnyOfExpectBuilder{Expect_: expect} +} + +func (e AnyOfExpectBuilder) Validate(t *testing.T, res *http.Response, localReport Reporter) { + t.Helper() + + for _, expect := range e.Expect_ { + checks := validateResponse(t, expect, res) + responseSucceeded := true + for _, c := range checks { + if !c.checkOutput.Success { + responseSucceeded = false + break + } + } + if responseSucceeded { + return + } + } + + localReport(t, "none of the response options were valid") +} + type HeaderBuilder struct { Key_ string `json:"key,omitempty"` Value_ string `json:"value,omitempty"` diff --git a/tooling/test/test.go b/tooling/test/test.go index 33245cc07..5e03c6c93 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -14,7 +14,7 @@ type SugarTest struct { Hint string Request RequestBuilder Requests []RequestBuilder - Response ExpectBuilder + Response ExpectValidator Responses ExpectsBuilder } @@ -55,7 +55,9 @@ func run(t *testing.T, tests SugarTests) { for _, req := range test.Requests { _, res, localReport := runRequest(timeout, t, test, req) - validateResponse(t, test.Response, res, localReport) + if test.Response != nil { + test.Response.Validate(t, res, localReport) + } responses = append(responses, res) } @@ -64,7 +66,9 @@ func run(t *testing.T, tests SugarTests) { } else { t.Run(test.Name, func(t *testing.T) { _, res, localReport := runRequest(timeout, t, test, test.Request) - validateResponse(t, test.Response, res, localReport) + if test.Response != nil { + test.Response.Validate(t, res, localReport) + } }) } } diff --git a/tooling/test/validate.go b/tooling/test/validate.go index 2187d06d4..ff62e56f1 100644 --- a/tooling/test/validate.go +++ b/tooling/test/validate.go @@ -9,75 +9,87 @@ import ( "github.com/ipfs/gateway-conformance/tooling/check" ) +type testCheckOutput struct { + testName string + checkOutput check.CheckOutput +} + func validateResponse( t *testing.T, expected ExpectBuilder, res *http.Response, - localReport Reporter, -) { +) []testCheckOutput { t.Helper() + var outputs []testCheckOutput + if expected.StatusCode_ != 0 { + output := testCheckOutput{testName: "Status Code", checkOutput: check.CheckOutput{Success: true}} if res.StatusCode != expected.StatusCode_ { - localReport(t, "Status code is not %d. It is %d", expected.StatusCode_, res.StatusCode) + output.checkOutput.Success = false + output.checkOutput.Reason = fmt.Sprintf("Status code is not %d. It is %d", expected.StatusCode_, res.StatusCode) } + outputs = append(outputs, output) } for _, header := range expected.Headers_ { - t.Run(fmt.Sprintf("Header %s", header.Key_), func(t *testing.T) { - actual := res.Header.Values(header.Key_) + testName := fmt.Sprintf("Header %s", header.Key_) + actual := res.Header.Values(header.Key_) - c := header.Check_ - if header.Not_ { - c = check.Not(c) - } - output := c.Check(actual) + c := header.Check_ + if header.Not_ { + c = check.Not(c) + } + output := c.Check(actual) - if !output.Success { - if header.Hint_ == "" { - localReport(t, "Header '%s' %s", header.Key_, output.Reason) - } else { - localReport(t, "Header '%s' %s (%s)", header.Key_, output.Reason, header.Hint_) - } + if !output.Success { + if header.Hint_ == "" { + output.Reason = fmt.Sprintf("Header '%s' %s", header.Key_, output.Reason) + } else { + output.Reason = fmt.Sprintf("Header '%s' %s (%s)", header.Key_, output.Reason, header.Hint_) } - }) + } + + outputs = append(outputs, testCheckOutput{testName: testName, checkOutput: output}) } if expected.Body_ != nil { - t.Run("Body", func(t *testing.T) { - defer res.Body.Close() - resBody, err := io.ReadAll(res.Body) - if err != nil { - localReport(t, err) - } + defer res.Body.Close() + resBody, err := io.ReadAll(res.Body) + if err != nil { + outputs = append(outputs, testCheckOutput{testName: "Body", checkOutput: check.CheckOutput{Success: false, Reason: err.Error()}}) + return outputs + } - var output check.CheckOutput - - switch v := expected.Body_.(type) { - case check.Check[string]: - output = v.Check(string(resBody)) - case check.Check[[]byte]: - output = v.Check(resBody) - case string: - output = check.IsEqual(v).Check(string(resBody)) - case []byte: - output = check.IsEqualBytes(v).Check(resBody) - default: - output = check.CheckOutput{ - Success: false, - Reason: fmt.Sprintf("Body check has an invalid type: %T", expected.Body_), - } + var output check.CheckOutput + + switch v := expected.Body_.(type) { + case check.Check[string]: + output = v.Check(string(resBody)) + case check.Check[[]byte]: + output = v.Check(resBody) + case string: + output = check.IsEqual(v).Check(string(resBody)) + case []byte: + output = check.IsEqualBytes(v).Check(resBody) + default: + output = check.CheckOutput{ + Success: false, + Reason: fmt.Sprintf("Body check has an invalid type: %T", expected.Body_), } + } - if !output.Success { - if output.Hint == "" { - localReport(t, "Body %s", output.Reason) - } else { - localReport(t, "Body %s (%s)", output.Reason, output.Hint) - } + if !output.Success { + if output.Hint == "" { + output.Reason = fmt.Sprintf("Body %s", output.Reason) + } else { + output.Reason = fmt.Sprintf("Body %s (%s)", output.Reason, output.Hint) } - }) + } + + outputs = append(outputs, testCheckOutput{testName: "Body", checkOutput: output}) } + return outputs } func readPayload(res *http.Response) ([]byte, error) {