Create a framework to exercise the go test output

Prior to this patch, there was no way to exercise the behavior of
`-rewrite` nor how errors and sub-tests would be reported in the
output of `go test -v`.

This patch introduces such "reflective" tests, where a test can
compare the output and modified file through running `go test`
recursively.

datadriven_test.go

@@ -16,9 +16,15 @@ package datadriven
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/cockroachdb/errors"
+	"github.com/otiai10/copy"
 )
 
 func TestNewLineBetweenDirectives(t *testing.T) {
@@ -124,7 +130,7 @@ while %d other monkeys watch %s
 
 func TestSubTest(t *testing.T) {
 	foundData := false
-	Walk(t, "testdata", func(t *testing.T, path string) {
+	Walk(t, "testdata/subtest", func(t *testing.T, path string) {
 		foundData = true
 		RunTest(t, path, func(t *testing.T, d *TestData) string {
 			switch d.Cmd {
@@ -148,3 +154,143 @@ func TestSubTest(t *testing.T) {
 		t.Fatalf("no data file found")
 	}
 }
+
+func TestGoTestInterface(t *testing.T) {
+	runTestOverDirectory(t, "TestRewrite", "testdata/no_rewrite_needed",
+		false /*verbose*/, []string{"-rewrite"},
+		`ok  	github.com/cockroachdb/datadriven`, ``)
+
+	runTestOverDirectory(t, "TestRewrite", "testdata/rewrite",
+		false /*verbose*/, []string{"-rewrite"},
+		`ok  	github.com/cockroachdb/datadriven`,
+		`diff -uNr testdata/rewrite/example <datadir>/example
+--- testdata/rewrite/example
++++ <datadir>/example
+@@ -1,6 +1,7 @@
+ hello universe
+ ----
+-incorrect output
++universe was said
+ 
+ hello planet
+ ----
++planet was said
+diff -uNr testdata/rewrite/whitespace <datadir>/whitespace
+--- testdata/rewrite/whitespace
++++ <datadir>/whitespace
+@@ -2,22 +2,23 @@
+ hello world
+ 
+ ----
+-wrong
++world was said
+ 
+ # Command with no output
+ nothing
+ ----
+-wrong
+ 
+ # Command with whitespace output
+ blank
+ ----
+-wrong
++----
+ 
++----
++----
+ blank
+ ----
+ ----
+-wrong
++
+ ----
+ ----
+ `)
+}
+
+func runTestOverDirectory(
+	t *testing.T, testName, dataDir string, verbose bool, extraArgs []string, refTest, refDiff string,
+) {
+	tmpDir, err := ioutil.TempDir("", "go-datadriven")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if t.Failed() {
+			t.Logf("test files remaining in: %s", tmpDir)
+		} else {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+
+	if err := copy.Copy(dataDir, tmpDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// Run go test.
+	args := []string{"test", "-tags", "gotest"}
+	if verbose {
+		args = append(args, "-v")
+	}
+	args = append(args,
+		"-run", testName+"$",
+		".",
+		"-args", "-datadir", tmpDir)
+	args = append(args, extraArgs...)
+	testCmd := exec.Command("go", args...)
+	testOut, err := testCmd.CombinedOutput()
+	if err != nil {
+		// Special case: if the exit code is specifically 1, we're going
+		// to ignore it -- this simply signals there was a test error. The
+		// expected/actual compare below will catch it.
+		if ee, ok := err.(*exec.ExitError); !ok || ee.ExitCode() != 1 {
+			t.Fatalf("cmd %v:\n%s\nerror: %v", testCmd, testOut, err)
+		}
+	}
+
+	if refTest != "" && !strings.HasSuffix(refTest, "\n") {
+		refTest += "\n"
+	}
+	actual := postProcessGoTestOutput(tmpDir, string(testOut))
+	if string(actual) != refTest {
+		t.Errorf("\nexpected go test output:\n%s\ngot:\n%s", refTest, actual)
+	}
+
+	// Diff the test files, to check if rewriting happened.
+	diffCmd := exec.Command("diff", "-uNr", dataDir, tmpDir)
+	diffOut, err := diffCmd.CombinedOutput()
+	if err != nil {
+		// Special case: if the exit code is specifically 1, we're going
+		// to ignore it -- this simply signals the diff is not empty. The
+		// expected/actual compare below will catch it.
+		if ee, ok := err.(*exec.ExitError); !ok || ee.ExitCode() != 1 {
+			t.Fatalf("cmd %v:\n%s\neerror: %v", diffCmd, diffOut, err)
+		}
+	}
+
+	if refDiff != "" && !strings.HasSuffix(refDiff, "\n") {
+		refDiff += "\n"
+	}
+	actual = postProcessDiffOutput(dataDir, tmpDir, string(diffOut))
+	if string(actual) != refDiff {
+		t.Errorf("\nexpected testadata diff:\n%s\ngot:\n%s", refDiff, actual)
+	}
+}
+
+var resultTs = regexp.MustCompile(`(?m:^((?:FAIL|ok)\s+\S+).*$)`)
+var intermediateTs = regexp.MustCompile(`(?m:^(\s*---.*)\s+\(.*\)$)`)
+
+func postProcessGoTestOutput(tmpDir, testOut string) string {
+	testOut = strings.ReplaceAll(testOut, tmpDir, "<datadir>")
+	testOut = resultTs.ReplaceAllString(testOut, "$1")
+	testOut = intermediateTs.ReplaceAllString(testOut, "$1")
+	return testOut
+}
+
+var diffTs = regexp.MustCompile(`(?m:^((?:\+\+\+|---)\s+\S+).*$)`)
+
+func postProcessDiffOutput(dataDir, tmpDir, testOut string) string {
+	testOut = strings.ReplaceAll(testOut, tmpDir, "<datadir>")
+	testOut = diffTs.ReplaceAllString(testOut, "$1")
+	return testOut
+}

gotest_test.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// +build gotest
+
+package datadriven
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+)
+
+var dataDir = flag.String("datadir", "", "directory where to find datafiles")
+
+func TestRewrite(t *testing.T) {
+	if *dataDir == "" {
+		t.Fatal("don't run this test directly; run TestGoTestInterface or similar instead")
+	}
+	Walk(t, *dataDir, func(t *testing.T, path string) {
+		RunTest(t, path, func(t *testing.T, d *TestData) string {
+			t.Logf("directive: %v %+v", d.Cmd, d.CmdArgs)
+			switch d.Cmd {
+			case "hello":
+				return d.CmdArgs[0].Key + " was said"
+			case "nothing":
+				return ""
+			case "blank":
+				l := 0
+				if len(d.CmdArgs) > 0 {
+					l = len(d.CmdArgs[1].Key)
+				}
+				return fmt.Sprintf("%*s\n", l, "")
+			case "skip":
+				// Verify that calling t.Skip() does not fail with an API error on
+				// testing.T.
+				t.Skip("woo")
+			case "error":
+				t.Error("some error: " + d.CmdArgs[0].Key)
+			default:
+				t.Fatalf("unknown directive: %s", d.Cmd)
+			}
+			return d.Expected
+		})
+	})
+}

testdata/no_rewrite_needed/example

@@ -0,0 +1,3 @@
+hello world
+----
+world was said

testdata/no_rewrite_needed/whitespace

@@ -0,0 +1,17 @@
+# Whitespace in input
+hello world
+
+----
+world was said
+
+# Command with no output
+nothing
+----
+
+# Command with whitespace output
+blank
+----
+----
+
+----
+----

testdata/rewrite/example

@@ -0,0 +1,6 @@
+hello universe
+----
+incorrect output
+
+hello planet
+----

testdata/rewrite/whitespace

@@ -0,0 +1,30 @@
+# Whitespace in input
+hello world
+
+----
+wrong
+
+# Command with no output
+nothing
+----
+wrong
+
+# Command with whitespace output
+blank
+----
+wrong
+
+blank
+----
+----
+wrong
+----
+----
+
+# This one is correct, we just test the rewrite does not do excess work.
+blank
+----
+----
+
+----
+----