Create a framework to exercise the go test output

Prior to this patch, there was no way to exercise the behavior of
`-rewrite` nor how errors and sub-tests would be reported in the
output of `go test -v`.

This patch introduces such "reflective" tests, where a test can
compare the output and modified file through running `go test`
recursively.

datadriven_test.go

@@ -15,10 +15,17 @@
 package datadriven
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/cockroachdb/errors"
+	"github.com/otiai10/copy"
 )
 
 func TestNewLineBetweenDirectives(t *testing.T) {
@@ -124,7 +131,7 @@ while %d other monkeys watch %s
 
 func TestSubTest(t *testing.T) {
 	foundData := false
-	Walk(t, "testdata", func(t *testing.T, path string) {
+	Walk(t, "testdata/subtest", func(t *testing.T, path string) {
 		foundData = true
 		RunTest(t, path, func(t *testing.T, d *TestData) string {
 			switch d.Cmd {
@@ -148,3 +155,100 @@ func TestSubTest(t *testing.T) {
 		t.Fatalf("no data file found")
 	}
 }
+
+func TestGoTestInterface(t *testing.T) {
+	runTestOverDirectory(t, "TestRewrite", "testdata/rewrite", []string{"-rewrite"},
+		`ok  	github.com/cockroachdb/datadriven
+
+test files after go test completes:
+diff -uNr testdata/rewrite/rewrite_needed <datadir>/rewrite_needed
+--- testdata/rewrite/rewrite_needed
++++ <datadir>/rewrite_needed
+@@ -1,6 +1,7 @@
+ hello universe
+ ----
+-incorrect output
++universe was said
+ 
+ hello planet
+ ----
++planet was said
+`)
+}
+
+func runTestOverDirectory(t *testing.T, testName, dataDir string, args []string, refOutput string) {
+	tmpDir, err := ioutil.TempDir("", "go-datadriven")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if t.Failed() {
+			t.Logf("test files remaining in: %s", tmpDir)
+		} else {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+
+	if err := copy.Copy(dataDir, tmpDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// Run go test.
+	args = append([]string{"test",
+		"-tags", "gotest",
+		"-run", testName + "$",
+		".",
+		"-args", "-datadir", tmpDir,
+	}, args...)
+	testCmd := exec.Command("go", args...)
+	testOut, err := testCmd.CombinedOutput()
+	if err != nil {
+		// Special case: if the exit code is specifically 1, we're going
+		// to ignore it -- this simply signals there was a test error. The
+		// expected/actual compare below will catch it.
+		if ee, ok := err.(*exec.ExitError); !ok || ee.ExitCode() != 1 {
+			t.Fatalf("cmd %v:\n%s\nerror: %v", testCmd, testOut, err)
+		}
+	}
+
+	var actual bytes.Buffer
+	actual.WriteString(postProcessGoTestOutput(tmpDir, string(testOut)))
+	actual.WriteByte('\n')
+
+	// Diff the test files, to check if rewriting happened.
+	diffCmd := exec.Command("diff", "-uNr", dataDir, tmpDir)
+	diffOut, err := diffCmd.CombinedOutput()
+	if err != nil {
+		// Special case: if the exit code is specifically 1, we're going
+		// to ignore it -- this simply signals the diff is not empty. The
+		// expected/actual compare below will catch it.
+		if ee, ok := err.(*exec.ExitError); !ok || ee.ExitCode() != 1 {
+			t.Fatalf("cmd %v:\n%s\neerror: %v", diffCmd, diffOut, err)
+		}
+	}
+
+	actual.Write([]byte("test files after go test completes:\n"))
+	actual.WriteString(postProcessDiffOutput(dataDir, tmpDir, string(diffOut)))
+
+	if actual.String() != refOutput {
+		t.Fatalf("expected:\n%s\ngot:\n%s", refOutput, actual.String())
+	}
+}
+
+var resultTs = regexp.MustCompile(`(?m:^((?:FAIL|ok)\s+\S+).*$)`)
+var intermediateTs = regexp.MustCompile(`(?m:^(\s*---.*)\s+\(.*\)$)`)
+
+func postProcessGoTestOutput(tmpDir, testOut string) string {
+	testOut = strings.ReplaceAll(testOut, tmpDir, "<datadir>")
+	testOut = resultTs.ReplaceAllString(testOut, "$1")
+	testOut = intermediateTs.ReplaceAllString(testOut, "$1")
+	return testOut
+}
+
+var diffTs = regexp.MustCompile(`(?m:^((?:\+\+\+|---)\s+\S+).*$)`)
+
+func postProcessDiffOutput(dataDir, tmpDir, testOut string) string {
+	testOut = strings.ReplaceAll(testOut, tmpDir, "<datadir>")
+	testOut = diffTs.ReplaceAllString(testOut, "$1")
+	return testOut
+}

gotest_test.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// +build gotest
+
+package datadriven
+
+import (
+	"flag"
+	"testing"
+)
+
+var dataDir = flag.String("datadir", "", "directory where to find datafiles")
+
+func TestRewrite(t *testing.T) {
+	if *dataDir == "" {
+		t.Fatal("don't run this test directly; run TestGoTestInterface or similar instead")
+	}
+	Walk(t, *dataDir, func(t *testing.T, path string) {
+		RunTest(t, path, func(t *testing.T, d *TestData) string {
+			switch d.Cmd {
+			case "hello":
+				return d.CmdArgs[0].Key + " was said"
+			case "skip":
+				// Verify that calling t.Skip() does not fail with an API error on
+				// testing.T.
+				t.Skip("woo")
+			case "error":
+				t.Error("some error: " + d.CmdArgs[0].Key)
+			default:
+				t.Fatalf("unknown directive: %s", d.Cmd)
+			}
+			return d.Expected
+		})
+	})
+}

testdata/rewrite/no_rewrite_needed

@@ -0,0 +1,3 @@
+hello world
+----
+world was said

testdata/rewrite/rewrite_needed

@@ -0,0 +1,6 @@
+hello universe
+----
+incorrect output
+
+hello planet
+----