Patch reporter

docs/command-line/generated/mull-cxx-cli-options.rst

@@ -20,6 +20,8 @@
 
     :Elements:	Generates mutation-testing-elements compatible JSON file
 
+    :Patches:	Generates a patchfile for each mutation. uses `--git-project-root` as base dir.
+
 --ide-reporter-show-killed		Makes IDEReporter to also report killed mutations (disabled by default)
 
 --debug		Enables Debug Mode: more logs are printed

docs/command-line/generated/mull-runner-cli-options.rst

@@ -16,6 +16,8 @@
 
     :Elements:	Generates mutation-testing-elements compatible JSON file
 
+    :Patches:	Generates a unified patchfile for each mutation
+
 --ide-reporter-show-killed		Makes IDEReporter to also report killed mutations (disabled by default)
 
 --debug		Enables Debug Mode: more logs are printed

include/mull/Reporters/PatchesReporter.h

@@ -0,0 +1,30 @@
+#include "Reporter.h"
+
+#include "mull/Reporters/SourceCodeReader.h"
+#include <memory>
+#include <string>
+#include <vector>
+#include <regex>
+
+namespace mull {
+
+class Result;
+class Diagnostics;
+
+class PatchesReporter : public Reporter {
+public:
+  explicit PatchesReporter(Diagnostics &diagnostics, const std::string &reportDir = "",
+                          const std::string &reportName = "", const std::string basePath = "");
+
+  void reportResults(const Result &result) override;
+
+  std::string getPatchesPath();
+
+private:
+  Diagnostics &diagnostics;
+  std::string patchesPath;
+  std::regex basePathRegex;
+  SourceCodeReader sourceCodeReader;
+};
+
+} // namespace mull

include/mull/Reporters/Reporter.h

@@ -4,7 +4,7 @@ namespace mull {
 
 class Result;
 
-enum class ReporterKind { IDE, SQLite, Elements };
+enum class ReporterKind { IDE, SQLite, Elements, Patches };
 
 class Reporter {
 public:

include/mull/Reporters/SourceCodeReader.h

@@ -13,6 +13,7 @@ class SourceCodeReader {
   SourceCodeReader();
   std::string getContext(const mull::SourceLocation &sourceLocation);
   std::string getSourceLineWithCaret(const SourceLocation &sourceLocation);
+  std::string getSourceLine(const SourceLocation &sourceLocation);
 private:
   SourceManager sourceManager;
 };

lib/CMakeLists.txt

@@ -39,6 +39,7 @@ set(mull_sources
   Reporters/SourceCodeReader.cpp
   Reporters/SourceManager.cpp
   Reporters/SQLiteReporter.cpp
+  Reporters/PatchesReporter.cpp
 
   SourceLocation.cpp
 

lib/Reporters/PatchesReporter.cpp

@@ -0,0 +1,111 @@
+#include "mull/Reporters/PatchesReporter.h"
+
+#include "mull/Bitcode.h"
+#include "mull/Diagnostics/Diagnostics.h"
+#include "mull/ExecutionResult.h"
+#include "mull/Mutant.h"
+#include "mull/Mutators/Mutator.h"
+#include "mull/Mutators/MutatorsFactory.h"
+#include "mull/Result.h"
+#include "mull/SourceLocation.h"
+#include "mull/Reporters/SourceCodeReader.h"
+
+#include <llvm/IR/DebugInfoMetadata.h>
+#include <llvm/IR/Function.h>
+#include <llvm/IR/Instruction.h>
+#include <llvm/IR/Module.h>
+#include <llvm/Support/FileSystem.h>
+
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <regex>
+#include <unistd.h>
+
+using namespace mull;
+using namespace llvm;
+
+
+static std::string getReportName(const std::string &name) {
+  std::string reportName = name;
+  if (reportName.empty()) {
+    time_t t;
+    time(&t);
+    reportName = std::to_string(t);
+  }
+  return reportName + "-patches";
+}
+
+static std::string getReportDir(const std::string &reportDir) {
+  if (reportDir.empty()) {
+    return std::string(".");
+  }
+  return reportDir;
+}
+
+PatchesReporter::PatchesReporter(Diagnostics &diagnostics, const std::string &reportDir,
+                               const std::string &reportName, const std::string basePath)
+    : diagnostics(diagnostics),
+      patchesPath(getReportDir(reportDir) + "/" + getReportName(reportName)) ,
+      basePathRegex("^" + getReportDir(basePath)),
+      sourceCodeReader() {
+  llvm::sys::fs::create_directories(patchesPath, true);
+}
+
+std::string mull::PatchesReporter::getPatchesPath() {
+  return patchesPath;
+}
+
+void mull::PatchesReporter::reportResults(const Result &result) {
+  MutatorsFactory factory(diagnostics);
+  factory.init();
+  for (auto &mutationResult : result.getMutationResults()) {
+
+    const ExecutionResult mutationExecutionResult = mutationResult->getExecutionResult();
+
+    const auto mutant = *mutationResult->getMutant();
+    const auto& sourceLocation = mutant.getSourceLocation();
+    const auto& sourceEndLocation = mutant.getEndLocation();
+    const std::string sourceBasename = std::regex_replace(sourceLocation.filePath.substr(sourceLocation.directory.size()+1), std::regex("([/]|\\.(?!patch))"), "_");;
+    const auto mutator = factory.getMutator(mutant.getMutatorIdentifier());
+    const std::string sourceLine = sourceCodeReader.getSourceLine(sourceLocation);
+    const std::string sourcePath = std::regex_replace(sourceLocation.filePath, basePathRegex, "");
+
+    const std::string prefix = [&mutationExecutionResult](){
+      switch(mutationExecutionResult.status){
+        case ExecutionStatus::Passed:
+          return "survived-";
+          break;
+        case ExecutionStatus::NotCovered:
+          return "uncovered-";
+          break;
+        default:
+          return "killed-";
+      }
+    }();
+
+    const std::string filename =[&](){
+      std::stringstream filenamebuilder;
+      filenamebuilder << patchesPath << "/" << prefix
+               << sourceBasename  << "-" << mutant.getMutatorIdentifier()
+               << "-L" << sourceLocation.line << "-C" << sourceLocation.column
+               << ".patch";
+      return filenamebuilder.str();
+    }();
+
+    diagnostics.debug(std::string("Writing Patchfile: ") + filename.c_str());
+    std::ofstream myfile{filename};
+    myfile << "--- a" << sourcePath << " 0" << "\n"
+           << "+++ b" << sourcePath << " 0" << "\n"
+           << "@@ -" << sourceLocation.line << ",1 +" << sourceLocation.line << ",1 @@\n"
+           << "-" << sourceLine
+           << "+" << sourceLine.substr(0, sourceLocation.column-1)
+           << mutator->getReplacement() << sourceLine.substr(sourceEndLocation.column-1) ;
+    myfile.flush();
+    if(!myfile.good())
+      diagnostics.warning(std::string("Writing Patchfile failed") + filename.c_str());
+    myfile.close();
+  }
+
+  diagnostics.info(std::string("Patchfiles can be found at '") + patchesPath + "'");
+}

lib/Reporters/SourceCodeReader.cpp

@@ -91,3 +91,9 @@ std::string SourceCodeReader::getSourceLineWithCaret(const SourceLocation &sourc
   ss << line << caret << "\n";
   return ss.str();
 }
+
+std::string SourceCodeReader::getSourceLine(const SourceLocation &sourceLocation) {
+  auto line = sourceManager.getLine(sourceLocation);
+  assert(sourceLocation.column < line.size());
+  return line;
+}

tests-lit/tests/patch-reporter/equality/main.cpp

@@ -0,0 +1,28 @@
+int equal(int a, int b) {
+  return a == b;
+}
+
+int main() {
+  return equal(2, 2) != 1;
+}
+
+// clang-format off
+/**
+
+RUN: cd %S
+RUN: mkdir -p %S/Output/sandbox
+RUN: cp %S/main.cpp %S/Output/sandbox/main.cpp
+RUN: cd %S/Output/sandbox
+
+/// We cd to the the test directory and compile using relative paths.
+RUN: cd %S; %clang_cxx %sysroot -fembed-bitcode -g -O0 Output/sandbox/main.cpp -o Output/main.cpp.exe
+
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_cxx -mutators=cxx_eq_to_ne -linker-flags="%sysroot" --linker=%clang_cxx -debug main.cpp.exe --report-name test --reporters Patches --reporters IDE; test $? = 0; ls -R %S/Output/test-patches; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_eq_to_ne-L2-C12.patch) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+
+CHECK:[debug] Writing Patchfile: {{.*}}
+CHECK:[info] Patchfiles can be found at './test-patches'
+CHECK:{{.*}}main_cpp{{.*}}
+CHECK:--- a{{.*}}/Output/sandbox/main.cpp 0
+CHECK:+{{\s+}}return a != b;
+
+*/

tests-lit/tests/patch-reporter/git_dir_relative/main.cpp

@@ -0,0 +1,28 @@
+int equal(int a, int b) {
+  return a == b;
+}
+
+int main() {
+  return equal(2, 2) != 1;
+}
+
+// clang-format off
+/**
+
+RUN: cd %S
+RUN: mkdir -p %S/Output/sandbox
+RUN: cp %S/main.cpp %S/Output/sandbox/main.cpp
+RUN: cd %S/Output/sandbox
+
+/// We cd to the the test directory and compile using relative paths.
+RUN: cd %S; %clang_cxx %sysroot -fembed-bitcode -g -O0 Output/sandbox/main.cpp -o Output/main.cpp.exe
+
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_cxx -mutators=cxx_eq_to_ne -linker-flags="%sysroot" --git-project-root %S --linker=%clang_cxx -debug main.cpp.exe --report-name test --reporters Patches --reporters IDE; test $? = 0; ls -R %S/Output/test-patches; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_eq_to_ne-L2-C12.patch) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+
+CHECK:[debug] Writing Patchfile: {{.*}}
+CHECK:[info] Patchfiles can be found at './test-patches'
+CHECK:{{.*}}main_cpp{{.*}}
+CHECK:--- a/Output/sandbox/main.cpp 0
+CHECK:+{{\s+}}return a != b;
+
+*/

tests-lit/tests/patch-reporter/remove_void_call/main.cpp

@@ -0,0 +1,37 @@
+static int globalVar = -1;
+void voidFunction() {
+  globalVar = 0;
+}
+
+int foo() {
+  voidFunction();
+  return 0;
+};
+
+int main() {
+  foo();
+  return globalVar;
+}
+
+// clang-format off
+/**
+
+RUN: cd %S
+RUN: mkdir -p %S/Output/sandbox
+RUN: cp %S/main.cpp %S/Output/sandbox/main.cpp
+RUN: cd %S/Output/sandbox
+
+/// We cd to the the test directory and compile using relative paths.
+RUN: cd %S; %clang_cxx %sysroot -fembed-bitcode -g -O0 Output/sandbox/main.cpp -o Output/main.cpp.exe
+
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_cxx -mutators=cxx_calls -linker-flags="%sysroot" --linker=%clang_cxx -debug main.cpp.exe --report-name test --reporters Patches --reporters IDE; test $? = 0; ls -R %S/Output/test-patches; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_remove_void_call-L7-C3.patch; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_replace_scalar_call-L12-C3.patch) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+
+CHECK:[debug] Writing Patchfile: {{.*}}
+CHECK:[info] Patchfiles can be found at './test-patches'
+CHECK:{{.*}}main_cpp-cxx_remove_void{{.*}}
+CHECK:{{.*}}main_cpp-cxx_replace_scalar{{.*}}
+CHECK:--- a{{.*}}/Output/sandbox/main.cpp 0
+CHECK:+{{\s+}};
+CHECK:+{{\s+}}42;
+
+*/

tests-lit/tests/patch-reporter/reporter_path_base/main.cpp

@@ -0,0 +1,28 @@
+int equal(int a, int b) {
+  return a == b;
+}
+
+int main() {
+  return equal(2, 2) != 1;
+}
+
+// clang-format off
+/**
+
+RUN: cd %S
+RUN: mkdir -p %S/Output/sandbox
+RUN: cp %S/main.cpp %S/Output/sandbox/main.cpp
+RUN: cd %S/Output/sandbox
+
+/// We cd to the the test directory and compile using relative paths.
+RUN: cd %S; %clang_cxx %sysroot -fembed-bitcode -g -O0 Output/sandbox/main.cpp -o Output/main.cpp.exe
+
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_cxx -mutators=cxx_eq_to_ne -linker-flags="%sysroot" --git-project-root %S/Output --report-patch-base %S --linker=%clang_cxx -debug main.cpp.exe --report-name test --reporters Patches --reporters IDE; test $? = 0; ls -R %S/Output/test-patches; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_eq_to_ne-L2-C12.patch) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+
+CHECK:[debug] Writing Patchfile: {{.*}}
+CHECK:[info] Patchfiles can be found at './test-patches'
+CHECK:{{.*}}main_cpp{{.*}}
+CHECK:--- a/Output/sandbox/main.cpp 0
+CHECK:+{{\s+}}return a != b;
+
+*/

tests-lit/tests/patch-reporter/shift/main.cpp

@@ -0,0 +1,28 @@
+int bitwise_left_shift(int a, int b) {
+  return a << b;
+}
+
+int main() {
+  return ! (bitwise_left_shift(1, 2) == 4);
+}
+
+// clang-format off
+/**
+
+RUN: cd %S
+RUN: mkdir -p %S/Output/sandbox
+RUN: cp %S/main.cpp %S/Output/sandbox/main.cpp
+RUN: cd %S/Output/sandbox
+
+/// We cd to the the test directory and compile using relative paths.
+RUN: cd %S; %clang_cxx %sysroot -fembed-bitcode -g -O0 Output/sandbox/main.cpp -o Output/main.cpp.exe
+
+RUN: cd %S/Output && echo $PATH; (unset TERM; %mull_cxx -mutators=cxx_bitwise -linker-flags="%sysroot" --linker=%clang_cxx -debug main.cpp.exe --report-name test --reporters Patches --reporters IDE; test $? = 0; ls -R %S/Output/test-patches; cat %S/Output/test-patches/killed-Output_sandbox_main_cpp-cxx_lshift_to_rshift-L2-C12.patch) | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
+
+CHECK:[debug] Writing Patchfile: {{.*}}
+CHECK:[info] Patchfiles can be found at './test-patches'
+CHECK:killed-{{.*}}main_cpp{{.*}}
+CHECK:--- a{{.*}}/Output/sandbox/main.cpp 0
+CHECK:+{{\s+}}return a >> b;
+
+*/

tools/CLIOptions/CLIOptions.cpp

@@ -4,6 +4,7 @@
 #include <mull/Reporters/IDEReporter.h>
 #include <mull/Reporters/MutationTestingElementsReporter.h>
 #include <mull/Reporters/SQLiteReporter.h>
+#include <mull/Reporters/PatchesReporter.h>
 
 using namespace mull;
 using namespace tool;
@@ -44,6 +45,7 @@ static std::vector<ReporterDefinition> reporterOptions({
     { "Elements",
       "Generates mutation-testing-elements compatible JSON file",
       ReporterKind::Elements },
+    { "Patches", "Generates patch file for each mutation", ReporterKind::Patches },
 });
 
 ReportersCLIOptions::ReportersCLIOptions(Diagnostics &diagnostics, list<ReporterKind> &parameter)
@@ -67,6 +69,9 @@ std::vector<std::unique_ptr<Reporter>> ReportersCLIOptions::reporters(ReporterPa
     case ReporterKind::SQLite: {
       reporters.emplace_back(new mull::SQLiteReporter(diagnostics, directory, name));
     } break;
+    case ReporterKind::Patches: {
+      reporters.emplace_back(new mull::PatchesReporter(diagnostics, directory, name, params.patchBasePathDir));
+    } break;
     case ReporterKind::Elements: {
       if (!params.compilationDatabaseAvailable) {
         diagnostics.warning("Mutation Testing Elements Reporter may not work without compilation "

tools/CLIOptions/CLIOptions.h

@@ -79,6 +79,15 @@ opt<std::string> ReportName( \
     init(""), \
     cat(MullCategory))
 
+#define ReportPatchBaseDirectory_() \
+opt<std::string> ReportPatchBaseDirectory( \
+    "report-patch-base", \
+    desc("Create Patches relative to this directory (defaults to git-project-root if available, else absolute path will be used)"), \
+    Optional, \
+    value_desc("directory"), \
+    init("."), \
+    cat(MullCategory))
+
 #define DisableJunkDetection_() \
 opt<bool> DisableJunkDetection( \
     "disable-junk-detection", \
@@ -311,6 +320,7 @@ class MutatorsCLIOptions {
 struct ReporterParameters {
   std::string reporterName;
   std::string reporterDirectory;
+  std::string patchBasePathDir;
   bool compilationDatabaseAvailable;
   bool IDEReporterShowKilled;
 };