Move tmp preprocess files to <my_project>/.terra/cxx_parser/preProces…

…s@<check_sum> (#34)

- Remove unused `CXXParserConfigs.customHeaders `
- Move tmp preprocess files to
`<my_project>/.terra/cxx_parser/preProcess@<check_sum>`

cxx-parser/__integration_test__/cxx_parser.integration.test.ts

@@ -4,16 +4,16 @@ import path from 'path';
 
 import { TerraContext } from '@agoraio-extensions/terra-core';
 
-import { CXXParser, dumpCXXAstJson } from '../src/cxx_parser';
+import { CXXParser, dumpCXXAstJson, generateChecksum } from '../src/cxx_parser';
 import { CXXFile, Struct } from '../src/cxx_terra_node';
 
 describe('cxx_parser', () => {
   let tmpDir: string = '';
-  let cppastBackendPath: string = '';
+  let cxxParserCacheDir: string = '';
 
   beforeEach(() => {
     tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'terra-ut-'));
-    cppastBackendPath = path.join(__dirname, '..', 'cxx', 'cppast_backend');
+    cxxParserCacheDir = path.join(tmpDir, 'cxx_parser');
   });
 
   afterEach(() => {
@@ -33,20 +33,26 @@ struct AAA {
 `
       );
 
+      let checkSum = generateChecksum([file1Path]);
+      let preProcessParseFilesDir = path.join(
+        cxxParserCacheDir,
+        `preProcess@${checkSum}`
+      );
+
       // TODO(littlegnal): Should move the tmp/*.h to the build dir in the future
       const expectedJson = `
       [
         {
           "__TYPE":"CXXFile",
-          "file_path":"${cppastBackendPath}/tmp/file1.h",
+          "file_path":"${preProcessParseFilesDir}/file1.h",
           "nodes":[
             {
               "__TYPE":"Struct",
               "attributes":[],
               "base_clazzs":[],
               "comment":"",
               "constructors":[],
-              "file_path":"${cppastBackendPath}/tmp/file1.h",
+              "file_path":"${preProcessParseFilesDir}/file1.h",
               "member_variables":[
                 {
                   "__TYPE":"MemberVariable",
@@ -66,21 +72,17 @@ struct AAA {
               "methods":[],
               "name":"AAA",
               "namespaces":[],
-              "parent_name":"${cppastBackendPath}/tmp/file1.h",
+              "parent_name":"${preProcessParseFilesDir}/file1.h",
               "source":""
             }
           ]
         }
       ]
       `;
 
-      let json = dumpCXXAstJson(
-        new TerraContext(tmpDir),
-        [],
-        [],
-        [file1Path],
-        []
-      );
+      let json = dumpCXXAstJson(new TerraContext(tmpDir), [], [file1Path], []);
+
+      expect(fs.existsSync(preProcessParseFilesDir)).toBe(true);
 
       // Use `JSON.parse` to parse the json string to avoid the format issue
       expect(JSON.parse(json)).toEqual(JSON.parse(expectedJson));

cxx-parser/__tests__/unit_test/cxx_parser.test.ts

@@ -81,6 +81,10 @@ describe('cxx_parser', () => {
         cppastBackendBuildDir,
         `dump_json_${checkSum}.json`
       );
+      let preProcessParseFilesDir = path.join(
+        cppastBackendBuildDir,
+        `preProcess@${checkSum}`
+      );
 
       (execSync as jest.Mock).mockImplementationOnce(() => {
         // Simulate generate the ast json file after run the bash script
@@ -92,12 +96,11 @@ describe('cxx_parser', () => {
       let json = dumpCXXAstJson(
         new TerraContext(tmpDir),
         [],
-        [],
         [file1Path, file2Path],
         []
       );
 
-      let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`;
+      let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`;
       expect(execSync).toHaveBeenCalledWith(expectedBashScript, {
         encoding: 'utf8',
         stdio: 'inherit',
@@ -152,6 +155,10 @@ describe('cxx_parser', () => {
         cppastBackendBuildDir,
         `dump_json_${checkSum}.json`
       );
+      let preProcessParseFilesDir = path.join(
+        cppastBackendBuildDir,
+        `preProcess@${checkSum}`
+      );
 
       let isBashCalled = false;
 
@@ -167,12 +174,11 @@ describe('cxx_parser', () => {
       let json = dumpCXXAstJson(
         new TerraContext(tmpDir, '', '', true, false),
         [],
-        [],
         [file1Path, file2Path],
         []
       );
 
-      let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`;
+      let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`;
       expect(execSync).toHaveBeenCalledWith(expectedBashScript, {
         encoding: 'utf8',
         stdio: 'inherit',
@@ -229,6 +235,7 @@ describe('cxx_parser', () => {
         cppastBackendBuildDir,
         `dump_json_${checkSum}.json`
       );
+
       // Simulate cached ast json file exists
       fs.writeFileSync(jsonFilePath, expectedJson);
 
@@ -242,7 +249,6 @@ describe('cxx_parser', () => {
       let json = dumpCXXAstJson(
         new TerraContext(tmpDir),
         [],
-        [],
         [file1Path, file2Path],
         []
       );

cxx-parser/cxx/cppast_backend/src/main.cc

@@ -31,6 +31,7 @@ int main(int argc, char **argv) {
     option_list.add_options()
         ("include-header-dirs", "The include C++ headers directories, split with \",\"", cxxopts::value<std::string>())
         ("output-dir", "The output directory, or output to ./build/iris-ast/ by default", cxxopts::value<std::string>())
+        ("pre-process-dir", "The pre-process parse files directory", cxxopts::value<std::string>())
         ("visit-headers", "The C++ headers to be visited, split with \",\"", cxxopts::value<std::string>())
         ("custom-headers", "The custom C++ headers to be visited, split with \",\"", cxxopts::value<std::string>())
         ("defines-macros", "Custom macros, split with \",\"", cxxopts::value<std::string>())
@@ -41,6 +42,7 @@ int main(int argc, char **argv) {
 
   std::string output_dir = "";
   std::string visit_headers = "";
+  std::string pre_process_dir = "";
   std::vector<std::string> include_header_dirs;
   std::vector<std::string> visit_files;
   std::vector<std::string> custom_headers;
@@ -66,9 +68,15 @@ int main(int argc, char **argv) {
     output_dir = std::string(std::filesystem::absolute(tmp_out).c_str());
   }
 
-  std::filesystem::path tmp_path = std::filesystem::current_path() / "tmp";
-  // make sure pre_processed_files as the first of the headers
-  include_header_dirs.push_back(tmp_path.c_str());
+  if (parse_result.count("pre-process-dir")) {
+    pre_process_dir = parse_result["pre-process-dir"].as<std::string>();
+  } else {
+    std::cerr << "The pre-process-dir option is missing." << std::endl;
+    return -1;
+  }
+
+  // make sure `pre_process_dir` as the first of the headers
+  include_header_dirs.push_back(pre_process_dir);
 
   if (parse_result.count("include-header-dirs")) {
     auto include_system_dir =
@@ -116,6 +124,7 @@ int main(int argc, char **argv) {
   }
 
   std::vector<std::string> pre_processed_files;
+  std::filesystem::path tmp_path = pre_process_dir;
   PreProcessVisitFiles(tmp_path, visit_files, pre_processed_files,
                        is_dump_json);
 

cxx-parser/src/cxx_parser.ts

@@ -35,13 +35,16 @@ export function getCppAstBackendDir() {
 export function dumpCXXAstJson(
   terraContext: TerraContext,
   includeHeaderDirs: string[],
-  customHeaders: string[],
   parseFiles: string[],
   defines: string[]
 ): string {
   let parseFilesChecksum = generateChecksum(parseFiles);
 
   let buildDir = getBuildDir(terraContext);
+  let preProcessParseFilesDir = path.join(
+    buildDir,
+    `preProcess@${parseFilesChecksum}`
+  );
 
   let agora_rtc_ast_dir_path = getCppAstBackendDir();
 
@@ -76,12 +79,10 @@ export function dumpCXXAstJson(
   let definess = defines.join(',');
   bashArgs += ` --defines-macros=\"${definess}\"`;
 
-  if (customHeaders) {
-    bashArgs += ` --custom-headers=${customHeaders.join(',')}`;
-  }
-
   bashArgs += ` --output-dir=${outputJsonPath}`;
 
+  bashArgs += ` --pre-process-dir=${preProcessParseFilesDir}`;
+
   bashArgs += ` --dump-json`;
 
   let buildScript = `bash ${build_shell_path} \"${buildDir}\" \"${bashArgs}\"`;
@@ -122,7 +123,6 @@ export function CXXParser(
   let jsonContent = dumpCXXAstJson(
     terraContext,
     cxxParserConfigs.includeHeaderDirs,
-    cxxParserConfigs.customHeaders,
     parseFiles,
     cxxParserConfigs.definesMacros
   );

cxx-parser/src/cxx_parser_configs.ts

@@ -15,7 +15,6 @@ export interface CXXParserConfigs {
   includeHeaderDirs: string[];
   definesMacros: string[];
   parseFiles: ParseFilesConfig;
-  customHeaders: string[];
 }
 
 export class CXXParserConfigs {
@@ -39,11 +38,6 @@ export class CXXParserConfigs {
           })
           .flat(1),
       },
-      customHeaders: (original.customHeaders ?? [])
-        .map((it) => {
-          return _resolvePaths(it, configDir);
-        })
-        .flat(1),
     };
   }
 }