From ff2bc6583ff820a3e93f4ae5723ab49373455653 Mon Sep 17 00:00:00 2001 From: littleGnAl Date: Wed, 15 Nov 2023 13:21:26 +0800 Subject: [PATCH 1/3] feat: Move tmp preprocess files to /.terra/cxx_parser/preProcess@ --- .../cxx_parser.integration.test.ts | 28 ++++++++++--------- .../__tests__/unit_test/cxx_parser.test.ts | 22 +++++++++------ cxx-parser/cxx/cppast_backend/src/main.cc | 15 ++++++++-- cxx-parser/src/cxx_parser.ts | 12 ++++---- cxx-parser/src/cxx_parser_configs.ts | 6 ---- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts index 207c3aa..95d5a94 100644 --- a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts +++ b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts @@ -4,16 +4,16 @@ import path from 'path'; import { TerraContext } from '@agoraio-extensions/terra-core'; -import { CXXParser, dumpCXXAstJson } from '../src/cxx_parser'; +import { CXXParser, dumpCXXAstJson, generateChecksum } from '../src/cxx_parser'; import { CXXFile, Struct } from '../src/cxx_terra_node'; describe('cxx_parser', () => { let tmpDir: string = ''; - let cppastBackendPath: string = ''; + let cxxParserCacheDir: string = ''; beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'terra-ut-')); - cppastBackendPath = path.join(__dirname, '..', 'cxx', 'cppast_backend'); + cxxParserCacheDir = path.join(tmpDir, 'cxx_parser'); }); afterEach(() => { @@ -33,12 +33,18 @@ struct AAA { ` ); + let checkSum = generateChecksum([file1Path]); + let preProcessParseFilesDir = path.join( + cxxParserCacheDir, + `preProcess@${checkSum}` + ); + // TODO(littlegnal): Should move the tmp/*.h to the build dir in the future const expectedJson = ` [ { "__TYPE":"CXXFile", - "file_path":"${cppastBackendPath}/tmp/file1.h", + "file_path":"${preProcessParseFilesDir}/file1.h", "nodes":[ { "__TYPE":"Struct", @@ -46,7 +52,7 @@ struct AAA { "base_clazzs":[], "comment":"", "constructors":[], - "file_path":"${cppastBackendPath}/tmp/file1.h", + "file_path":"${preProcessParseFilesDir}/file1.h", "member_variables":[ { "__TYPE":"MemberVariable", @@ -66,7 +72,7 @@ struct AAA { "methods":[], "name":"AAA", "namespaces":[], - "parent_name":"${cppastBackendPath}/tmp/file1.h", + "parent_name":"${preProcessParseFilesDir}/file1.h", "source":"" } ] @@ -74,13 +80,9 @@ struct AAA { ] `; - let json = dumpCXXAstJson( - new TerraContext(tmpDir), - [], - [], - [file1Path], - [] - ); + let json = dumpCXXAstJson(new TerraContext(tmpDir), [], [], [file1Path]); + + expect(fs.existsSync(preProcessParseFilesDir)).toBe(true); // Use `JSON.parse` to parse the json string to avoid the format issue expect(JSON.parse(json)).toEqual(JSON.parse(expectedJson)); diff --git a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts index aaded87..6ee00d9 100644 --- a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts +++ b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts @@ -81,6 +81,10 @@ describe('cxx_parser', () => { cppastBackendBuildDir, `dump_json_${checkSum}.json` ); + let preProcessParseFilesDir = path.join( + cppastBackendBuildDir, + `preProcess@${checkSum}` + ); (execSync as jest.Mock).mockImplementationOnce(() => { // Simulate generate the ast json file after run the bash script @@ -93,11 +97,10 @@ describe('cxx_parser', () => { new TerraContext(tmpDir), [], [], - [file1Path, file2Path], - [] + [file1Path, file2Path] ); - let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`; + let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: 'utf8', stdio: 'inherit', @@ -152,6 +155,10 @@ describe('cxx_parser', () => { cppastBackendBuildDir, `dump_json_${checkSum}.json` ); + let preProcessParseFilesDir = path.join( + cppastBackendBuildDir, + `preProcess@${checkSum}` + ); let isBashCalled = false; @@ -168,11 +175,10 @@ describe('cxx_parser', () => { new TerraContext(tmpDir, '', '', true, false), [], [], - [file1Path, file2Path], - [] + [file1Path, file2Path] ); - let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --dump-json"`; + let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: 'utf8', stdio: 'inherit', @@ -229,6 +235,7 @@ describe('cxx_parser', () => { cppastBackendBuildDir, `dump_json_${checkSum}.json` ); + // Simulate cached ast json file exists fs.writeFileSync(jsonFilePath, expectedJson); @@ -243,8 +250,7 @@ describe('cxx_parser', () => { new TerraContext(tmpDir), [], [], - [file1Path, file2Path], - [] + [file1Path, file2Path] ); expect(execSync).not.toHaveBeenCalled(); diff --git a/cxx-parser/cxx/cppast_backend/src/main.cc b/cxx-parser/cxx/cppast_backend/src/main.cc index c046d5e..57b03d1 100644 --- a/cxx-parser/cxx/cppast_backend/src/main.cc +++ b/cxx-parser/cxx/cppast_backend/src/main.cc @@ -31,6 +31,7 @@ int main(int argc, char **argv) { option_list.add_options() ("include-header-dirs", "The include C++ headers directories, split with \",\"", cxxopts::value()) ("output-dir", "The output directory, or output to ./build/iris-ast/ by default", cxxopts::value()) + ("pre-process-dir", "The pre-process parse files directory", cxxopts::value()) ("visit-headers", "The C++ headers to be visited, split with \",\"", cxxopts::value()) ("custom-headers", "The custom C++ headers to be visited, split with \",\"", cxxopts::value()) ("defines-macros", "Custom macros, split with \",\"", cxxopts::value()) @@ -41,6 +42,7 @@ int main(int argc, char **argv) { std::string output_dir = ""; std::string visit_headers = ""; + std::string pre_process_dir = ""; std::vector include_header_dirs; std::vector visit_files; std::vector custom_headers; @@ -66,9 +68,15 @@ int main(int argc, char **argv) { output_dir = std::string(std::filesystem::absolute(tmp_out).c_str()); } - std::filesystem::path tmp_path = std::filesystem::current_path() / "tmp"; - // make sure pre_processed_files as the first of the headers - include_header_dirs.push_back(tmp_path.c_str()); + if (parse_result.count("pre-process-dir")) { + pre_process_dir = parse_result["pre-process-dir"].as(); + } else { + std::cerr << "The pre-process-dir option is missing." << std::endl; + return -1; + } + + // make sure `pre_process_dir` as the first of the headers + include_header_dirs.push_back(pre_process_dir); if (parse_result.count("include-header-dirs")) { auto include_system_dir = @@ -116,6 +124,7 @@ int main(int argc, char **argv) { } std::vector pre_processed_files; + std::filesystem::path tmp_path = pre_process_dir; PreProcessVisitFiles(tmp_path, visit_files, pre_processed_files, is_dump_json); diff --git a/cxx-parser/src/cxx_parser.ts b/cxx-parser/src/cxx_parser.ts index fe2b320..7d9a429 100644 --- a/cxx-parser/src/cxx_parser.ts +++ b/cxx-parser/src/cxx_parser.ts @@ -35,13 +35,16 @@ export function getCppAstBackendDir() { export function dumpCXXAstJson( terraContext: TerraContext, includeHeaderDirs: string[], - customHeaders: string[], parseFiles: string[], defines: string[] ): string { let parseFilesChecksum = generateChecksum(parseFiles); let buildDir = getBuildDir(terraContext); + let preProcessParseFilesDir = path.join( + buildDir, + `preProcess@${parseFilesChecksum}` + ); let agora_rtc_ast_dir_path = getCppAstBackendDir(); @@ -76,12 +79,10 @@ export function dumpCXXAstJson( let definess = defines.join(','); bashArgs += ` --defines-macros=\"${definess}\"`; - if (customHeaders) { - bashArgs += ` --custom-headers=${customHeaders.join(',')}`; - } - bashArgs += ` --output-dir=${outputJsonPath}`; + bashArgs += ` --pre-process-dir=${preProcessParseFilesDir}`; + bashArgs += ` --dump-json`; let buildScript = `bash ${build_shell_path} \"${buildDir}\" \"${bashArgs}\"`; @@ -122,7 +123,6 @@ export function CXXParser( let jsonContent = dumpCXXAstJson( terraContext, cxxParserConfigs.includeHeaderDirs, - cxxParserConfigs.customHeaders, parseFiles, cxxParserConfigs.definesMacros ); diff --git a/cxx-parser/src/cxx_parser_configs.ts b/cxx-parser/src/cxx_parser_configs.ts index edee2f6..9ccb8a6 100644 --- a/cxx-parser/src/cxx_parser_configs.ts +++ b/cxx-parser/src/cxx_parser_configs.ts @@ -15,7 +15,6 @@ export interface CXXParserConfigs { includeHeaderDirs: string[]; definesMacros: string[]; parseFiles: ParseFilesConfig; - customHeaders: string[]; } export class CXXParserConfigs { @@ -39,11 +38,6 @@ export class CXXParserConfigs { }) .flat(1), }, - customHeaders: (original.customHeaders ?? []) - .map((it) => { - return _resolvePaths(it, configDir); - }) - .flat(1), }; } } From 55f7c6d835da04fe856e3126ad39ad6abee70c20 Mon Sep 17 00:00:00 2001 From: littleGnAl Date: Wed, 15 Nov 2023 13:38:04 +0800 Subject: [PATCH 2/3] feat: Move tmp preprocess files to /.terra/cxx_parser/preProcess@ --- .../cxx_parser.integration.test.ts | 2 +- cxx-parser/__tests__/unit_test/cxx_parser.test.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts index 95d5a94..890a5d3 100644 --- a/cxx-parser/__integration_test__/cxx_parser.integration.test.ts +++ b/cxx-parser/__integration_test__/cxx_parser.integration.test.ts @@ -80,7 +80,7 @@ struct AAA { ] `; - let json = dumpCXXAstJson(new TerraContext(tmpDir), [], [], [file1Path]); + let json = dumpCXXAstJson(new TerraContext(tmpDir), [], [file1Path], []); expect(fs.existsSync(preProcessParseFilesDir)).toBe(true); diff --git a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts index 6ee00d9..589b572 100644 --- a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts +++ b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts @@ -96,8 +96,8 @@ describe('cxx_parser', () => { let json = dumpCXXAstJson( new TerraContext(tmpDir), [], - [], - [file1Path, file2Path] + [file1Path, file2Path], + [] ); let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; @@ -174,8 +174,8 @@ describe('cxx_parser', () => { let json = dumpCXXAstJson( new TerraContext(tmpDir, '', '', true, false), [], - [], - [file1Path, file2Path] + [file1Path, file2Path], + [] ); let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; @@ -249,8 +249,8 @@ describe('cxx_parser', () => { let json = dumpCXXAstJson( new TerraContext(tmpDir), [], - [], - [file1Path, file2Path] + [file1Path, file2Path], + [] ); expect(execSync).not.toHaveBeenCalled(); From ec1a4424988a457e4884f59b537a416211c388f2 Mon Sep 17 00:00:00 2001 From: littleGnAl Date: Wed, 15 Nov 2023 13:40:23 +0800 Subject: [PATCH 3/3] feat: Move tmp preprocess files to /.terra/cxx_parser/preProcess@ --- cxx-parser/__tests__/unit_test/cxx_parser.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts index 589b572..d7a23e1 100644 --- a/cxx-parser/__tests__/unit_test/cxx_parser.test.ts +++ b/cxx-parser/__tests__/unit_test/cxx_parser.test.ts @@ -100,7 +100,7 @@ describe('cxx_parser', () => { [] ); - let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; + let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: 'utf8', stdio: 'inherit', @@ -178,7 +178,7 @@ describe('cxx_parser', () => { [] ); - let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --custom-headers= --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; + let expectedBashScript = `bash ${cppastBackendBuildBashPath} \"${cppastBackendBuildDir}\" "--visit-headers=${file1Path},${file2Path} --include-header-dirs= --defines-macros="" --output-dir=${jsonFilePath} --pre-process-dir=${preProcessParseFilesDir} --dump-json"`; expect(execSync).toHaveBeenCalledWith(expectedBashScript, { encoding: 'utf8', stdio: 'inherit',