From b061a112260dd8070552ef44482b37a86ec1eb23 Mon Sep 17 00:00:00 2001 From: Adam Ogiba Date: Mon, 20 Dec 2021 03:06:19 +0100 Subject: [PATCH 1/3] Add support for keeping single line comments from typescript source code --- parse_node.ts | 14 +- parse_node/parse_node_with_comments.ts | 180 +++++++++++++++++++++++++ project/tsgd_json.ts | 4 + tests/test.ts | 37 +++-- 4 files changed, 222 insertions(+), 13 deletions(-) create mode 100644 parse_node/parse_node_with_comments.ts diff --git a/parse_node.ts b/parse_node.ts index b1e1ac62..bec7755d 100644 --- a/parse_node.ts +++ b/parse_node.ts @@ -63,6 +63,7 @@ import { parseTemplateExpression } from "./parse_node/parse_template_expression" import { parseNoSubstitutionTemplateLiteral } from "./parse_node/parse_no_substitution_template_expression" import { AssetSourceFile } from "./project/assets/asset_source_file" import { LibraryFunctionName } from "./parse_node/library_functions" +import { parseNodeWithComments } from "./parse_node/parse_node_with_comments" export type ParseState = { isConstructor: boolean @@ -82,6 +83,7 @@ export type ParseState = { usages: Map sourceFile: ts.SourceFile sourceFileAsset: AssetSourceFile + commentsStack?: ts.CommentRange[] } export enum ExtraLineType { @@ -245,7 +247,7 @@ export function combine(args: { } } -export const parseNode = ( +export const parseNodeWithoutComments = ( genericNode: ts.Node, props: ParseState ): ParseNodeType => { @@ -500,3 +502,13 @@ Try rewriting it, or opening an issue on the ts2gd GitHub repo. } } } + +export const parseNode = ( + genericNode: ts.Node, + props: ParseState +): ParseNodeType => { + // TsGdProjectClass.Paths is undefined when running tests + return (TsGdProjectClass.Paths ?? {}).removeComments + ? parseNodeWithoutComments(genericNode, props) + : parseNodeWithComments(genericNode, props) +} diff --git a/parse_node/parse_node_with_comments.ts b/parse_node/parse_node_with_comments.ts new file mode 100644 index 00000000..2ca84a50 --- /dev/null +++ b/parse_node/parse_node_with_comments.ts @@ -0,0 +1,180 @@ +import ts from "typescript" +import { + ParseNodeType, + parseNodeWithoutComments, + ParseState, +} from "../parse_node" +import { Test } from "../tests/test" + +const getBackwardsClosestNewLinesCount = (text: string, pos: number) => { + let count = 0 + + for (let i = pos - 1; i >= 0; i--) { + const char = text.charAt(i) + if (char === "\t" || char === " ") { + continue + } + + if (char === "\n") { + count++ + continue + } + + break + } + + return count +} + +export const parseNodeWithComments = ( + node: ts.Node, + props: ParseState +): ParseNodeType => { + if (ts.isSourceFile(node)) { + return parseNodeWithoutComments(node, props) + } + + let leadingComments = + ts + .getLeadingCommentRanges( + node.getSourceFile().getFullText(), + node.getFullStart() + ) + ?.filter((v) => v.kind === ts.SyntaxKind.SingleLineCommentTrivia) ?? [] + + let tarilingComments = + ts + .getTrailingCommentRanges( + node.getSourceFile().getFullText(), + node.getFullStart() + node.getFullWidth() + ) + ?.filter((v) => v.kind === ts.SyntaxKind.SingleLineCommentTrivia) ?? [] + + let commentStackUnwind = props.commentsStack?.length ?? 0 + + if (leadingComments.length > 0 || tarilingComments.length > 0) { + if (!props.commentsStack) { + props.commentsStack = [] + } + + leadingComments = leadingComments.filter( + (v) => + !props.commentsStack!.find((c) => c.pos === v.pos && c.end === v.end) + ) + tarilingComments = tarilingComments.filter( + (v) => + !props.commentsStack!.find((c) => c.pos === v.pos && c.end === v.end) + ) + + props.commentsStack.push(...leadingComments, ...tarilingComments) + } + + const result = parseNodeWithoutComments(node, props) + + if (leadingComments.length > 0 || tarilingComments.length > 0) { + const fullText = node.getSourceFile().getFullText() + + if (leadingComments.length > 0) { + const leadingCommentsText = leadingComments.map( + (v) => "#" + fullText.slice(v.pos + 2, v.end) + ) + const addFirstNewLineToLeadingComments = + getBackwardsClosestNewLinesCount(fullText, leadingComments[0].pos) > + 1 || !result.content.endsWith("\n") + + result.content = + (addFirstNewLineToLeadingComments ? "\n" : "") + + leadingCommentsText.join("\n") + + "\n" + + result.content + } + + if (tarilingComments.length > 0) { + const trailingCommentsText = tarilingComments.map( + (v) => "#" + fullText.slice(v.pos + 2, v.end) + ) + const addFirstNewLineToTrailingComments = + getBackwardsClosestNewLinesCount(fullText, tarilingComments[0].pos) > 0 + + result.content = + result.content + + (addFirstNewLineToTrailingComments ? "\n" : " ") + + trailingCommentsText.join("\n") + + "\n" + } + + props.commentsStack!.length = commentStackUnwind + } + + return result +} + +export const testComments: Test = { + keepComments: true, + ts: ` +// This is a test class +// abc +class Test extends Area2D { + // This a constructor + constructor() { + // This is a super call + + } + + // this is a method + method() { + // this is a print + print("a") + + // This is an if statement + if (2 == 3) { + // This is an empty block + } + + print(/* this is block comment */"l") + + print( + // this is a call with parameters in multiple lines + "hello", + // this is second parameter + "world" + ) + + // this is a print + // this is a print + // and this is third comment + print("b") + + print("x") // leading comment + } +} + `, + expected: ` +# This file has been autogenerated by ts2gd. DO NOT EDIT! +extends Area2D +class_name Test +# This is a test class +# abc +# This a constructor +func _ready(): + pass +# this is a method +func method(): + # this is a print + print("a") + # This is an if statement + if ((typeof(2) == typeof(3)) and (2 == 3)): + pass + print("l") + print( + # this is a call with parameters in multiple lines + "hello", + # this is second parameter + "world") + # this is a print + # this is a print + # and this is third comment + print("b") + print("x") # leading comment + `, +} diff --git a/project/tsgd_json.ts b/project/tsgd_json.ts index cf9c43ac..4a773cc7 100644 --- a/project/tsgd_json.ts +++ b/project/tsgd_json.ts @@ -31,6 +31,9 @@ export class Paths { /** The path to the Godot repository, e.g. /Users/johnfn/Godot */ godotSourceRepoPath: string | undefined + /** When set to true does not generates comments in .gd code */ + removeComments: boolean + constructor(args: ParsedArgs) { if (args.init) { this.init() @@ -95,6 +98,7 @@ export class Paths { "dynamic" ) + this.removeComments = tsgdJson.removeComments || false this.godotSourceRepoPath = tsgdJson.godotSourceRepoPath || undefined this.tsconfigPath = path.join( diff --git a/tests/test.ts b/tests/test.ts index c488bcfe..6df82818 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -160,6 +160,7 @@ export type Test = { isAutoload?: boolean only?: boolean expectFail?: boolean + keepComments?: boolean } type TestResult = TestResultPass | TestResultFail @@ -191,13 +192,19 @@ const removeCommentLines = (s: string) => { .join("\n") } -const normalize = (s: string) => { - return removeCommentLines(trim(s)) +const normalize = (s: string, keepComments?: boolean) => { + return keepComments ? trim(s) : removeCommentLines(trim(s)) } -const areOutputsEqual = (left: string, right: string) => { - const leftTrimmed = removeCommentLines(trim(left)) - const rightTrimmed = removeCommentLines(trim(right)) +const areOutputsEqual = ( + left: string, + right: string, + keepComments?: boolean +) => { + const leftTrimmed = keepComments ? trim(left) : removeCommentLines(trim(left)) + const rightTrimmed = keepComments + ? trim(right) + : removeCommentLines(trim(right)) return leftTrimmed === rightTrimmed } @@ -208,7 +215,7 @@ const test = ( testFileName: string, path: string ): TestResult => { - const { ts, expected } = props + const { ts, expected, keepComments } = props let compiled: ParseNodeType | null = null let errors: TsGdError[] = [] @@ -277,7 +284,7 @@ ${errors[0].description} } if (typeof expected === "string") { - if (areOutputsEqual(output, expected)) { + if (areOutputsEqual(output, expected, keepComments)) { return { type: "success" } } } else { @@ -300,12 +307,18 @@ ${errors[0].description} for (const actualFile of compiled.files ?? []) { if (actualFile.filePath === expectedFile.fileName) { - if (!areOutputsEqual(actualFile.body, expectedFile.expected)) { + if ( + !areOutputsEqual( + actualFile.body, + expectedFile.expected, + keepComments + ) + ) { return { type: "fail", fileName: actualFile.filePath, - result: normalize(actualFile.body), - expected: normalize(expectedFile.expected), + result: normalize(actualFile.body, keepComments), + expected: normalize(expectedFile.expected, keepComments), name, expectFail: props.expectFail ?? false, path, @@ -337,8 +350,8 @@ ${errors[0].description} return { type: "fail", - result: normalize(output), - expected: normalize(expected), + result: normalize(output, keepComments), + expected: normalize(expected, keepComments), name, expectFail: props.expectFail ?? false, path, From c944d62d2d5b143188541db326f35701180e3da8 Mon Sep 17 00:00:00 2001 From: Adam Ogiba Date: Mon, 20 Dec 2021 14:06:43 +0100 Subject: [PATCH 2/3] Refactor names and add comments --- parse_node.ts | 4 +-- parse_node/parse_node_with_comments.ts | 40 +++++++++++++++++--------- project/tsgd_json.ts | 2 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/parse_node.ts b/parse_node.ts index bec7755d..1f5d4485 100644 --- a/parse_node.ts +++ b/parse_node.ts @@ -63,7 +63,7 @@ import { parseTemplateExpression } from "./parse_node/parse_template_expression" import { parseNoSubstitutionTemplateLiteral } from "./parse_node/parse_no_substitution_template_expression" import { AssetSourceFile } from "./project/assets/asset_source_file" import { LibraryFunctionName } from "./parse_node/library_functions" -import { parseNodeWithComments } from "./parse_node/parse_node_with_comments" +import { parseComments } from "./parse_node/parse_node_with_comments" export type ParseState = { isConstructor: boolean @@ -510,5 +510,5 @@ export const parseNode = ( // TsGdProjectClass.Paths is undefined when running tests return (TsGdProjectClass.Paths ?? {}).removeComments ? parseNodeWithoutComments(genericNode, props) - : parseNodeWithComments(genericNode, props) + : parseComments(genericNode, props) } diff --git a/parse_node/parse_node_with_comments.ts b/parse_node/parse_node_with_comments.ts index 2ca84a50..7e53fb8e 100644 --- a/parse_node/parse_node_with_comments.ts +++ b/parse_node/parse_node_with_comments.ts @@ -6,7 +6,7 @@ import { } from "../parse_node" import { Test } from "../tests/test" -const getBackwardsClosestNewLinesCount = (text: string, pos: number) => { +const countNewlinesBeforePosition = (text: string, pos: number) => { let count = 0 for (let i = pos - 1; i >= 0; i--) { @@ -26,7 +26,7 @@ const getBackwardsClosestNewLinesCount = (text: string, pos: number) => { return count } -export const parseNodeWithComments = ( +export const parseComments = ( node: ts.Node, props: ParseState ): ParseNodeType => { @@ -34,6 +34,7 @@ export const parseNodeWithComments = ( return parseNodeWithoutComments(node, props) } + // gather up all leading and trailing comments from a node let leadingComments = ts .getLeadingCommentRanges( @@ -42,7 +43,7 @@ export const parseNodeWithComments = ( ) ?.filter((v) => v.kind === ts.SyntaxKind.SingleLineCommentTrivia) ?? [] - let tarilingComments = + let trailingComments = ts .getTrailingCommentRanges( node.getSourceFile().getFullText(), @@ -50,28 +51,36 @@ export const parseNodeWithComments = ( ) ?.filter((v) => v.kind === ts.SyntaxKind.SingleLineCommentTrivia) ?? [] - let commentStackUnwind = props.commentsStack?.length ?? 0 + let lengthBeforeParsingThisNode = props.commentsStack?.length ?? 0 - if (leadingComments.length > 0 || tarilingComments.length > 0) { + if (leadingComments.length > 0 || trailingComments.length > 0) { if (!props.commentsStack) { props.commentsStack = [] } + /* all nodes in the same line report the same comments + we need to take only first of the node into account and + ignore the rest + + example code returning the same commment multiple times: + // some comment + myFunc(1 + 2, someVar) + */ leadingComments = leadingComments.filter( (v) => !props.commentsStack!.find((c) => c.pos === v.pos && c.end === v.end) ) - tarilingComments = tarilingComments.filter( + trailingComments = trailingComments.filter( (v) => !props.commentsStack!.find((c) => c.pos === v.pos && c.end === v.end) ) - props.commentsStack.push(...leadingComments, ...tarilingComments) + props.commentsStack.push(...leadingComments, ...trailingComments) } const result = parseNodeWithoutComments(node, props) - if (leadingComments.length > 0 || tarilingComments.length > 0) { + if (leadingComments.length > 0 || trailingComments.length > 0) { const fullText = node.getSourceFile().getFullText() if (leadingComments.length > 0) { @@ -79,8 +88,8 @@ export const parseNodeWithComments = ( (v) => "#" + fullText.slice(v.pos + 2, v.end) ) const addFirstNewLineToLeadingComments = - getBackwardsClosestNewLinesCount(fullText, leadingComments[0].pos) > - 1 || !result.content.endsWith("\n") + countNewlinesBeforePosition(fullText, leadingComments[0].pos) > 1 || + !result.content.endsWith("\n") result.content = (addFirstNewLineToLeadingComments ? "\n" : "") + @@ -89,12 +98,12 @@ export const parseNodeWithComments = ( result.content } - if (tarilingComments.length > 0) { - const trailingCommentsText = tarilingComments.map( + if (trailingComments.length > 0) { + const trailingCommentsText = trailingComments.map( (v) => "#" + fullText.slice(v.pos + 2, v.end) ) const addFirstNewLineToTrailingComments = - getBackwardsClosestNewLinesCount(fullText, tarilingComments[0].pos) > 0 + countNewlinesBeforePosition(fullText, trailingComments[0].pos) > 0 result.content = result.content + @@ -103,7 +112,10 @@ export const parseNodeWithComments = ( "\n" } - props.commentsStack!.length = commentStackUnwind + props.commentsStack = props.commentsStack?.slice( + 0, + lengthBeforeParsingThisNode + ) } return result diff --git a/project/tsgd_json.ts b/project/tsgd_json.ts index 4a773cc7..87c033d1 100644 --- a/project/tsgd_json.ts +++ b/project/tsgd_json.ts @@ -31,7 +31,7 @@ export class Paths { /** The path to the Godot repository, e.g. /Users/johnfn/Godot */ godotSourceRepoPath: string | undefined - /** When set to true does not generates comments in .gd code */ + /** Should we strip comments from the godot output? */ removeComments: boolean constructor(args: ParsedArgs) { From 0d693b368c41294b0619f15270b41291cbbf5cc7 Mon Sep 17 00:00:00 2001 From: Adam Ogiba Date: Mon, 20 Dec 2021 14:35:10 +0100 Subject: [PATCH 3/3] Add options to parseNode instead of warpping it in another function for comments --- parse_node.ts | 23 +++++++++++------------ parse_node/parse_node_with_comments.ts | 10 +++------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/parse_node.ts b/parse_node.ts index 1f5d4485..8bd6e21f 100644 --- a/parse_node.ts +++ b/parse_node.ts @@ -247,10 +247,19 @@ export function combine(args: { } } -export const parseNodeWithoutComments = ( +export const parseNode = ( genericNode: ts.Node, - props: ParseState + props: ParseState, + options?: { ignoreComments?: boolean } ): ParseNodeType => { + // TsGdProjectClass.Paths is undefined when running tests + if ( + !(TsGdProjectClass.Paths ?? {}).removeComments && + !options?.ignoreComments + ) { + return parseComments(genericNode, props) + } + switch (genericNode.kind) { case SyntaxKind.SourceFile: return parseSourceFile(genericNode as ts.SourceFile, props) @@ -502,13 +511,3 @@ Try rewriting it, or opening an issue on the ts2gd GitHub repo. } } } - -export const parseNode = ( - genericNode: ts.Node, - props: ParseState -): ParseNodeType => { - // TsGdProjectClass.Paths is undefined when running tests - return (TsGdProjectClass.Paths ?? {}).removeComments - ? parseNodeWithoutComments(genericNode, props) - : parseComments(genericNode, props) -} diff --git a/parse_node/parse_node_with_comments.ts b/parse_node/parse_node_with_comments.ts index 7e53fb8e..3f1658d7 100644 --- a/parse_node/parse_node_with_comments.ts +++ b/parse_node/parse_node_with_comments.ts @@ -1,9 +1,5 @@ import ts from "typescript" -import { - ParseNodeType, - parseNodeWithoutComments, - ParseState, -} from "../parse_node" +import { parseNode, ParseNodeType, ParseState } from "../parse_node" import { Test } from "../tests/test" const countNewlinesBeforePosition = (text: string, pos: number) => { @@ -31,7 +27,7 @@ export const parseComments = ( props: ParseState ): ParseNodeType => { if (ts.isSourceFile(node)) { - return parseNodeWithoutComments(node, props) + return parseNode(node, props, { ignoreComments: true }) } // gather up all leading and trailing comments from a node @@ -78,7 +74,7 @@ export const parseComments = ( props.commentsStack.push(...leadingComments, ...trailingComments) } - const result = parseNodeWithoutComments(node, props) + const result = parseNode(node, props, { ignoreComments: true }) if (leadingComments.length > 0 || trailingComments.length > 0) { const fullText = node.getSourceFile().getFullText()