MAT-7784: Associate comments and inline comments with cql definitions

src/CqlAntlr.ts

@@ -1,4 +1,4 @@
-import { CharStreams, CommonTokenStream } from "antlr4ts";
+import {BufferedTokenStream, CharStreams, CommonTokenStream} from "antlr4ts";
 import { CodePointCharStream } from "antlr4ts/CodePointCharStream";
 import { ParseTreeWalker } from "antlr4ts/tree";
 import { cqlLexer, cqlParser, LibraryContext, cqlListener } from "../generated";
@@ -12,7 +12,12 @@ class CqlAntlr {
   parse(): CqlResult {
     const result = CqlAntlr.initCqlResult();
     const tree: LibraryContext = this.buildTree(result);
-    const listener: cqlListener = new CqlAntlrListener(result);
+    const charStream: CodePointCharStream = CharStreams.fromString(this.cql);
+    const lexer: cqlLexer = new cqlLexer(charStream);
+    const bufferedTokenStream = new BufferedTokenStream(lexer);
+    bufferedTokenStream.fill()
+
+    const listener: cqlListener = new CqlAntlrListener(result, bufferedTokenStream);
     ParseTreeWalker.DEFAULT.walk(listener, tree);
     /**
      * Disabled. Only partially implemented and would be

src/CqlAntlrListener.ts

@@ -3,6 +3,7 @@ import {
   CodeDefinitionContext,
   CodesystemDefinitionContext,
   ContextDefinitionContext,
+  cqlLexer,
   cqlListener,
   ExpressionDefinitionContext,
   IncludeDefinitionContext,
@@ -31,9 +32,14 @@ import CqlIdentifier from "./dto/CqlIdentifier";
 import CqlIdentifierCreator from "./CqlIdentifierCreator";
 import CqlRetrieve from "./dto/CqlRetrieve";
 import CqlRetrieveCreator from "./CqlRetrieveCreator";
+import {BufferedTokenStream } from "antlr4ts";
 
 export default class CqlAntlrListener implements cqlListener {
-  constructor(private cqlResult: CqlResult) {}
+  // save bufferedTokenStream from lexer
+  bufferedTokenStream: BufferedTokenStream;
+  constructor(private cqlResult: CqlResult, tokenStream: BufferedTokenStream) {
+    this.bufferedTokenStream = tokenStream;
+  }
 
   enterLibraryDefinition(ctx: LibraryDefinitionContext): void {
     const cqlVersionCreator = new CqlVersionCreator(ctx);
@@ -102,6 +108,19 @@ export default class CqlAntlrListener implements cqlListener {
       new CqlExpressionDefinitionCreator(ctx).buildDao();
 
     if (cqlCode) {
+      if (ctx.start.inputStream) {
+        const hiddenTokens = this.bufferedTokenStream.getHiddenTokensToLeft(ctx.start.tokenIndex, cqlLexer.HIDDEN)
+        let comment = "";
+        hiddenTokens.forEach((token) => {
+          if (token.text){
+            comment += token.text;
+          }
+        })
+        comment = comment.trim();
+        if (comment){
+          cqlCode.comment = comment;
+        }
+      }
       this.cqlResult.expressionDefinitions.push(cqlCode);
     }
   }

src/dto/CqlExpressionDefinition.ts

@@ -4,4 +4,5 @@ export default interface CqlExpressionDefinition extends CqlText {
   expression?: string;
   expressionClass?: string;
   hits: number;
+  comment?: string;
 }

test/CqlAntlr.test.ts

@@ -29,6 +29,15 @@ describe("test antlr", () => {
       expect(def.name).toBeDefined();
     });
   });
+  it("Can now parse comments and associate them with functions", () => {
+    const cqlAntlr = new CqlAntlr(simpleDefinitionCql);
+    const cqlResult: CqlResult = cqlAntlr.parse();
+    const expressions = cqlResult.expressionDefinitions;
+    expect(expressions.length).toEqual(4);
+    expect(expressions[0].comment).toEqual("// ehnicity comment");
+    expect(expressions[1].comment).toContain("multi line");
+    expect(expressions[2].comment).toBeUndefined();
+  });
   it("parse fhir cql", () => {
     const cqlAntlr = new CqlAntlr(fhirTestCql);
 

test/commentTest.ts

@@ -0,0 +1,86 @@
+export const commentTestCql = `cql test is library T1 version '0.0.000'
+// ignore
+using QDM version '5.6'
+
+// ignore
+codesystem "SNOMEDCT": 'urn:oid:2.16.840.1.113883.6.96'
+valueset "Active Bleeding": 'urn:oid:2.16.840.1.113762.1.4.1206.28' 
+
+// ignore
+parameter "Measurement Period" Interval<DateTime>
+
+context Patient
+// outside 
+define "SDE Ethnicity": 
+  //inside
+  ["Patient Characteristic Ethnicity": "Ethnicity"]
+
+ 
+define "SDE Payer": 
+//inside
+  ["Patient Characteristic Payer": "Payer Type"]
+  
+  
+// define "SDE Payer1":
+// //inside
+//   ["Patient Characteristic Payer": "Payer Type"]
+
+// SDE Race
+define "SDE Race":
+/* ignore   */
+  ["Patient Characteristic Race": "Race"]
+
+/* SDE Sex */
+
+define "SDE Sex":
+  ["Patient Characteristic Sex": "ONC Administrative Sex"]
+
+/*
+asdfasdf
+*/
+define "Initial Population":
+  "Qualifying Encounters"
+
+define "Qualifying Encounters":
+  ( ["Encounter, Performed": "Encounter Inpatient"]
+    union ["Encounter, Performed": "Emergency Department Visit"]
+    union ["Encounter, Performed": "Acute Inpatient"]
+    union ["Encounter, Performed": "Active Bleeding"]
+    union ["Encounter, Performed": "Observation Services"] ) Encounter
+    where Encounter.relevantPeriod ends during "Measurement Period"
+
+define "Denominator":
+  ["Encounter, Performed": "Acute Inpatient"] AcuteInpatient
+    where AcuteInpatient.lengthOfStay > 10 days
+
+define "Numerator":
+  ["Encounter, Performed": "Active Bleeding"] ActiveBleeding
+    where ActiveBleeding.relevantPeriod overlaps day of "Measurement Period"
+    
+    /*
+        outside of define
+    */
+define "Denominator 2":
+  ["Encounter, Performed": "Encounter Inpatient"] EncounterInpatient
+    where EncounterInpatient.lengthOfStay > 15 days
+
+// outside of define
+define "Numerator 2":
+  ["Encounter, Performed": "Emergency Department Visit"]
+  // inside of definition to ignore
+  EmergencyDepartmentVisit
+    where EmergencyDepartmentVisit.lengthOfStay > 15 days
+    
+/*
+multiline comment outside of a funciton with multiple
+rows
+*/
+
+define function "Denominator Observation"(Encounter "Encounter, Performed" ):
+  // inside of definition to ignore
+  duration in hours of Encounter.relevantPeriod
+
+// comment outside of function
+define function "Numerator Observation"(Encounter "Encounter, Performed" ):
+  duration in hours of Encounter.relevantPeriod
+`

test/testCql.ts

@@ -4,10 +4,12 @@ const simpleDefinitionCql = `
   using QICore version '4.1.1'
 
   context Patient
-
+  // ehnicity comment
   define "SDE Ethnicity":
     SDE."SDE Ethnicity"
 
+  /* multi line 
+  */
   define InitialPopulation:
     "VTE Prophylaxis by Medication Administered or Device Applied"