From 5e6c079a17601c5c78a87122b297f2f9aa4a23a7 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 25 Sep 2024 19:08:06 +0000 Subject: [PATCH 1/3] Add CSV stringification functionality with tests and update dependencies --- docs/genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- packages/auto/genaiscript.d.ts | 45 ++++++++++++++----- packages/core/package.json | 1 + packages/core/src/csv.test.ts | 32 ++++++++++++- packages/core/src/csv.ts | 30 +++++++++---- packages/core/src/genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- packages/core/src/globals.ts | 3 +- packages/core/src/types/prompt_template.d.ts | 45 ++++++++++++++----- .../sample/genaisrc/blog/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- .../sample/genaisrc/node/genaiscript.d.ts | 45 ++++++++++++++----- .../sample/genaisrc/python/genaiscript.d.ts | 45 ++++++++++++++----- .../sample/genaisrc/style/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/aici/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/errors/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/makecode/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/tla/genaiscript.d.ts | 45 ++++++++++++++----- packages/sample/src/vision/genaiscript.d.ts | 45 ++++++++++++++----- packages/vscode/genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- slides/genaisrc/genaiscript.d.ts | 45 ++++++++++++++----- yarn.lock | 5 +++ 23 files changed, 691 insertions(+), 190 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/core/package.json b/packages/core/package.json index 9f28477f86..b1387ac9da 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -32,6 +32,7 @@ "ajv": "^8.17.1", "cross-fetch": "^4.0.0", "csv-parse": "^5.5.6", + "csv-stringify": "^6.5.1", "dotenv": "^16.4.5", "esbuild": "^0.24.0", "fast-xml-parser": "^4.5.0", diff --git a/packages/core/src/csv.test.ts b/packages/core/src/csv.test.ts index 3ebb6a6c46..f61b96d47a 100644 --- a/packages/core/src/csv.test.ts +++ b/packages/core/src/csv.test.ts @@ -1,6 +1,6 @@ import { describe, test, beforeEach } from "node:test" import assert from "node:assert/strict" -import { CSVParse, CSVTryParse, CSVToMarkdown } from "./csv" +import { CSVParse, CSVTryParse, CSVToMarkdown, CSVStringify } from "./csv" describe("CSVParse", () => { test("Parse simple CSV data with default options", () => { @@ -82,3 +82,33 @@ describe("CSVToMarkdown", () => { assert.equal(result, "") }) }) +describe("CSVStringify", () => { + test("Stringify simple CSV data with default options", () => { + const csv = [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" }, + ] + const result = CSVStringify(csv) + const expected = "John,30\nJane,25\n" + assert.equal(result, expected) + }) + test("Stringify simple CSV data with headers", () => { + const csv = [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" }, + ] + const result = CSVStringify(csv, { header: true }) + const expected = "name,age\nJohn,30\nJane,25\n" + assert.equal(result, expected) + }) + + test("Stringify CSV data with custom delimiter", () => { + const csv = [ + { name: "John", age: "30" }, + { name: "Jane", age: "25" }, + ] + const result = CSVStringify(csv, { header: true, delimiter: "|" }) + const expected = "name|age\nJohn|30\nJane|25\n" + assert.equal(result, expected) + }) +}) diff --git a/packages/core/src/csv.ts b/packages/core/src/csv.ts index 9fbf3475dc..855800e79c 100644 --- a/packages/core/src/csv.ts +++ b/packages/core/src/csv.ts @@ -1,7 +1,9 @@ -// Import the CSV parsing function from the csv-parse library +// This module provides functions for parsing and converting CSV data, +// including error handling and conversion to Markdown table format. + import { parse } from "csv-parse/sync" -// Import the TraceOptions interface for logging trace information import { TraceOptions } from "./trace" +import { stringify } from "csv-stringify/sync" /** * Parses a CSV string into an array of objects. @@ -61,6 +63,18 @@ export function CSVTryParse( } } +/** + * Converts an array of objects into a CSV string. + * + * @param csv - An array of objects to be converted into CSV format. + * @param options - Optional configuration for CSV stringification. + * @returns A string representing the CSV formatted data. + */ +export function CSVStringify(csv: object[], options?: CSVStringifyOptions) { + // Convert objects to CSV string using the provided options + return stringify(csv, options) +} + /** * Converts an array of objects into a Markdown table format. * @@ -70,7 +84,7 @@ export function CSVTryParse( * @returns A string representing the CSV data in Markdown table format. */ export function CSVToMarkdown(csv: object[], options?: { headers?: string[] }) { - if (!csv?.length) return "" + if (!csv?.length) return "" // Return empty string if CSV is empty const { headers = Object.keys(csv[0]) } = options || {} const res: string[] = [ @@ -84,11 +98,11 @@ export function CSVToMarkdown(csv: object[], options?: { headers?: string[] }) { const s = v === undefined || v === null ? "" : String(v) // Escape special Markdown characters and format cell content return s - .replace(/\s+$/, "") - .replace(/[\\`*_{}[\]()#+\-.!]/g, (m) => "\\" + m) - .replace(//g, "gt;") - .replace(/\r?\n/g, "
") + .replace(/\s+$/, "") // Trim trailing whitespace + .replace(/[\\`*_{}[\]()#+\-.!]/g, (m) => "\\" + m) // Escape special characters + .replace(//g, "gt;") // Replace '>' with its HTML entity + .replace(/\r?\n/g, "
") // Replace newlines with
}) .join("|")}|` // Join columns with '|' ), diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/core/src/globals.ts b/packages/core/src/globals.ts index d5bea9e6b5..3af428506f 100644 --- a/packages/core/src/globals.ts +++ b/packages/core/src/globals.ts @@ -1,6 +1,6 @@ // Import various parsing and stringifying utilities import { YAMLParse, YAMLStringify } from "./yaml" -import { CSVParse, CSVToMarkdown } from "./csv" +import { CSVParse, CSVToMarkdown, CSVStringify } from "./csv" import { INIParse, INIStringify } from "./ini" import { XMLParse } from "./xml" import { @@ -46,6 +46,7 @@ export function installGlobals() { // Freeze CSV utilities glb.CSV = Object.freeze({ parse: CSVParse, + stringify: CSVStringify, markdownify: CSVToMarkdown, }) diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 0a18293f3f..845a97244e 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1302,26 +1302,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index ef3161baf2..6307f93a15 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1335,26 +1335,51 @@ interface INI { stringify(value: any): string } +interface CSVStringifyOptions { + delimiter?: string + header?: boolean +} + +/** + * Interface representing CSV operations. + */ interface CSV { /** - * Parses a CSV string to an array of objects - * @param text - * @param options + * Parses a CSV string to an array of objects. + * + * @param text - The CSV string to parse. + * @param options - Optional settings for parsing. + * @param options.delimiter - The delimiter used in the CSV string. Defaults to ','. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row. + * @returns An array of objects representing the parsed CSV data. */ parse( text: string, options?: { - delimiter?: string - headers?: string[] + delimiter?: string; + headers?: string[]; } - ): object[] + ): object[]; /** - * Converts an array of object that represents a data table to a markdown table - * @param csv - * @param options + * Converts an array of objects to a CSV string. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for stringifying. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A CSV string representing the data. + */ + stringify(csv: object[], options?: CSVStringifyOptions): string; + + /** + * Converts an array of objects that represents a data table to a markdown table. + * + * @param csv - The array of objects to convert. + * @param options - Optional settings for markdown conversion. + * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the object keys. + * @returns A markdown string representing the data table. */ - markdownify(csv: object[], options?: { headers?: string[] }): string + markdownify(csv: object[], options?: { headers?: string[] }): string; } interface HighlightOptions { diff --git a/yarn.lock b/yarn.lock index 040fdac8f1..2e87d01f3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2132,6 +2132,11 @@ csv-parse@^5.5.6: resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a" integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A== +csv-stringify@^6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.1.tgz#a31837dd35e34787e3c248159c982a21af964d94" + integrity sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ== + cubic2quad@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/cubic2quad/-/cubic2quad-1.2.1.tgz#2442260b72c02ee4b6a2fe998fcc1c4073622286" From 37b8a3840cb91903e27da2498685b668ea26f5f7 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 25 Sep 2024 19:14:42 +0000 Subject: [PATCH 2/3] Remove verbose error logging from token estimation function --- packages/core/src/tokens.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/src/tokens.ts b/packages/core/src/tokens.ts index d48bff8f75..ae81b78e4f 100644 --- a/packages/core/src/tokens.ts +++ b/packages/core/src/tokens.ts @@ -17,8 +17,6 @@ export function estimateTokens(text: string, encoder: TokenEncoder) { // Return the length of the encoded text plus a constant overhead return encoder(text).length + ESTIMATE_TOKEN_OVERHEAD } catch (e) { - // If encoding fails, log the error in verbose mode - logVerbose(e) // Fallback: Estimate token count as one-fourth of text length plus overhead // This provides a rough estimate in case of encoding errors return (text.length >> 2) + ESTIMATE_TOKEN_OVERHEAD From 981d1dd691a4511553dd303f448d61af664ddf19 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 25 Sep 2024 19:15:56 +0000 Subject: [PATCH 3/3] Add error logging for encoding failures in token estimation function --- packages/core/src/tokens.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/tokens.ts b/packages/core/src/tokens.ts index ae81b78e4f..d48bff8f75 100644 --- a/packages/core/src/tokens.ts +++ b/packages/core/src/tokens.ts @@ -17,6 +17,8 @@ export function estimateTokens(text: string, encoder: TokenEncoder) { // Return the length of the encoded text plus a constant overhead return encoder(text).length + ESTIMATE_TOKEN_OVERHEAD } catch (e) { + // If encoding fails, log the error in verbose mode + logVerbose(e) // Fallback: Estimate token count as one-fourth of text length plus overhead // This provides a rough estimate in case of encoding errors return (text.length >> 2) + ESTIMATE_TOKEN_OVERHEAD