diff --git a/packages/cli/src/parse.ts b/packages/cli/src/parse.ts index c0409b9de3..cf6d96589a 100644 --- a/packages/cli/src/parse.ts +++ b/packages/cli/src/parse.ts @@ -2,9 +2,14 @@ import replaceExt from "replace-ext" import { readFile } from "node:fs/promises" import { DOCXTryParse } from "../../core/src/docx" import { extractFenced } from "../../core/src/fence" -import { expandFiles, writeText, readText } from "../../core/src/fs" +import { + expandFiles, + writeText, + readText, + tryReadText, +} from "../../core/src/fs" import { HTMLToText } from "../../core/src/html" -import { isJSONLFilename, readJSONL } from "../../core/src/jsonl" +import { isJSONLFilename, JSONLTryParse } from "../../core/src/jsonl" import { parsePdf } from "../../core/src/pdf" import { estimateTokens } from "../../core/src/tokens" import { YAMLStringify } from "../../core/src/yaml" @@ -12,6 +17,7 @@ import { resolveTokenEncoder } from "../../core/src/encoders" import { DEFAULT_MODEL } from "../../core/src/constants" import { promptyParse, promptyToGenAIScript } from "../../core/src/prompty" import { basename, join } from "node:path" +import { JSONLLMTryParse } from "../../core/src/json5" export async function parseFence(language: string, file: string) { const res = await parsePdf(file) @@ -44,7 +50,8 @@ export async function jsonl2json(files: string[]) { console.log(`skipping ${file}`) continue } - const objs = await readJSONL(file) + const content = await tryReadText(file) + const objs = await JSONLTryParse(content, { repair: true }) const out = replaceExt(file, ".json") await writeText(out, JSON.stringify(objs, null, 2)) console.log(`${file} -> ${out}`) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index e9ddc0f408..a782533f2b 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,10 +1,12 @@ // Import necessary modules and types -import { appendJSONL, readJSONL, writeJSONL } from "./jsonl" -import { host, runtimeHost } from "./host" +import { appendJSONL, JSONLTryParse, writeJSONL } from "./jsonl" +import { host } from "./host" import { dotGenaiscriptPath, sha256string } from "./util" import { CHANGE } from "./constants" import { TraceOptions } from "./trace" import { CORE_VERSION } from "./version" +import { tryReadText } from "./fs" +import { JSON5TryParse } from "./json5" /** * Represents a cache entry with a hashed identifier (`sha`), `key`, and `val`. @@ -60,7 +62,8 @@ export class JSONLineCache extends EventTarget { if (this._entries) return this._entries = {} await host.createDirectory(this.folder()) // Ensure directory exists - const objs: CacheEntry[] = await readJSONL(this.path()) + const content = await tryReadText(this.path()) + const objs: CacheEntry[] = (await JSON5TryParse(content)) ?? [] let numdup = 0 // Counter for duplicates for (const obj of objs) { if (this._entries[obj.sha]) numdup++ // Count duplicates diff --git a/packages/core/src/jsonl.ts b/packages/core/src/jsonl.ts index 0900ceafe2..fe38e5fab3 100644 --- a/packages/core/src/jsonl.ts +++ b/packages/core/src/jsonl.ts @@ -13,31 +13,6 @@ export function isJSONLFilename(fn: string) { return /\.(jsonl|mdjson|ldjson)$/i.test(fn) } -export async function readJSONL(fn: string) { - const buf = await tryReadFile(fn) - const res: any[] = [] - if (buf == null) return res - let line = 1 - let numerr = 0 - const decoder = host.createUTF8Decoder() - for (let pos = 0; pos < buf.length; ) { - let ep = buf.indexOf(10, pos) - if (ep < 0) ep = buf.length - const str = decoder.decode(buf.slice(pos, ep)) - if (!/^\s*$/.test(str)) { - try { - res.push(JSON.parse(str)) - } catch (e) { - if (!numerr) logWarn(`${fn}(${line}): JSON error`) - numerr++ - } - } - pos = ep + 1 - line++ - } - return res -} - export function JSONLTryParse( text: string, options?: { @@ -49,7 +24,7 @@ export function JSONLTryParse( for (const line of text.split("\n")) { if (!line) continue const obj = JSON5TryParse(line, options) - res.push(obj) + if (obj !== undefined && obj !== null) res.push(obj) } return res } @@ -57,7 +32,7 @@ export function JSONLTryParse( export function JSONLStringify(objs: any[]) { const acc: string[] = [] if (objs?.length) - for (const o of objs) { + for (const o of objs.filter((o) => o !== undefined && o !== null)) { const s = JSON.stringify(o) acc.push(s) } diff --git a/packages/vscode/genaisrc/cmt.genai.mts b/packages/vscode/genaisrc/cmt.genai.mts index 10728324eb..6a9262f58d 100644 --- a/packages/vscode/genaisrc/cmt.genai.mts +++ b/packages/vscode/genaisrc/cmt.genai.mts @@ -21,8 +21,6 @@ script({ const { format, build } = env.vars const saveLimit = pLimit(1) -console.log({ format, build }) - // Get files from environment or modified files from Git if none provided let files = env.files if (files.length === 0) { @@ -122,7 +120,8 @@ You should analyze it, and add/update appropriate comments as needed. To add or update comments to this code, follow these steps: 1. Analyze the code to understand its structure and functionality. -- If you are not familiar with the programming language, ignore the file. +- If you are not familiar with the programming language, emit an empty file. +- If there is no code, emit an empty file. 2. Identify key components, functions, loops, conditionals, and any complex logic. 3. Add comments that explain: - The purpose of functions or code blocks using the best comment format for that programming language. @@ -158,7 +157,7 @@ Your comments should provide insight into the code's purpose, logic, and any imp ) const { text, fences } = res const newContent = fences?.[0]?.content ?? text - if (!newContent) return undefined + if (!newContent?.trim()) return undefined if (newContent === content) break content = newContent }