clean cli

packages/cli/src/build.ts

@@ -1,4 +1,6 @@
-import { GENAI_ANYJS_GLOB, host, parseProject } from "genaiscript-core"
+import { GENAI_ANYJS_GLOB } from "../../core/src/constants"
+import { host } from "../../core/src/host"
+import { parseProject } from "../../core/src/parser"
 
 export async function buildProject(options?: {
     toolFiles?: string[]

packages/cli/src/cli.ts

@@ -1,16 +1,3 @@
-import {
-    RequestError,
-    CORE_VERSION,
-    isRequestError,
-    TOOL_ID,
-    TOOL_NAME,
-    GITHUB_REPO,
-    SERVER_PORT,
-    RUNTIME_ERROR_CODE,
-    UNHANDLED_ERROR_CODE,
-    errorMessage,
-    dotGenaiscriptPath,
-} from "genaiscript-core"
 import { NodeHost } from "./nodehost"
 import { program } from "commander"
 import { error, isQuiet, setConsoleColors, setQuiet } from "./log"
@@ -40,6 +27,20 @@ import { scriptTestsView, scriptsTest } from "./test"
 import { emptyDir } from "fs-extra"
 import { join } from "path"
 import "node:console"
+import {
+    UNHANDLED_ERROR_CODE,
+    RUNTIME_ERROR_CODE,
+    TOOL_ID,
+    TOOL_NAME,
+    SERVER_PORT,
+} from "../../core/src/constants"
+import {
+    errorMessage,
+    isRequestError,
+    RequestError,
+} from "../../core/src/error"
+import { dotGenaiscriptPath } from "../../core/src/util"
+import { CORE_VERSION, GITHUB_REPO } from "../../core/src/version"
 
 async function cacheClear(name: string) {
     let dir = dotGenaiscriptPath("cache")

packages/cli/src/codequery.ts

@@ -1,4 +1,4 @@
-import { YAMLStringify, treeSitterQuery } from "genaiscript-core"
+import { treeSitterQuery } from "../../core/src/treesitter"
 
 export async function codeQuery(file: string, query: string) {
     const res = await treeSitterQuery(

packages/cli/src/docker.ts

@@ -1,21 +1,20 @@
 import MemoryStream from "memorystream"
-import {
-    CORE_VERSION,
-    DOCKER_CONTAINER_VOLUME,
-    DOCKER_DEFAULT_IMAGE,
-    DOCKER_VOLUMES_DIR,
-    TraceOptions,
-    dotGenaiscriptPath,
-    errorMessage,
-    host,
-    installImport,
-    logError,
-    randomHex,
-} from "genaiscript-core"
 import { finished } from "stream/promises"
 import { ensureDir, remove } from "fs-extra"
 import { copyFile, readFile, writeFile } from "fs/promises"
 import { DOCKERODE_VERSION } from "./version"
+import {
+    DOCKER_DEFAULT_IMAGE,
+    DOCKER_VOLUMES_DIR,
+    DOCKER_CONTAINER_VOLUME,
+} from "../../core/src/constants"
+import { randomHex } from "../../core/src/crypto"
+import { errorMessage } from "../../core/src/error"
+import { host } from "../../core/src/host"
+import { installImport } from "../../core/src/import"
+import { TraceOptions } from "../../core/src/trace"
+import { logError, dotGenaiscriptPath } from "../../core/src/util"
+import { CORE_VERSION } from "../../core/src/version"
 
 type DockerodeType = import("dockerode")
 

packages/cli/src/info.ts

@@ -1,10 +1,10 @@
+import { host } from "../../core/src/host"
 import {
-    CORE_VERSION,
     ModelConnectionInfo,
-    YAMLStringify,
-    host,
     resolveModelConnectionInfo,
-} from "genaiscript-core"
+} from "../../core/src/models"
+import { CORE_VERSION } from "../../core/src/version"
+import { YAMLStringify } from "../../core/src/yaml"
 import { buildProject } from "./build"
 
 export async function systemInfo() {

packages/cli/src/llamaindexretrieval.ts

@@ -1,40 +1,41 @@
+import { type BaseReader } from "llamaindex"
+import type { GenericFileSystem } from "@llamaindex/env"
+import { fileTypeFromBuffer } from "file-type"
+import { LLAMAINDEX_VERSION } from "./version"
+import assert from "assert"
+import { writeJSON } from "fs-extra"
 import {
-    Host,
-    LLAMAINDEX_MIN_SCORE,
-    LLAMAINDEX_SIMILARITY_TOPK,
-    MarkdownTrace,
-    PromiseType,
-    RETRIEVAL_DEFAULT_INDEX,
-    ResponseStatus,
-    RetrievalSearchOptions,
-    RetrievalSearchResponse,
-    RetrievalService,
-    RetrievalUpsertOptions,
-    dotGenaiscriptPath,
-    installImport,
-    lookupMime,
-    createFetch,
+    JAVASCRIPT_MIME_TYPE,
+    JSON_MIME_TYPE,
+    JSON_SCHEMA_MIME_TYPE,
+    PDF_MIME_TYPE,
+    DOCX_MIME_TYPE,
     RETRIEVAL_PERSIST_DIR,
-    parseModelIdentifier,
+    RETRIEVAL_DEFAULT_INDEX,
+    MODEL_PROVIDER_OLLAMA,
+    TOOL_ID,
     RETRIEVAL_DEFAULT_LLM_MODEL,
     RETRIEVAL_DEFAULT_EMBED_MODEL,
     RETRIEVAL_DEFAULT_TEMPERATURE,
+    LLAMAINDEX_SIMILARITY_TOPK,
+    LLAMAINDEX_MIN_SCORE,
+} from "../../core/src/constants"
+import { createFetch } from "../../core/src/fetch"
+import { tryReadJSON } from "../../core/src/fs"
+import {
+    RetrievalService,
     ModelService,
-    MODEL_PROVIDER_OLLAMA,
-    TOOL_ID,
-    tryReadJSON,
-    writeJSON,
-    JAVASCRIPT_MIME_TYPE,
-    JSON_MIME_TYPE,
-    PDF_MIME_TYPE,
-    DOCX_MIME_TYPE,
-    JSON_SCHEMA_MIME_TYPE,
-    assert,
-} from "genaiscript-core"
-import { type BaseReader } from "llamaindex"
-import type { GenericFileSystem } from "@llamaindex/env"
-import { fileTypeFromBuffer } from "file-type"
-import { LLAMAINDEX_VERSION } from "./version"
+    Host,
+    ResponseStatus,
+    RetrievalUpsertOptions,
+    RetrievalSearchOptions,
+    RetrievalSearchResponse,
+} from "../../core/src/host"
+import { installImport, PromiseType } from "../../core/src/import"
+import { lookupMime } from "../../core/src/mime"
+import { parseModelIdentifier } from "../../core/src/models"
+import { MarkdownTrace } from "../../core/src/trace"
+import { dotGenaiscriptPath } from "../../core/src/util"
 
 class BlobFileSystem implements GenericFileSystem {
     constructor(
@@ -190,7 +191,7 @@ export class LlamaIndexRetrievalService
             OpenAIEmbedding,
             Ollama,
             OpenAI,
-            OllamaEmbedding
+            OllamaEmbedding,
         } = this.module
         const { provider: llmProvider, model: llmModel } =
             parseModelIdentifier(llmModel_)

packages/cli/src/nodehost.ts

@@ -1,31 +1,5 @@
 import dotenv from "dotenv"
-import prompts from "prompts"
-import {
-    AZURE_OPENAI_TOKEN_SCOPES,
-    AbortSignalOptions,
-    DEFAULT_MODEL,
-    DEFAULT_TEMPERATURE,
-    Host,
-    LanguageModel,
-    LanguageModelConfiguration,
-    LogLevel,
-    MODEL_PROVIDER_AZURE,
-    ModelService,
-    RetrievalService,
-    SHELL_EXEC_TIMEOUT,
-    ServerManager,
-    TraceOptions,
-    UTF8Decoder,
-    UTF8Encoder,
-    createBundledParsers,
-    createFileSystem,
-    filterGitIgnore,
-    parseDefaultsFromEnv,
-    parseTokenFromEnv,
-    resolveLanguageModel,
-    setHost,
-    unique,
-} from "genaiscript-core"
+
 import { TextDecoder, TextEncoder } from "util"
 import { readFile, unlink, writeFile } from "node:fs/promises"
 import { ensureDir, existsSync, remove } from "fs-extra"
@@ -38,6 +12,31 @@ import { LlamaIndexRetrievalService } from "./llamaindexretrieval"
 import { createNodePath } from "./nodepath"
 import { DockerManager } from "./docker"
 import { DefaultAzureCredential, AccessToken } from "@azure/identity"
+import { LanguageModel } from "../../core/src/chat"
+import {
+    DEFAULT_MODEL,
+    DEFAULT_TEMPERATURE,
+    MODEL_PROVIDER_AZURE,
+    AZURE_OPENAI_TOKEN_SCOPES,
+    SHELL_EXEC_TIMEOUT,
+} from "../../core/src/constants"
+import { createFileSystem, filterGitIgnore } from "../../core/src/fs"
+import {
+    ServerManager,
+    Host,
+    RetrievalService,
+    ModelService,
+    setHost,
+    LanguageModelConfiguration,
+    LogLevel,
+    UTF8Decoder,
+    UTF8Encoder,
+} from "../../core/src/host"
+import { resolveLanguageModel } from "../../core/src/models"
+import { createBundledParsers } from "../../core/src/pdf"
+import { AbortSignalOptions, TraceOptions } from "../../core/src/trace"
+import { unique } from "../../core/src/util"
+import { parseGHTokenFromEnv } from "../../core/src/github"
 
 class NodeServerManager implements ServerManager {
     async start(): Promise<void> {

packages/cli/src/parse.ts

@@ -1,20 +1,15 @@
-import {
-    DOCXTryParse,
-    YAMLStringify,
-    expandFiles,
-    extractFenced,
-    isJSONLFilename,
-    readJSONL,
-    writeText,
-    parsePdf,
-    HTMLToText,
-    estimateTokens,
-    readText,
-} from "genaiscript-core"
 import { createProgressSpinner } from "./spinner"
 import replaceExt from "replace-ext"
 import getStdin from "get-stdin"
 import { readFile } from "node:fs/promises"
+import { DOCXTryParse } from "../../core/src/docx"
+import { extractFenced } from "../../core/src/fence"
+import { expandFiles, writeText, readText } from "../../core/src/fs"
+import { HTMLToText } from "../../core/src/html"
+import { isJSONLFilename, readJSONL } from "../../core/src/jsonl"
+import { parsePdf } from "../../core/src/pdf"
+import { estimateTokens } from "../../core/src/tokens"
+import { YAMLStringify } from "../../core/src/yaml"
 
 export async function parseFence(language: string) {
     const stdin = await getStdin()

packages/cli/src/retrieval.ts

@@ -1,13 +1,12 @@
+import { expandFiles } from "../../core/src/fs"
+import { fuzzSearch } from "../../core/src/fuzzsearch"
 import {
-    YAMLStringify,
     upsertVector,
-    vectorSearch,
     clearVectorIndex,
-    normalizeInt,
-    expandFiles,
-    normalizeFloat,
-    fuzzSearch,
-} from "genaiscript-core"
+    vectorSearch,
+} from "../../core/src/retrieval"
+import { normalizeFloat, normalizeInt } from "../../core/src/util"
+import { YAMLStringify } from "../../core/src/yaml"
 import { createProgressSpinner } from "./spinner"
 
 export async function retrievalIndex(

packages/cli/src/run.ts

@@ -1,51 +1,52 @@
-import {
-    GenerationResult,
-    YAMLStringify,
-    diagnosticsToCSV,
-    host,
-    isJSONLFilename,
-    logVerbose,
-    runTemplate,
-    writeText,
-    normalizeInt,
-    normalizeFloat,
-    GENAI_ANYJS_REGEX,
-    FILES_NOT_FOUND_ERROR_CODE,
-    appendJSONL,
-    RUNTIME_ERROR_CODE,
-    ANNOTATION_ERROR_CODE,
-    writeFileEdits,
-    logError,
-    isCancelError,
-    USER_CANCELLED_ERROR_CODE,
-    errorMessage,
-    MarkdownTrace,
-    HTTPS_REGEX,
-    resolveModelConnectionInfo,
-    CONFIGURATION_ERROR_CODE,
-    parseKeyValuePairs,
-    JSONSchemaStringifyToTypeScript,
-    filePathOrUrlToWorkspaceFile,
-    JSONSchemaStringify,
-    CSV_REGEX,
-    CLI_RUN_FILES_FOLDER,
-    parseGHTokenFromEnv,
-    githubUpdatePullRequestDescription,
-    githubCreatePullRequestReviews,
-    githubCreateIssueComment,
-    PromptScriptRunOptions,
-    TraceOptions,
-    CancellationOptions,
-    Fragment,
-    ChatCompletionsProgressReport,
-} from "genaiscript-core"
 import { capitalize } from "inflection"
 import { resolve, join, relative } from "node:path"
 import { isQuiet } from "./log"
 import { emptyDir, ensureDir } from "fs-extra"
 import { convertDiagnosticsToSARIF } from "./sarif"
 import { buildProject } from "./build"
 import { createProgressSpinner } from "./spinner"
+import { diagnosticsToCSV } from "../../core/src/ast"
+import { CancellationOptions } from "../../core/src/cancellation"
+import { ChatCompletionsProgressReport } from "../../core/src/chat"
+import { Fragment, runTemplate } from "../../core/src/promptrunner"
+import {
+    githubCreateIssueComment,
+    githubCreatePullRequestReviews,
+    githubUpdatePullRequestDescription,
+    parseGHTokenFromEnv,
+} from "../../core/src/github"
+import {
+    GENAI_ANYJS_REGEX,
+    HTTPS_REGEX,
+    FILES_NOT_FOUND_ERROR_CODE,
+    CONFIGURATION_ERROR_CODE,
+    USER_CANCELLED_ERROR_CODE,
+    RUNTIME_ERROR_CODE,
+    CSV_REGEX,
+    CLI_RUN_FILES_FOLDER,
+    ANNOTATION_ERROR_CODE,
+} from "../../core/src/constants"
+import { isCancelError, errorMessage } from "../../core/src/error"
+import { GenerationResult } from "../../core/src/expander"
+import { parseKeyValuePairs } from "../../core/src/fence"
+import { filePathOrUrlToWorkspaceFile, writeText } from "../../core/src/fs"
+import { host } from "../../core/src/host"
+import { isJSONLFilename, appendJSONL } from "../../core/src/jsonl"
+import { resolveModelConnectionInfo } from "../../core/src/models"
+import {
+    JSONSchemaStringifyToTypeScript,
+    JSONSchemaStringify,
+} from "../../core/src/schema"
+import { TraceOptions, MarkdownTrace } from "../../core/src/trace"
+import {
+    normalizeFloat,
+    normalizeInt,
+    logVerbose,
+    logError,
+} from "../../core/src/util"
+import { YAMLStringify } from "../../core/src/yaml"
+import { PromptScriptRunOptions } from "../../core/src/server/messages"
+import { writeFileEdits } from "../../core/src/edits"
 
 export async function runScriptWithExitCode(
     scriptId: string,