Skip to content

Commit

Permalink
Enhance tracing and error handling across multiple modules with expan…
Browse files Browse the repository at this point in the history 
…ded details options
  • Loading branch information
@pelikhan
pelikhan committed Sep 30, 2024
1 parent a69e3bc commit a5e3d44
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 88 deletions.
2 changes: 1 addition & 1 deletion packages/core/src/chat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,6 @@ export async function executeChatSession(

export function tracePromptResult(trace: MarkdownTrace, resp: RunPromptResult) {
const { json, text } = resp
trace.details(`🔠 output`, text)
trace.details(`🔠 output`, text, { expanded: true })
if (resp.json) trace.detailsFenced("📩 JSON (parsed)", json, "json")
}
89 changes: 52 additions & 37 deletions packages/core/src/expander.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,48 +258,63 @@ export async function expandTemplate(
messages.push(toChatCompletionUserMessage(prompt.text, prompt.images))
if (prompt.aici) messages.push(prompt.aici)

for (let i = 0; i < systems.length; ++i) {
if (cancellationToken?.isCancellationRequested)
return {
status: "cancelled",
statusText: "user cancelled",
messages,
}
if (systems.length)
try {
trace.startDetails("👾 systems")
for (let i = 0; i < systems.length; ++i) {
if (cancellationToken?.isCancellationRequested)
return {
status: "cancelled",
statusText: "user cancelled",
messages,
}

const system = prj.getTemplate(systems[i])
if (!system) throw new Error(`system template ${systems[i]} not found`)
const system = prj.getTemplate(systems[i])
if (!system)
throw new Error(`system template ${systems[i]} not found`)

trace.startDetails(`👾 ${system.id}`)
const sysr = await callExpander(prj, system, env, trace, options)
trace.startDetails(`👾 ${system.id}`)
const sysr = await callExpander(
prj,
system,
env,
trace,
options
)

if (sysr.images) images.push(...sysr.images)
if (sysr.schemas) Object.assign(schemas, sysr.schemas)
if (sysr.functions) functions.push(...sysr.functions)
if (sysr.fileMerges) fileMerges.push(...sysr.fileMerges)
if (sysr.outputProcessors)
outputProcessors.push(...sysr.outputProcessors)
if (sysr.chatParticipants)
chatParticipants.push(...sysr.chatParticipants)
if (sysr.fileOutputs) fileOutputs.push(...sysr.fileOutputs)
if (sysr.logs?.length) trace.details("📝 console.log", sysr.logs)
if (sysr.text) {
systemMessage.content += SYSTEM_FENCE + "\n" + sysr.text + "\n"
trace.fence(sysr.text, "markdown")
}
if (sysr.aici) {
trace.fence(sysr.aici, "yaml")
messages.push(sysr.aici)
}
trace.detailsFenced("js", system.jsSource, "js")
trace.endDetails()
if (sysr.images) images.push(...sysr.images)
if (sysr.schemas) Object.assign(schemas, sysr.schemas)
if (sysr.functions) functions.push(...sysr.functions)
if (sysr.fileMerges) fileMerges.push(...sysr.fileMerges)
if (sysr.outputProcessors)
outputProcessors.push(...sysr.outputProcessors)
if (sysr.chatParticipants)
chatParticipants.push(...sysr.chatParticipants)
if (sysr.fileOutputs) fileOutputs.push(...sysr.fileOutputs)
if (sysr.logs?.length)
trace.details("📝 console.log", sysr.logs)
if (sysr.text) {
systemMessage.content +=
SYSTEM_FENCE + "\n" + sysr.text + "\n"
trace.fence(sysr.text, "markdown")
}
if (sysr.aici) {
trace.fence(sysr.aici, "yaml")
messages.push(sysr.aici)
}
trace.detailsFenced("js", system.jsSource, "js")
trace.endDetails()

if (sysr.status !== "success")
return {
status: sysr.status,
statusText: sysr.statusText,
messages,
if (sysr.status !== "success")
return {
status: sysr.status,
statusText: sysr.statusText,
messages,
}
}
}
} finally {
trace.endDetails()
}

const responseSchema = promptParametersSchemaToJSONSchema(
template.responseSchema
Expand Down
1 change: 0 additions & 1 deletion packages/core/src/github.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,6 @@ export class GitHubClient implements GitHub {
}
)
const res = await paginatorToArray(ite, count, (i) => i.data)
res[0].reactions
return res
}

Expand Down
94 changes: 52 additions & 42 deletions packages/core/src/promptcontext.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,50 +319,60 @@ export async function createPromptContext(
role: "system",
content: "",
}
for (const systemId of resolveSystems(prj, runOptions ?? {})) {
checkCancelled(cancellationToken)
const systemScripts = resolveSystems(prj, runOptions ?? {})
if (systemScripts.length)
try {
trace.startDetails("👾 systems")
for (const systemId of systemScripts) {
checkCancelled(cancellationToken)

const system = prj.getTemplate(systemId)
if (!system)
throw new Error(`system template ${systemId} not found`)
runTrace.startDetails(`👾 ${system.id}`)
const sysr = await callExpander(
prj,
system,
env,
runTrace,
genOptions
)
if (sysr.images?.length)
throw new NotSupportedError("images")
if (sysr.schemas) Object.assign(schemas, sysr.schemas)
if (sysr.functions) tools.push(...sysr.functions)
if (sysr.fileMerges?.length)
throw new NotSupportedError("fileMerges")
if (sysr.outputProcessors?.length)
throw new NotSupportedError("outputProcessors")
if (sysr.chatParticipants)
chatParticipants.push(...sysr.chatParticipants)
if (sysr.fileOutputs?.length)
throw new NotSupportedError("fileOutputs")
if (sysr.logs?.length)
runTrace.details("📝 console.log", sysr.logs)
if (sysr.text) {
systemMessage.content +=
SYSTEM_FENCE + "\n" + sysr.text + "\n"
runTrace.fence(sysr.text, "markdown")
}
if (sysr.aici) {
runTrace.fence(sysr.aici, "yaml")
messages.push(sysr.aici)
const system = prj.getTemplate(systemId)
if (!system)
throw new Error(
`system template ${systemId} not found`
)
runTrace.startDetails(`👾 ${system.id}`)
const sysr = await callExpander(
prj,
system,
env,
runTrace,
genOptions
)
if (sysr.images?.length)
throw new NotSupportedError("images")
if (sysr.schemas)
Object.assign(schemas, sysr.schemas)
if (sysr.functions) tools.push(...sysr.functions)
if (sysr.fileMerges?.length)
throw new NotSupportedError("fileMerges")
if (sysr.outputProcessors?.length)
throw new NotSupportedError("outputProcessors")
if (sysr.chatParticipants)
chatParticipants.push(...sysr.chatParticipants)
if (sysr.fileOutputs?.length)
throw new NotSupportedError("fileOutputs")
if (sysr.logs?.length)
runTrace.details("📝 console.log", sysr.logs)
if (sysr.text) {
systemMessage.content +=
SYSTEM_FENCE + "\n" + sysr.text + "\n"
runTrace.fence(sysr.text, "markdown")
}
if (sysr.aici) {
runTrace.fence(sysr.aici, "yaml")
messages.push(sysr.aici)
}
runTrace.detailsFenced("js", system.jsSource, "js")
runTrace.endDetails()
if (sysr.status !== "success")
throw new Error(
`system ${system.id} failed ${sysr.status} ${sysr.statusText}`
)
}
} finally {
trace.endDetails()
}
runTrace.detailsFenced("js", system.jsSource, "js")
runTrace.endDetails()
if (sysr.status !== "success")
throw new Error(
`system ${system.id} failed ${sysr.status} ${sysr.statusText}`
)
}
if (systemMessage.content) messages.unshift(systemMessage)

const connection = await resolveModelConnectionInfo(
Expand Down
5 changes: 3 additions & 2 deletions packages/core/src/promptdom.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,9 @@ async function tracePromptNode(
)
if (value.length > 0) title += `: ${value}`
if (n.children?.length || n.preview) {
trace.startDetails(title, n.error ? false : undefined)
trace.startDetails(title, {
success: n.error ? false : undefined,
})
if (n.preview) trace.fence(n.preview, "markdown")
} else trace.resultItem(!n.error, title)
},
Expand Down Expand Up @@ -871,7 +873,6 @@ export async function renderPromptNode(
const truncated = await truncatePromptNode(model, node, options)
if (truncated) await tracePromptNode(trace, node, { label: "truncated" })

let systemPrompt = ""
let userPrompt = ""
let assistantPrompt = ""
const images: PromptImage[] = []
Expand Down
17 changes: 13 additions & 4 deletions packages/core/src/trace.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,15 @@ export class MarkdownTrace extends EventTarget implements ToolCallTrace {
)
}

startDetails(title: string, success?: boolean) {
startDetails(
title: string,
options?: { success?: boolean; expanded?: boolean }
) {
const { success, expanded } = options || {}
this.detailsDepth++
title = title?.trim() || ""
this.appendContent(`\n\n<details class="${TOOL_ID}">
this
.appendContent(`\n\n<details class="${TOOL_ID}"${expanded ? ` open="true"` : ""}>
<summary>
${this.toResultIcon(success, "")}${title}
</summary>
Expand All @@ -115,9 +120,13 @@ ${this.toResultIcon(success, "")}${title}
}
}

details(title: string, body: string | object) {
details(
title: string,
body: string | object,
options?: { success?: boolean; expanded?: boolean }
) {
this.disableChange(() => {
this.startDetails(title)
this.startDetails(title, options)
if (body) {
if (typeof body === "string") this.appendContent(body)
else this.appendContent(yamlStringify(body))
Expand Down
2 changes: 1 addition & 1 deletion packages/sample/genaisrc/gai.genai.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ ${ls ? "- LOG_DIFF contains a diff of 2 runs in GitHub Action" : "- LOG contains
Add links to run logs.
Analyze the diff in LOG_DIFF and provide a summary of the root cause of the failure.
Analyze the diff in LOG_DIFF and provide a summary of the root cause of the failure. Show the code that is responsible for the failure.
If you cannot find the root cause, stop.
Expand Down

0 comments on commit a5e3d44

Please sign in to comment.