Add templates and minor improvements (#142)

CHANGELOG.md

@@ -10,6 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+## [0.23.0]
+
+### Added
+- Added new prompt templates for "Expert" tasks like `LinuxBashExpertAsk`, `JavascriptExpertTask`, etc.
+- Added new prompt templates for self-critiquing agents like `ChiefEditorTranscriptCritic`, `JuliaExpertTranscriptCritic`, etc.
+
+### Updated
+- Extended `aicodefixer_feedback` methods to work with `AICode` and `AIGenerate`.
+
 ## [0.22.0]
 
 ### Added

Project.toml

@@ -1,7 +1,7 @@
 name = "PromptingTools"
 uuid = "670122d1-24a8-4d70-bfce-740807c42192"
 authors = ["J S @svilupp and contributors"]
-version = "0.22.0"
+version = "0.23.0"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"

docs/generate_prompt_library.jl

@@ -87,4 +87,4 @@ for file_path in keys(loaded_templates)
     end
     ## write to file
     write(file_path, String(take!(io)))
-end
+end

examples/working_with_aitemplates.jl

@@ -46,7 +46,7 @@ msgs = PT.render(AITemplate(:JuliaExpertAsk))
 # 
 # Let's adjust the previous template to be more specific to a data analysis question:
 tpl = [PT.SystemMessage("You are a world-class Julia language programmer with the knowledge of the latest syntax. You're also a senior Data Scientist and proficient in data analysis in Julia. Your communication is brief and concise. You're precise and answer only when you're confident in the high quality of your answer.")
-    PT.UserMessage("# Question\n\n{{ask}}")]
+       PT.UserMessage("# Question\n\n{{ask}}")]
 # Templates are saved in the `templates` directory of the package. Name of the file will become the template name (eg, call `:JuliaDataExpertAsk`)
 filename = joinpath(pkgdir(PromptingTools),
     "templates",

src/Experimental/APITools/tavily_api.jl

@@ -65,7 +65,7 @@ function create_websearch(query::AbstractString;
         max_results::Integer = 5,
         include_images::Bool = false,
         include_domains::AbstractVector{<:AbstractString} = String[],
-        exclude_domains::AbstractVector{<:AbstractString} = String[],)
+        exclude_domains::AbstractVector{<:AbstractString} = String[])
     @assert search_depth in ["basic", "advanced"] "Search depth must be either 'basic' or 'advanced'"
     @assert max_results>0 "Max results must be a positive integer"
 

src/Experimental/AgentTools/code_feedback.jl

@@ -8,9 +8,12 @@ struct CodeSuccess <: AbstractCodeOutcome end
 
 # Feedback function skeleton
 """
+    aicodefixer_feedback(cb::AICode; max_length::Int = 512) -> NamedTuple(; feedback::String)
     aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage}; max_length::Int = 512) -> NamedTuple(; feedback::String)
+    aicodefixer_feedback(msg::PT.AIMessage; max_length::Int = 512) -> NamedTuple(; feedback::String)
+    aicodefixer_feedback(aicall::AICall; max_length::Int = 512) -> NamedTuple(; feedback::String)
 
-Generate feedback for an AI code fixing session based on the conversation history.
+Generate feedback for an AI code fixing session based on the AICode block /or conversation history (that will be used to extract and evaluate a code block).
 Function is designed to be extensible for different types of feedback and code evaluation outcomes. 
 
 The highlevel wrapper accepts a conversation and returns new kwargs for the AICall.
@@ -20,14 +23,18 @@ Individual feedback functions are dispatched on different subtypes of `AbstractC
 See also: `AIGenerate`, `AICodeFixer`
 
 # Arguments
-- `conversation::AbstractVector{<:PT.AbstractMessage}`: A vector of messages representing the conversation history, where the last message is expected to contain the code to be analyzed.
+- `cb::AICode`: AICode block to evaluate and provide feedback on.
 - `max_length::Int=512`: An optional argument that specifies the maximum length of the feedback message.
 
 # Returns
 - `NamedTuple`: A feedback message as a kwarg in NamedTuple based on the analysis of the code provided in the conversation.
 
 # Example
 ```julia
+cb = AICode(msg; skip_unsafe = true, capture_stdout = true)
+new_kwargs = aicodefixer_feedback(cb)
+
+new_kwargs = aicodefixer_feedback(msg)
 new_kwargs = aicodefixer_feedback(conversation)
 ```
 
@@ -45,11 +52,9 @@ It dispatches for the code feedback based on the subtypes of `AbstractCodeOutcom
 
 You can override the individual methods to customize the feedback.
 """
-function aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage};
+function aicodefixer_feedback(cb::AICode;
         max_length::Int = 512)
     @assert max_length>0 "max_length must be positive (provided: $max_length)"
-    # Extract the last message, evaluate code, determine outcome
-    cb = AICode(last(conversation); skip_unsafe = true, capture_stdout = true)
     outcome = if isempty(cb.code)
         CodeEmpty() # No code provided
     elseif !PT.isparsed(cb)
@@ -65,6 +70,14 @@ function aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage}
     new_kwargs = (; feedback = aicodefixer_feedback(outcome, cb; max_length))
     return new_kwargs
 end
+function aicodefixer_feedback(msg::PT.AIMessage; kwargs...)
+    # Extract the last message, evaluate code, determine outcome
+    cb = AICode(msg; skip_unsafe = true, capture_stdout = true)
+    aicodefixer_feedback(cb; kwargs...)
+end
+function aicodefixer_feedback(conversation::AbstractVector{<:PT.AbstractMessage}; kwargs...)
+    aicodefixer_feedback(last(conversation); kwargs...)
+end
 
 function aicodefixer_feedback(::CodeEmpty, args...; kwargs...)
     "**Error Detected**: No Julia code found. Always enclose Julia code in triple backticks code fence (\`\`\`julia\\n ... \\n\`\`\`)."

src/Experimental/AgentTools/lazy_types.jl

@@ -355,6 +355,10 @@ function Base.var"=="(c1::AICallBlock, c2::AICallBlock)
     all(f -> getfield(c1, f) == getfield(c2, f), fieldnames(typeof(c1)))
 end
 
+function aicodefixer_feedback(aicall::AICall; kwargs...)
+    aicodefixer_feedback(aicall.conversation; kwargs...)
+end
+
 """
     AICodeFixer(aicall::AICall, templates::Vector{<:PT.UserMessage}; num_rounds::Int = 3, feedback_func::Function = aicodefixer_feedback; kwargs...)
     AICodeFixer(aicall::AICall, template::Union{AITemplate, Symbol} = :CodeFixerRCI; kwargs...)

src/Experimental/AgentTools/mcts.jl

@@ -75,6 +75,9 @@ end
 function Base.getindex(node::SampleNode, id::Integer)
     find_node(node, id)
 end
+function Base.length(node::SampleNode)
+    PreOrderDFS(node) |> collect |> length
+end
 function Base.var"=="(n1::SampleNode, n2::SampleNode)
     all(fieldnames(typeof(n1))) do f
         if f == :parent

src/Experimental/AgentTools/utils.jl

@@ -89,7 +89,8 @@ function truncate_conversation(conversation::AbstractVector{<:PT.AbstractMessage
                           length(conversation) > 2
         # start with the last two messages' length (always included)
         new_conversation = similar(conversation) |> empty!
-        current_length = sum(length.(getproperty.(conversation[(end - 1):end],
+        current_length = sum(
+            length.(getproperty.(conversation[(end - 1):end],
                 :content)); init = 0)
         for i in eachindex(conversation[begin:(end - 2)])
             length_ = length(conversation[i].content)

src/Experimental/RAGTools/annotation.jl

@@ -651,4 +651,4 @@ function print_html(
     io = IOBuffer()
     print_html(io, rag_or_parent_node; kwargs...)
     String(take!(io))
-end
+end

src/Experimental/RAGTools/generation.jl

@@ -665,4 +665,4 @@ function PT.pprint(io::IO, airag_result::Tuple{PT.AIMessage, AbstractRAGResult},
         text_width::Int = displaysize(io)[2])
     rag_details = airag_result[2]
     pprint(io, rag_details; text_width)
-end
+end

src/Experimental/RAGTools/preparation.jl

@@ -524,4 +524,4 @@ end
 const DEFAULT_INDEXER = SimpleIndexer()
 function build_index(files_or_docs::Vector{<:AbstractString}; kwargs...)
     build_index(DEFAULT_INDEXER, files_or_docs; kwargs...)
-end
+end

src/Experimental/RAGTools/rag_interface.jl

@@ -269,4 +269,4 @@ function build_context! end
 function build_context end
 function answer! end
 function refine! end
-function postprocess! end
+function postprocess! end

src/Experimental/RAGTools/retrieval.jl

@@ -703,4 +703,4 @@ function retrieve(index::AbstractChunkIndex, question::AbstractString;
         kwargs...)
     return retrieve(DEFAULT_RETRIEVER, index, question;
         kwargs...)
-end
+end

src/code_eval.jl

@@ -149,7 +149,8 @@ function (CB::Type{T})(md::AbstractString;
     end
     if !isempty(removed)
         ## Add to STDOUT what we removed
-        warning = string("!!! IMPORTANT: Unsafe lines blocked from execution (eg, Pkg operations or imports of non-existent packages):",
+        warning = string(
+            "!!! IMPORTANT: Unsafe lines blocked from execution (eg, Pkg operations or imports of non-existent packages):",
             "\n$removed\n",
             "Fix or find a workaround!")
         if isnothing(cb.stdout)

src/code_expressions.jl

@@ -44,7 +44,7 @@ JULIA_EXPR_HEADS = [
     :comprehension,
     :generator,
     :kw,
-    :where,
+    :where
 ]
 # Checks if the provided expression `ex` has some hallmarks of Julia code. Very naive!
 # Serves as a quick check to avoid trying to eval output cells (```plaintext ... ```)
@@ -66,8 +66,9 @@ function remove_macro_expr!(expr, sym::Symbol = Symbol("@testset"))
        expr.args[1] == sym
         return Expr(:block)
     elseif expr isa Expr && !isempty(expr.args)
-        expr.args = filter(x -> !(x isa Expr && x.head == :macrocall && !isempty(x.args) &&
-                                  x.args[1] == sym),
+        expr.args = filter(
+            x -> !(x isa Expr && x.head == :macrocall && !isempty(x.args) &&
+                   x.args[1] == sym),
             expr.args)
         foreach(x -> remove_macro_expr!(x, sym), expr.args)
     end

src/code_parsing.jl

@@ -39,11 +39,13 @@ function extract_julia_imports(input::AbstractString; base_or_main::Bool = false
             packages = filter(x -> !isempty(x), split(subparts, " "))
             if base_or_main
                 ## keep only them
-                packages = filter(x -> startswith(x, "Base") ||
+                packages = filter(
+                    x -> startswith(x, "Base") ||
                         startswith(x, "Main"), packages)
             else
                 ## exclude them
-                packages = filter(x -> !startswith(x, "Base") &&
+                packages = filter(
+                    x -> !startswith(x, "Base") &&
                         !startswith(x, "Main"), packages)
             end
             append!(package_names, Symbol.(packages))
@@ -252,7 +254,8 @@ function extract_code_blocks(markdown_content::T) where {T <: AbstractString}
     end
 
     # Filter out nested blocks (only if they have full overlap)
-    filtered_positions = filter(inner -> !any(outer -> (outer[1] < inner[1]) &&
+    filtered_positions = filter(
+        inner -> !any(outer -> (outer[1] < inner[1]) &&
                 (inner[2] < outer[2]),
             block_positions),
         block_positions)
@@ -434,8 +437,9 @@ function detect_base_main_overrides(code_block::AbstractString)
     base_imports = extract_julia_imports(code_block; base_or_main = true) .|>
                    x -> split(string(x), ".")[end]
     ## check Base/Main method overrides
-    overriden_methods = filter(f -> occursin("Base.", f) || occursin("Main.", f) ||
-                                        in(f, base_imports),
+    overriden_methods = filter(
+        f -> occursin("Base.", f) || occursin("Main.", f) ||
+                 in(f, base_imports),
         funcs)
     detected = !isempty(overriden_methods)
     return detected, overriden_methods

src/llm_ollama_managed.jl

@@ -79,7 +79,8 @@ Simple wrapper for a call to Ollama API.
 - `port`: The port of the Ollama API. Defaults to 11434.
 - `kwargs`: Prompt variables to be used to fill the prompt/template
 """
-function ollama_api(prompt_schema::Union{AbstractOllamaManagedSchema, AbstractOllamaSchema},
+function ollama_api(
+        prompt_schema::Union{AbstractOllamaManagedSchema, AbstractOllamaSchema},
         prompt::Union{AbstractString, Nothing} = nothing;
         system::Union{Nothing, AbstractString} = nothing,
         messages::Vector{<:AbstractDict{String, <:Any}} = Vector{Dict{String, Any}}(),
@@ -196,7 +197,8 @@ msg = aigenerate(schema, conversation; model="openhermes2.5-mistral")
 
 Note: Managed Ollama currently supports at most 1 User Message and 1 System Message given the API limitations. If you want more, you need to use the `ChatMLSchema`.
 """
-function aigenerate(prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE;
+function aigenerate(
+        prompt_schema::AbstractOllamaManagedSchema, prompt::ALLOWED_PROMPT_TYPE;
         verbose::Bool = true,
         api_key::String = "",
         model::String = MODEL_CHAT,
@@ -348,12 +350,12 @@ function aiembed(prompt_schema::AbstractOllamaManagedSchema,
     model_id = get(MODEL_ALIASES, model, model)
     ## Send each document individually (no parallelism)
     messages = [aiembed(prompt_schema,
-        doc,
-        postprocess;
-        verbose = false,
-        api_key,
-        model = model_id,
-        kwargs...)
+                    doc,
+                    postprocess;
+                    verbose = false,
+                    api_key,
+                    model = model_id,
+                    kwargs...)
                 for doc in docs]
     ## Aggregate results
     msg = DataMessage(;

src/llm_openai.jl

@@ -1429,4 +1429,4 @@ function aiimage(prompt_schema::AbstractOpenAISchema, prompt::ALLOWED_PROMPT_TYP
         kwargs...)
 
     return output
-end
+end

src/llm_shared.jl

@@ -32,7 +32,7 @@ function render(schema::NoSchema,
         if msg isa Union{SystemMessage, UserMessage, UserMessageWithImages}
             replacements = ["{{$(key)}}" => value
                             for (key, value) in pairs(replacement_kwargs)
-                                if key in msg.variables]
+                            if key in msg.variables]
             # Rebuild the message with the replaced content
             MSGTYPE = typeof(msg)
             new_msg = MSGTYPE(;

src/messages.jl

@@ -508,4 +508,4 @@ function pprint(
     for msg in conversation
         pprint(io, msg; text_width)
     end
-end
+end

src/precompilation.jl

@@ -6,12 +6,14 @@ load_templates!();
 @load_preference("MODEL_CHAT", default="x")
 
 # API Calls prep
-mock_response = Dict(:choices => [
-        Dict(:message => Dict(:content => "Hello!",
-                :tool_calls => [
-                    Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1)))),
-                ]),
-            :finish_reason => "stop"),
+mock_response = Dict(
+    :choices => [
+        Dict(
+        :message => Dict(:content => "Hello!",
+            :tool_calls => [
+                Dict(:function => Dict(:arguments => JSON3.write(Dict(:x => 1))))
+            ]),
+        :finish_reason => "stop")
     ],
     :usage => Dict(:total_tokens => 3, :prompt_tokens => 2, :completion_tokens => 1))
 schema = TestEchoOpenAISchema(; response = mock_response, status = 200)

src/serialization.jl

@@ -109,4 +109,4 @@ end
 function save_conversations(filename::AbstractString,
         conversations::Vector{<:AbstractVector{<:AbstractMessage}})
     save_conversations(ShareGPTSchema(), filename, conversations)
-end
+end

templates/critic/ChiefEditorTranscriptCritic.json

@@ -0,0 +1,21 @@
+[
+    {
+        "content": "Template Metadata",
+        "description": "Chief editor auto-reply critic template that critiques a text written by AI assistant. Returns answers with fields: Reflections, Suggestions, Outcome (REVISE/DONE). Placeholders: `transcript`",
+        "version": "1.0",
+        "source": "",
+        "_type": "metadatamessage"
+    },
+    {
+        "content": "Act as a world-class Chief Editor specialized in critiquing a variety of written texts such as blog posts, reports, and other documents as specified by user instructions.\n\nYou will be provided a transcript of conversation between a user and an AI writer assistant.\nYour task is to review the text written by the AI assistant, understand the intended audience, purpose, and context as described by the user, and provide a constructive critique for the AI writer to enhance their work.\n\n**Response Format:**\n----------\nChief Editor says:\nReflection: [provide a reflection on the submitted text, focusing on how well it meets the intended purpose and audience, along with evaluating content accuracy, clarity, style, grammar, and engagement]\nSuggestions: [offer detailed critique with specific improvement points tailored to the user's instructions, such as adjustments in tone, style corrections, structural reorganization, and enhancing readability and engagement]\nOutcome: [DONE or REVISE]\n----------\n\n**Instructions:**\n- Always follow the three-step workflow: Reflection, Suggestions, Outcome.\n- Begin by understanding the user's instructions which may define the text's target audience, desired tone, length, and key messaging goals.\n- Analyze the text to assess how well it aligns with these instructions and its effectiveness in reaching the intended audience.\n- Be extremely strict about adherence to user's instructions.\n- Reflect on aspects such as clarity of expression, content relevance, stylistic consistency, and grammatical integrity.\n- Provide actionable suggestions to address any discrepancies between the text and the user's goals. Emphasize improvements in content organization, clarity, engagement, and adherence to stylistic guidelines.\n- Consider the text's overall impact and how well it communicates its message to the intended audience.\n- Be pragmatic. If the text closely meets the user's requirements and professional standards, conclude with \"Outcome: DONE\".\n- If adjustments are needed to better align with the user's goals or enhance clarity and impact, indicate \"Outcome: REVISE\".\n\n",
+        "variables": [],
+        "_type": "systemmessage"
+    },
+    {
+        "content": "**Conversation Transcript:**\n----------\n{{transcript}}\n----------\n\nRemember to follow the three-step workflow: Reflection, Suggestions, Outcome.\n\nChief Editor says: ",
+        "variables": [
+            "transcript"
+        ],
+        "_type": "usermessage"
+    }
+]

templates/critic/GenericTranscriptCritic.json

@@ -0,0 +1,21 @@
+[
+    {
+        "content": "Template Metadata",
+        "description": "Generic auto-reply critic template that critiques a given conversation transcript. Returns answers with fields: Reflections, Suggestions, Outcome (REVISE/DONE). Placeholders: `transcript`",
+        "version": "1.0",
+        "source": "",
+        "_type": "metadatamessage"
+    },
+    {
+        "content": "Act as a world-class critic specialized in the domain of the user's request.\n\nYour task is to review a transcript of the conversation between a user and AI assistant and provide a helpful critique for the AI assistant to improve their answer.\n\n**Response Format:**\n----------\nCritic says:\nReflection: [provide a reflection on the user request and the AI assistant's answers]\nSuggestions: [provide helpful critique with specific improvement points]\nOutcome: [DONE or REVISE]\n----------\n\n**Instructions:**\n- Always follow the three-step workflow: Reflection, Suggestions, Outcome.\n- Analyze the user request to identify its constituent parts (e.g., requirements, constraints, goals)\n- Reflect on the conversation between the user and the AI assistant. Highlight any ambiguities, inconsistencies, or unclear aspects in the assistant's answers.\n- Generate a list of specific, actionable suggestions for improving the request (if they have not been addressed yet)\n- Provide explanations for each suggestion, highlighting what is missing or unclear\n- Be pragmatic. If the conversation is satisfactory or close to satisfactory, finish with \"Outcome: DONE\".\n- Evaluate the completeness and clarity of the AI Assistant's responses based on the reflections. If the assistant's answer requires revisions or clarification, finish your response with \"Outcome: REVISE\"\n  ",
+        "variables": [],
+        "_type": "systemmessage"
+    },
+    {
+        "content": "**Conversation Transcript:**\n----------\n{{transcript}}\n----------\n\nRemember to follow the three-step workflow: Reflection, Suggestions, Outcome.\n\nCritic says:",
+        "variables": [
+            "transcript"
+        ],
+        "_type": "usermessage"
+    }
+]