Touchups for pr tool calling

* branch naming convention
* hallucination minimization

lib/console/ai/evidence/base.ex

@@ -27,6 +27,9 @@ defmodule Console.AI.Evidence.Base do
   def prepend(list, l) when is_list(l), do: l ++ list
   def prepend(list, msg), do: [msg | list]
 
+  def append(list, l) when is_list(l), do: list ++ l
+  def append(list, msg), do: list ++ [msg]
+
   def distro(:byok), do: "vanilla"
   def distro(distro), do: distro
 

lib/console/ai/fixer.ex

@@ -3,6 +3,7 @@ defmodule Console.AI.Fixer do
   Owns logic for generating service/stack/etc insight fix recommendations
   """
   use Console.Services.Base
+  import Console.AI.Evidence.Base, only: [prepend: 2, append: 2]
   import Console.AI.Policy
   alias Console.Schema.{AiInsight, Service, Stack, User, PullRequest}
   alias Console.AI.Fixer.Service, as: ServiceFixer
@@ -12,12 +13,14 @@ defmodule Console.AI.Fixer do
   @prompt """
   Please provide the most straightforward code or configuration change available based on the information I've already provided above to fix this issue.
 
-  Be sure to explicitly state the Git repository and full file names that are needed to change, alongside the complete content of the files that need to be modified.
+  Be sure to explicitly state the Git repository and full file names that are needed to change, alongside the content of the files that need to be modified with enough surrounding context to understand what changed.
   """
 
   @tool """
   Please spawn a Pull Request to fix the issue described above.  The code change should be the most direct
-  and straightforward way to fix the issue described, avoid any extraneous changes or modifying files not listed.
+  and straightforward way to fix the issue described.  Change only the minimal amount of lines in the original files
+  provided to successfully fix the issue, avoid any extraneous changes as they will potentially break additional
+  functionality upon application.
   """
 
   @callback prompt(struct, binary) :: {:ok, Provider.history} | Console.error
@@ -42,18 +45,12 @@ defmodule Console.AI.Fixer do
   Generate a fix recommendation from an ai insight struct
   """
   @spec pr(AiInsight.t, Provider.history) :: {:ok, PullRequest.t} | Console.error
-  def pr(%AiInsight{service: %Service{} = svc, text: text}, history) do
-    pr_prompt(text, "service", history)
-    |> ask(@tool)
-    |> Provider.tool_call([Pr])
-    |> handle_tool_call(%{service_id: svc.id})
-  end
-
-  def pr(%AiInsight{stack: %Stack{} = stack, text: text}, history) do
-    pr_prompt(text, "stack", history)
-    |> ask(@tool)
-    |> Provider.tool_call([Pr])
-    |> handle_tool_call(%{stack_id: stack.id})
+  def pr(%AiInsight{service: svc, stack: stack} = insight, history) when is_map(svc) or is_map(stack) do
+    with {:ok, prompt} <- pr_prompt(insight, history) do
+      ask(prompt, @tool)
+      |> Provider.tool_call([Pr])
+      |> handle_tool_call(pluck(insight))
+    end
   end
 
   def pr(_, _), do: {:error, "ai fix recommendations not supported for this insight"}
@@ -93,15 +90,34 @@ defmodule Console.AI.Fixer do
 
   defp ask(prompt, task \\ @prompt), do: prompt ++ [{:user, task}]
 
-  defp pr_prompt(insight, scope, history) when is_list(history) do
-    [
-      {:user, """
-      We've found an issue with a failing Plural #{scope}:
+  defp pr_prompt(%AiInsight{text: insight} = i, history) do
+    with {:ok, msgs} <- fix_prompt(i) do
+      msgs
+      |> prepend({:user, """
+      We've found an issue with a failing Plural #{insight_scope(i)}:
 
       #{insight}
 
-      We've also found the appropriate fix. I'll list it below:
-      """} | history
-    ]
+      We'll want to make a code change to fix the issue identified.  Here's the evidence used to generate the code change:
+      """})
+      |> maybe_add_fix(history)
+      |> ok()
+    end
+  end
+
+  defp fix_prompt(%AiInsight{stack: %Stack{} = stack, text: text}), do: StackFixer.prompt(stack, text)
+  defp fix_prompt(%AiInsight{service: %Service{} = stack, text: text}), do: ServiceFixer.prompt(stack, text)
+
+  defp insight_scope(%AiInsight{service: %Service{}}), do: :service
+  defp insight_scope(%AiInsight{stack: %Stack{}}), do: :stack
+
+  defp pluck(%AiInsight{service: %Service{id: id}}), do: %{service_id: id}
+  defp pluck(%AiInsight{stack: %Stack{id: id}}), do: %{stack_id: id}
+
+  defp maybe_add_fix(prompt, [_ | _] = history) do
+    prompt
+    |> append({:user, "We've also found a code change needed to fix the above issue, described below.  Note that sometimes this will sometimes represent a PARTIAL change to the underlying file, don't delete unrelated content if that's not what's relevant to change:"})
+    |> append(history)
   end
+  defp maybe_add_fix(prompt, _), do: prompt
 end

lib/console/ai/provider/openai.ex

@@ -12,7 +12,7 @@ defmodule Console.AI.OpenAI do
 
   def default_model(), do: @model
 
-  defstruct [:access_key, :model, :base_url, :params, :stream]
+  defstruct [:access_key, :model, :tool_model, :base_url, :params, :stream]
 
   @type t :: %__MODULE__{}
 
@@ -83,7 +83,7 @@ defmodule Console.AI.OpenAI do
   @spec tool_call(t(), Console.AI.Provider.history, [atom]) :: {:ok, binary} | {:ok, [Console.AI.Tool.t]} | Console.error
   def tool_call(%__MODULE__{} = openai, messages, tools) do
     history = Enum.map(messages, fn {role, msg} -> %{role: role, content: msg} end)
-    case chat(%{openai | stream: nil}, history, tools) do
+    case chat(%{openai | stream: nil, model: tool_model(openai)}, history, tools) do
       {:ok, %CompletionResponse{choices: [%Choice{message: %Message{tool_calls: [_ | _] = calls}} | _]}} ->
         {:ok, gen_tools(calls)}
       {:ok, %CompletionResponse{choices: [%Choice{message: %Message{content: content}} | _]}} ->
@@ -148,6 +148,8 @@ defmodule Console.AI.OpenAI do
     |> Enum.filter(& &1)
   end
 
+  defp tool_model(%__MODULE__{model: m, tool_model: tm}), do: tm || m || "o1-mini"
+
   defp tool_args(tool) do
     %{
       type: :function,

lib/console/ai/provider/vertex.ex

@@ -30,7 +30,12 @@ defmodule Console.AI.Vertex do
   @spec completion(t(), Console.AI.Provider.history) :: {:ok, binary} | Console.error
   def completion(%__MODULE__{} = vertex, messages) do
     with {:ok, %{token: token}} <- client(vertex) do
-      OpenAI.new(base_url: openai_url(vertex), access_token: token, model: openai_model(vertex))
+      OpenAI.new(
+        base_url: openai_url(vertex),
+        access_token: token,
+        model: openai_model(vertex),
+        tool_model: openai_model(vertex)
+      )
       |> OpenAI.completion(messages)
     end
   end
@@ -41,7 +46,12 @@ defmodule Console.AI.Vertex do
   @spec tool_call(t(), Console.AI.Provider.history, [atom]) :: {:ok, binary} | {:ok, [Console.AI.Tool.t]} | Console.error
   def tool_call(%__MODULE__{} = vertex, messages, tools) do
     with {:ok, %{token: token}} <- client(vertex) do
-      OpenAI.new(base_url: openai_url(vertex), access_token: token, model: openai_model(vertex))
+      OpenAI.new(
+        base_url: openai_url(vertex),
+        access_token: token,
+        model: openai_model(vertex),
+        tool_model: openai_model(vertex)
+      )
       |> OpenAI.tool_call(messages, tools)
     end
   end

lib/console/ai/tools/pr.ex

@@ -35,6 +35,7 @@ defmodule Console.AI.Tools.Pr do
   def description(), do: "Creates a pull request or merge request against a configured Source Control Management provider"
 
   def implement(%__MODULE__{repo_url: url, branch_name: branch, commit_message: msg} = pr) do
+    branch = "plrl/ai/#{branch}-#{Console.rand_alphanum(6)}"
     with {:conn, %ScmConnection{} = conn} <- {:conn, Tool.scm_connection()},
          conn <- %{conn | author: Tool.actor()},
          url = to_http(conn, url),
@@ -43,7 +44,7 @@ defmodule Console.AI.Tools.Pr do
          {:ok, _} <- commit(conn, msg),
          {:ok, _} <- push(conn, branch),
          {:ok, identifier} <- slug(conn, url),
-         impl <- Dispatcher.dispatcher(conn) do
+         impl = Dispatcher.dispatcher(conn) do
       impl.create(%PrAutomation{
         connection: conn,
         title: pr.pr_title,

test/console/ai/fixer_test.exs

@@ -6,14 +6,14 @@ defmodule Console.AI.FixerTest do
   describe "#pr/2" do
     test "it can spawn a fix pr" do
       insert(:scm_connection, token: "some-pat", default: true)
-      expect(Tentacat.Pulls, :create, fn _, "pluralsh", "console", %{head: "pr-test"} ->
+      expect(Tentacat.Pulls, :create, fn _, "pluralsh", "console", %{head: "plrl/ai/pr-test" <> _} ->
         {:ok, %{"html_url" => "https://github.com/pr/url"}, %HTTPoison.Response{}}
       end)
-      expect(Console.Deployments.Pr.Git, :setup, fn conn, "https://github.com/pluralsh/console.git", "pr-test" ->
+      expect(Console.Deployments.Pr.Git, :setup, fn conn, "https://github.com/pluralsh/console.git", "plrl/ai/pr-test" <> _ ->
         {:ok, %{conn | dir: Briefly.create!(directory: true)}}
       end)
       expect(Console.Deployments.Pr.Git, :commit, fn _, _ -> {:ok, ""} end)
-      expect(Console.Deployments.Pr.Git, :push, fn _, "pr-test" -> {:ok, ""} end)
+      expect(Console.Deployments.Pr.Git, :push, fn _, "plrl/ai/pr-test" <> _ -> {:ok, ""} end)
       expect(File, :write, fn _, "first" -> :ok end)
       expect(File, :write, fn _, "second" -> :ok end)
       expect(HTTPoison, :post, fn _, _, _, _ ->