Skip to content

Commit

Permalink
Touchups for pr tool calling
Browse files Browse the repository at this point in the history
* branch naming convention
* hallucination minimization
  • Loading branch information
michaeljguarino committed Dec 3, 2024
1 parent d05fc9f commit 982786d
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 24 deletions.
3 changes: 3 additions & 0 deletions lib/console/ai/evidence/base.ex
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ defmodule Console.AI.Evidence.Base do
def prepend(list, l) when is_list(l), do: l ++ list
def prepend(list, msg), do: [msg | list]

def append(list, l) when is_list(l), do: list ++ l
def append(list, msg), do: list ++ [msg]

def distro(:byok), do: "vanilla"
def distro(distro), do: distro

Expand Down
54 changes: 34 additions & 20 deletions lib/console/ai/fixer.ex
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ defmodule Console.AI.Fixer do
Owns logic for generating service/stack/etc insight fix recommendations
"""
use Console.Services.Base
import Console.AI.Evidence.Base, only: [prepend: 2, append: 2]
import Console.AI.Policy
alias Console.Schema.{AiInsight, Service, Stack, User, PullRequest}
alias Console.AI.Fixer.Service, as: ServiceFixer
Expand All @@ -12,7 +13,7 @@ defmodule Console.AI.Fixer do
@prompt """
Please provide the most straightforward code or configuration change available based on the information I've already provided above to fix this issue.
Be sure to explicitly state the Git repository and full file names that are needed to change, alongside the complete content of the files that need to be modified.
Be sure to explicitly state the Git repository and full file names that are needed to change, alongside the content of the files that need to be modified with enough surrounding context to understand what changed.
"""

@tool """
Expand Down Expand Up @@ -42,18 +43,12 @@ defmodule Console.AI.Fixer do
Generate a fix recommendation from an ai insight struct
"""
@spec pr(AiInsight.t, Provider.history) :: {:ok, PullRequest.t} | Console.error
def pr(%AiInsight{service: %Service{} = svc, text: text}, history) do
pr_prompt(text, "service", history)
|> ask(@tool)
|> Provider.tool_call([Pr])
|> handle_tool_call(%{service_id: svc.id})
end

def pr(%AiInsight{stack: %Stack{} = stack, text: text}, history) do
pr_prompt(text, "stack", history)
|> ask(@tool)
|> Provider.tool_call([Pr])
|> handle_tool_call(%{stack_id: stack.id})
def pr(%AiInsight{service: svc, stack: stack} = insight, history) when is_map(svc) or is_map(stack) do
with {:ok, prompt} <- pr_prompt(insight, history) do
ask(prompt, @tool)
|> Provider.tool_call([Pr])
|> handle_tool_call(pluck(insight))
end
end

def pr(_, _), do: {:error, "ai fix recommendations not supported for this insight"}
Expand Down Expand Up @@ -93,15 +88,34 @@ defmodule Console.AI.Fixer do

defp ask(prompt, task \\ @prompt), do: prompt ++ [{:user, task}]

defp pr_prompt(insight, scope, history) when is_list(history) do
[
{:user, """
We've found an issue with a failing Plural #{scope}:
defp pr_prompt(%AiInsight{text: insight} = i, history) do
with {:ok, msgs} <- fix_prompt(i) do
msgs
|> prepend({:user, """
We've found an issue with a failing Plural #{insight_scope(i)}:
#{insight}
We've also found the appropriate fix. I'll list it below:
"""} | history
]
We'll want to make a code change to fix the issue identified. Here's the evidence used to generate the code change:
"""})
|> maybe_add_fix(history)
|> ok()
end
end

defp fix_prompt(%AiInsight{stack: %Stack{} = stack, text: text}), do: StackFixer.prompt(stack, text)
defp fix_prompt(%AiInsight{service: %Service{} = stack, text: text}), do: ServiceFixer.prompt(stack, text)

defp insight_scope(%AiInsight{service: %Service{}}), do: :service
defp insight_scope(%AiInsight{stack: %Stack{}}), do: :stack

defp pluck(%AiInsight{service: %Service{id: id}}), do: %{service_id: id}
defp pluck(%AiInsight{stack: %Stack{id: id}}), do: %{stack_id: id}

# defp maybe_add_fix(prompt, [_ | _] = history) do
# prompt
# |> append({:user, "We've also found a code change needed to fix the above issue, described below. Note that sometimes you'll need to merge this fix in with the current content of the described file:"})
# |> append(history)
# end
defp maybe_add_fix(prompt, _), do: prompt
end
3 changes: 2 additions & 1 deletion lib/console/ai/tools/pr.ex
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ defmodule Console.AI.Tools.Pr do
def description(), do: "Creates a pull request or merge request against a configured Source Control Management provider"

def implement(%__MODULE__{repo_url: url, branch_name: branch, commit_message: msg} = pr) do
branch = "plrl/ai/#{branch}-#{Console.rand_alphanum(6)}"
with {:conn, %ScmConnection{} = conn} <- {:conn, Tool.scm_connection()},
conn <- %{conn | author: Tool.actor()},
url = to_http(conn, url),
Expand All @@ -43,7 +44,7 @@ defmodule Console.AI.Tools.Pr do
{:ok, _} <- commit(conn, msg),
{:ok, _} <- push(conn, branch),
{:ok, identifier} <- slug(conn, url),
impl <- Dispatcher.dispatcher(conn) do
impl = Dispatcher.dispatcher(conn) do
impl.create(%PrAutomation{
connection: conn,
title: pr.pr_title,
Expand Down
6 changes: 3 additions & 3 deletions test/console/ai/fixer_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ defmodule Console.AI.FixerTest do
describe "#pr/2" do
test "it can spawn a fix pr" do
insert(:scm_connection, token: "some-pat", default: true)
expect(Tentacat.Pulls, :create, fn _, "pluralsh", "console", %{head: "pr-test"} ->
expect(Tentacat.Pulls, :create, fn _, "pluralsh", "console", %{head: "plrl/ai/pr-test" <> _} ->
{:ok, %{"html_url" => "https://github.com/pr/url"}, %HTTPoison.Response{}}
end)
expect(Console.Deployments.Pr.Git, :setup, fn conn, "https://github.com/pluralsh/console.git", "pr-test" ->
expect(Console.Deployments.Pr.Git, :setup, fn conn, "https://github.com/pluralsh/console.git", "plrl/ai/pr-test" <> _ ->
{:ok, %{conn | dir: Briefly.create!(directory: true)}}
end)
expect(Console.Deployments.Pr.Git, :commit, fn _, _ -> {:ok, ""} end)
expect(Console.Deployments.Pr.Git, :push, fn _, "pr-test" -> {:ok, ""} end)
expect(Console.Deployments.Pr.Git, :push, fn _, "plrl/ai/pr-test" <> _ -> {:ok, ""} end)
expect(File, :write, fn _, "first" -> :ok end)
expect(File, :write, fn _, "second" -> :ok end)
expect(HTTPoison, :post, fn _, _, _, _ ->
Expand Down

0 comments on commit 982786d

Please sign in to comment.