Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/environment var creds #116

Merged
merged 13 commits into from
Oct 14, 2024
Merged

Feature/environment var creds #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
e214a97
Azure env vars can provide endpoint/token
cybermaggedon Oct 13, 2024
1588d6a
Other env vars
cybermaggedon Oct 14, 2024
0369072
Update Azure with new macros for env vars in secrets
cybermaggedon Oct 14, 2024
ed605f0
New macros working with Docker Compose
cybermaggedon Oct 14, 2024
3b3809f
Partial K8s implementation
cybermaggedon Oct 14, 2024
a880ca2
Mostly working in k8s
cybermaggedon Oct 14, 2024
b7f5ec0
Mapping secret to envs looks correct
cybermaggedon Oct 14, 2024
56cfe31
Tidier templating for environment variables
cybermaggedon Oct 14, 2024
735c8f1
Updated Azure template
cybermaggedon Oct 14, 2024
aab6472
More LLMs ported to environment secrets
cybermaggedon Oct 14, 2024
0eee19b
LLM migration to env secrets complete
cybermaggedon Oct 14, 2024
fc87c12
Fix Ollama host
cybermaggedon Oct 14, 2024
cc29e6a
Updating templates
cybermaggedon Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 19 additions & 30 deletions templates/components/azure-openai.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ local prompts = import "prompts/mixtral.jsonnet";

{

"azure-openai-token":: "${AZURE_OPENAI_TOKEN}",
"azure-openai-model":: "GPT-3.5-Turbo",
"azure-openai-max-output-tokens":: 4192,
"azure-openai-temperature":: 0.0,
Expand All @@ -14,55 +13,34 @@ local prompts = import "prompts/mixtral.jsonnet";

create:: function(engine)

local envSecrets = engine.envSecrets("azure-openai-credentials")
.with_env_var("AZURE_TOKEN", "azure-token");

local container =
engine.container("text-completion")
.with_image(images.trustgraph)
.with_command([
"text-completion-azure-openai",
"-p",
url.pulsar,
"-k",
$["azure-openai-token"],
"-m",
$["azure-openai-model"],
"-x",
std.toString($["azure-openai-max-output-tokens"]),
"-t",
std.toString($["azure-openai-temperature"]),
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion", [ container ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

engine.resources([
containerSet,
service,
])

},

"text-completion-rag" +: {

create:: function(engine)

local container =
local containerRag =
engine.container("text-completion-rag")
.with_image(images.trustgraph)
.with_command([
"text-completion-azure",
"-p",
url.pulsar,
"-k",
$["azure-openai-token"],
"-e",
$["azure-openai-model"],
"-x",
std.toString($["azure-openai-max-output-tokens"]),
"-t",
Expand All @@ -72,24 +50,35 @@ local prompts = import "prompts/mixtral.jsonnet";
"-o",
"non-persistent://tg/response/text-completion-rag-response",
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion-rag", [ container ]
"text-completion", [ container ]
);

local containerSetRag = engine.containers(
"text-completion-rag", [ containerRag ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

local serviceRag =
engine.internalService(containerSetRag)
.with_port(8000, 8000, "metrics");

engine.resources([
envSecrets,
containerSet,
containerSetRag,
service,
serviceRag,
])


}
},

} + prompts

51 changes: 19 additions & 32 deletions templates/components/azure.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,64 +5,40 @@ local prompts = import "prompts/mixtral.jsonnet";

{

"azure-token":: "${AZURE_TOKEN}",
"azure-endpoint":: "${AZURE_ENDPOINT}",
"azure-max-output-tokens":: 4096,
"azure-temperature":: 0.0,

"text-completion" +: {

create:: function(engine)

local envSecrets = engine.envSecrets("azure-credentials")
.with_env_var("AZURE_TOKEN", "azure-token")
.with_env_var("AZURE_ENDPOINT", "azure-endpoint");

local container =
engine.container("text-completion")
.with_image(images.trustgraph)
.with_command([
"text-completion-azure",
"-p",
url.pulsar,
"-k",
$["azure-token"],
"-e",
$["azure-endpoint"],
"-x",
std.toString($["azure-max-output-tokens"]),
"-t",
std.toString($["azure-temperature"]),
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion", [ container ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

engine.resources([
containerSet,
service,
])

},

"text-completion-rag" +: {

create:: function(engine)

local container =
local containerRag =
engine.container("text-completion-rag")
.with_image(images.trustgraph)
.with_command([
"text-completion-azure",
"-p",
url.pulsar,
"-k",
$["azure-token"],
"-e",
$["azure-endpoint"],
"-x",
std.toString($["azure-max-output-tokens"]),
"-t",
Expand All @@ -72,23 +48,34 @@ local prompts = import "prompts/mixtral.jsonnet";
"-o",
"non-persistent://tg/response/text-completion-rag-response",
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion-rag", [ container ]
"text-completion", [ container ]
);

local containerSetRag = engine.containers(
"text-completion-rag", [ containerRag ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

local serviceRag =
engine.internalService(containerSetRag)
.with_port(8000, 8000, "metrics");

engine.resources([
envSecrets,
containerSet,
containerSetRag,
service,
serviceRag,
])


}

} + prompts
Expand Down
59 changes: 21 additions & 38 deletions templates/components/bedrock.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ local chunker = import "chunker-recursive.jsonnet";

{

"aws-id-key":: "${AWS_ID_KEY}",
"aws-secret-key":: "${AWS_SECRET_KEY}",
"aws-region":: "us-west-2",
"bedrock-max-output-tokens":: 4096,
"bedrock-temperature":: 0.0,
"bedrock-model":: "mistral.mixtral-8x7b-instruct-v0:1",
Expand All @@ -17,61 +14,36 @@ local chunker = import "chunker-recursive.jsonnet";

create:: function(engine)

local envSecrets = engine.envSecrets("bedrock-credentials")
.with_env_var("AWS_ID_KEY", "aws-id-key")
.with_env_var("AWS_SECRET_KEY", "aws-secret-key")
.with_env_var("AWS_REGION", "aws-region");

local container =
engine.container("text-completion")
.with_image(images.trustgraph)
.with_command([
"text-completion-bedrock",
"-p",
url.pulsar,
"-z",
$["aws-id-key"],
"-k",
$["aws-secret-key"],
"-r",
$["aws-region"],
"-x",
std.toString($["bedrock-max-output-tokens"]),
"-t",
std.toString($["bedrock-temperature"]),
"-m",
$["bedrock-model"],
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion", [ container ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

engine.resources([
containerSet,
service,
])

},

"text-completion-rag" +: {

create:: function(engine)

local container =
local containerRag =
engine.container("text-completion-rag")
.with_image(images.trustgraph)
.with_command([
"text-completion-bedrock",
"-p",
url.pulsar,
"-z",
$["aws-id-key"],
"-k",
$["aws-secret-key"],
"-r",
$["aws-region"],
"-x",
std.toString($["bedrock-max-output-tokens"]),
"-t",
Expand All @@ -83,24 +55,35 @@ local chunker = import "chunker-recursive.jsonnet";
"-o",
"non-persistent://tg/response/text-completion-rag-response",
])
.with_env_var_secrets(envSecrets)
.with_limits("0.5", "128M")
.with_reservations("0.1", "128M");

local containerSet = engine.containers(
"text-completion-rag", [ container ]
"text-completion", [ container ]
);

local containerSetRag = engine.containers(
"text-completion-rag", [ containerRag ]
);

local service =
engine.internalService(containerSet)
.with_port(8000, 8000, "metrics");

local serviceRag =
engine.internalService(containerSetRag)
.with_port(8000, 8000, "metrics");

engine.resources([
envSecrets,
containerSet,
containerSetRag,
service,
serviceRag,
])


}
},

} + prompts + chunker

Loading
Loading