diff --git a/fixcore/fixcore/cli/command.py b/fixcore/fixcore/cli/command.py index 772256f0d7..09e1e22558 100644 --- a/fixcore/fixcore/cli/command.py +++ b/fixcore/fixcore/cli/command.py @@ -4240,7 +4240,7 @@ async def perform_request(e: JsonElement) -> int: headers=template.headers, params=template.params, data=data, - compress=template.compress, + compress="deflate" if template.compress else None, timeout=template.timeout, ssl=False if template.no_ssl_verify else self.dependencies.cert_handler.client_context, auth=BasicAuth(login=authuser, password=(authpass if authpass else "")) if authuser else None, diff --git a/fixcore/fixcore/db/db_access.py b/fixcore/fixcore/db/db_access.py index d4c644a84e..e6c5ebcce3 100644 --- a/fixcore/fixcore/db/db_access.py +++ b/fixcore/fixcore/db/db_access.py @@ -275,7 +275,7 @@ def get_graph_db(self, name: GraphName, no_check: bool = False) -> GraphDB: else: if not no_check and not self.database.has_graph(name): raise NoSuchGraph(name) - graph_db = ArangoGraphDB(self.db, name, self.adjust_node, self.config.graph) + graph_db = ArangoGraphDB(self.db, name, self.adjust_node, self.config.graph, self.lock_db) event_db = EventGraphDB(graph_db, self.event_sender) self.graph_dbs[name] = event_db return event_db diff --git a/fixcore/fixcore/db/graphdb.py b/fixcore/fixcore/db/graphdb.py index b34fd8e8da..868440e1b6 100644 --- a/fixcore/fixcore/db/graphdb.py +++ b/fixcore/fixcore/db/graphdb.py @@ -9,7 +9,6 @@ from numbers import Number from textwrap import dedent from typing import ( - DefaultDict, Optional, Callable, AsyncGenerator, @@ -18,7 +17,6 @@ Dict, List, Tuple, - TypeVar, cast, AsyncIterator, Literal, @@ -39,7 +37,8 @@ from fixcore.db import arango_query, EstimatedSearchCost from fixcore.db.arango_query import fulltext_delimiter from fixcore.db.async_arangodb import AsyncArangoDB, AsyncArangoTransactionDB, AsyncArangoDBBase, AsyncCursorContext -from fixcore.db.model import GraphUpdate, QueryModel +from fixcore.db.lockdb import LockDB +from fixcore.db.model import GraphUpdate, QueryModel, GraphChange from fixcore.db.usagedb import resource_usage_db from fixcore.error import InvalidBatchUpdate, ConflictingChangeInProgress, NoSuchChangeError, OptimisticLockingFailed from fixcore.ids import NodeId, GraphName @@ -275,6 +274,7 @@ def __init__( name: GraphName, adjust_node: AdjustNode, config: GraphConfig, + lock_db: LockDB, ) -> None: super().__init__() self._name = name @@ -283,6 +283,7 @@ def __init__( self.in_progress = f"{name}_in_progress" self.node_history = f"{name}_node_history" self.usage_db = resource_usage_db(db, f"{name}_usage") + self.lock_db = lock_db self.db = db self.config = config @@ -309,8 +310,8 @@ async def create_node(self, model: Model, node_id: NodeId, data: Json, under_nod graph.add_node(node_id, data) graph.add_edge(under_node_id, node_id, EdgeTypes.default) access = GraphAccess(graph.graph, node_id, {under_node_id}) - _, node_inserts, _, _ = self.prepare_nodes(access, [], model) - _, edge_inserts, _, _ = self.prepare_edges(access, [], EdgeTypes.default) + node_inserts = self.prepare_nodes(access, [], model).node_inserts + edge_inserts = self.prepare_edges(access, [], EdgeTypes.default).edge_inserts[EdgeTypes.default] assert len(node_inserts) == 1 assert len(edge_inserts) == 1 edge_collection = self.edge_collection(EdgeTypes.default) @@ -974,7 +975,7 @@ async def move_temp_to_proper(self, change_id: str, temp_name: str, update_histo + edge_updates + edge_deletes + usage_updates - + [f'remove {{_key: "{change_key}"}} in {self.in_progress}'], + + [f'remove {{_key: "{change_key}"}} in {self.in_progress} OPTIONS {{ ignoreErrors: true }}'], ) ) cmd = f'function () {{\nvar db=require("@arangodb").db;\n{updates}\n}}' @@ -1034,15 +1035,9 @@ def adjust_node( # adjuster has the option to manipulate the resulting json return self.node_adjuster.adjust(json) - def prepare_nodes( - self, access: GraphAccess, node_cursor: Iterable[Json], model: Model - ) -> Tuple[GraphUpdate, List[Json], List[Json], List[Json]]: + def prepare_nodes(self, access: GraphAccess, node_cursor: Iterable[Json], model: Model) -> GraphChange: log.info(f"Prepare nodes for subgraph {access.root()}") - info = GraphUpdate() - resource_inserts: List[Json] = [] - resource_updates: List[Json] = [] - resource_deletes: List[Json] = [] - + change = GraphChange() optional_properties = [*Section.all_ordered, "refs", "kinds", "flat", "hash", "hist_hash"] def insert_node(node: Json) -> None: @@ -1052,8 +1047,7 @@ def insert_node(node: Json) -> None: value = node.get(prop, None) if value: js_doc[prop] = value - resource_inserts.append(js_doc) - info.nodes_created += 1 + change.node_inserts.append(js_doc) def update_or_delete_node(node: Json) -> None: key = node["_key"] @@ -1061,8 +1055,7 @@ def update_or_delete_node(node: Json) -> None: elem = access.node(key) if elem is None: # node is in db, but not in the graph any longer: delete node - resource_deletes.append({"_key": key, "deleted": access.at_json, "history": True}) - info.nodes_deleted += 1 + change.node_deletes.append({"_key": key, "deleted": access.at_json, "history": True}) elif elem["hash"] != hash_string: # node is in db and in the graph, content is different adjusted: Json = self.adjust_node(model, elem, node["created"], access.at_json) @@ -1072,15 +1065,14 @@ def update_or_delete_node(node: Json) -> None: value = adjusted.get(prop, None) if value: js[prop] = value - resource_updates.append(js) - info.nodes_updated += 1 + change.node_updates.append(js) for doc in node_cursor: update_or_delete_node(doc) for not_visited in access.not_visited_nodes(): insert_node(not_visited) - return info, resource_inserts, resource_updates, resource_deletes + return change def _edge_to_json( self, from_node: str, to_node: str, data: Optional[Json], refs: Optional[Dict[str, str]] = None, **kwargs: Any @@ -1096,14 +1088,9 @@ def _edge_to_json( js["_to"] = f"{self.vertex_name}/{to_node}" return js - def prepare_edges( - self, access: GraphAccess, edge_cursor: Iterable[Json], edge_type: EdgeType - ) -> Tuple[GraphUpdate, List[Json], List[Json], List[Json]]: + def prepare_edges(self, access: GraphAccess, edge_cursor: Iterable[Json], edge_type: EdgeType) -> GraphChange: log.info(f"Prepare edges of type {edge_type} for subgraph {access.root()}") - info = GraphUpdate() - edge_inserts: List[Json] = [] - edge_updates: List[Json] = [] - edge_deletes: List[Json] = [] + change = GraphChange() def edge_json(from_node: str, to_node: str, edge_data: Optional[Json]) -> Json: # Take the refs with the lower number of entries (or none): @@ -1117,8 +1104,7 @@ def edge_json(from_node: str, to_node: str, edge_data: Optional[Json]) -> Json: return self._edge_to_json(from_node, to_node, edge_data, refs) def insert_edge(from_node: str, to_node: str, edge_data: Optional[Json]) -> None: - edge_inserts.append(edge_json(from_node, to_node, edge_data)) - info.edges_created += 1 + change.edge_inserts[edge_type].append(edge_json(from_node, to_node, edge_data)) def update_edge(edge: Json) -> None: from_node = edge["_from"].split("/")[1] # vertex/id @@ -1126,12 +1112,10 @@ def update_edge(edge: Json) -> None: has_edge, edge_data = access.has_edge(from_node, to_node, edge_type) edge_hash = edge_data.get("hash") if edge_data else None if not has_edge: - edge_deletes.append(edge) - info.edges_deleted += 1 + change.edge_deletes[edge_type].append(edge) elif edge_hash != edge.get("hash"): js = edge_json(from_node, to_node, edge_data) - edge_updates.append(js) - info.edges_updated += 1 + change.edge_updates[edge_type].append(js) for doc in edge_cursor: update_edge(doc) @@ -1139,7 +1123,7 @@ def update_edge(edge: Json) -> None: for edge_from, edge_to, data in access.not_visited_edges(edge_type): insert_edge(edge_from, edge_to, data) - return info, edge_inserts, edge_updates, edge_deletes + return change async def merge_graph( self, @@ -1154,37 +1138,21 @@ async def merge_graph( async def prepare_graph( sub: GraphAccess, node_query: Tuple[str, Json], edge_query: Callable[[EdgeType], Tuple[str, Json]] - ) -> Tuple[ - GraphUpdate, - List[Json], # node insert - List[Json], # node update - List[Json], # node delete - Dict[EdgeType, List[Json]], # edge insert - Dict[EdgeType, List[Json]], # edge update - Dict[EdgeType, List[Json]], # edge delete - ]: - graph_info = GraphUpdate() + ) -> GraphChange: + graph_change = GraphChange() # check all nodes for this subgraph query, bind = node_query log.debug(f"Query for nodes: {sub.root()}") with await self.db.aql(query, bind_vars=bind, batch_size=50000) as node_cursor: - node_info, ni, nu, nd = self.prepare_nodes(sub, node_cursor, model) - graph_info += node_info + graph_change += self.prepare_nodes(sub, node_cursor, model) # check all edges in all relevant edge-collections - edge_inserts: DefaultDict[EdgeType, List[Json]] = defaultdict(list) - edge_updates: DefaultDict[EdgeType, List[Json]] = defaultdict(list) - edge_deletes: DefaultDict[EdgeType, List[Json]] = defaultdict(list) for edge_type in EdgeTypes.all: query, bind = edge_query(edge_type) log.debug(f"Query for edges of type {edge_type}: {sub.root()}") with await self.db.aql(query, bind_vars=bind, batch_size=50000) as ec: - edge_info, gei, geu, ged = self.prepare_edges(sub, ec, edge_type) - graph_info += edge_info - edge_inserts[edge_type] = gei - edge_updates[edge_type] = geu - edge_deletes[edge_type] = ged - return graph_info, ni, nu, nd, edge_inserts, edge_updates, edge_deletes + graph_change += self.prepare_edges(sub, ec, edge_type) + return graph_change roots, parent, graphs = GraphAccess.merge_graphs(graph_to_merge) log.info(f"merge_graph {len(roots)} merge nodes found. change_id={change_id}, is_batch={is_batch}.") @@ -1192,27 +1160,23 @@ async def prepare_graph( def merge_edges(merge_node: str, merge_node_kind: str, edge_type: EdgeType) -> Tuple[str, Json]: return self.query_update_edges(edge_type, merge_node_kind), {"update_id": merge_node} - K = TypeVar("K") # noqa: N806 - V = TypeVar("V") # noqa: N806 - - def combine_dict(left: Dict[K, List[V]], right: Dict[K, List[V]]) -> Dict[K, List[V]]: - result = dict(left) - for key, right_values in right.items(): - left_values = left.get(key) - result[key] = left_values + right_values if left_values else right_values - return result + def parent_edges(edge_type: EdgeType) -> Tuple[str, Json]: + edge_ids = [self.db_edge_key(f, t) for f, t, k in parent.g.edges(keys=True) if k.edge_type == edge_type] + return self.edges_by_ids_and_until_replace_node(edge_type, preserve_parent_structure, parent, edge_ids) # this will throw an exception, in case of a conflicting update (--> outside try block) log.debug("Mark all parent nodes for this update to avoid conflicting changes") await self.mark_update(roots, list(parent.nodes), change_id, is_batch) try: - - def parent_edges(edge_type: EdgeType) -> Tuple[str, Json]: - edge_ids = [self.db_edge_key(f, t) for f, t, k in parent.g.edges(keys=True) if k.edge_type == edge_type] - return self.edges_by_ids_and_until_replace_node(edge_type, preserve_parent_structure, parent, edge_ids) - + # store parent nodes and edges with a mutex to avoid conflicts parents_nodes = self.nodes_by_ids_and_until_replace_node(preserve_parent_structure, parent) - info, nis, nus, nds, eis, eus, eds = await prepare_graph(parent, parents_nodes, parent_edges) + async with self.lock_db.lock("merge_graph_parents"): + parent_change_id = change_id + "_parent" + parent_change = await prepare_graph(parent, parents_nodes, parent_edges) + if parent_change.change_count(): # only persist in case of changes + await self._persist_update(parent_change_id, False, parent_change, update_history) + + change = GraphChange() for num, (root, graph) in enumerate(graphs): root_kind = GraphResolver.resolved_kind(graph_to_merge.nodes[root]) if root_kind: @@ -1220,39 +1184,21 @@ def parent_edges(edge_type: EdgeType) -> Tuple[str, Json]: log.info(f"Update subgraph: root={root} ({root_kind}, {num+1} of {len(roots)})") node_query = self.query_update_nodes(root_kind), {"update_id": root} edge_query = partial(merge_edges, root, root_kind) - - i, ni, nu, nd, ei, eu, ed = await prepare_graph(graph, node_query, edge_query) - info += i - nis += ni - nus += nu - nds += nd - eis = combine_dict(eis, ei) - eus = combine_dict(eus, eu) - eds = combine_dict(eds, ed) + change += await prepare_graph(graph, node_query, edge_query) else: - # Already checked in GraphAccess - only here as safeguard. + # Already checked in GraphAccess - only here as a safeguard. raise AttributeError(f"Kind of update root {root} is not a pre-resolved and can not be used!") - log.debug(f"Update prepared: {info}. Going to persist the changes.") + graph_update = parent_change.to_update() + change.to_update() + log.debug(f"Update prepared: {graph_update}. Going to persist the changes.") await self._refresh_marked_update(change_id) - await self._persist_update(change_id, is_batch, nis, nus, nds, eis, eus, eds, update_history) - return roots, info + await self._persist_update(change_id, is_batch, change, update_history) + return roots, graph_update except Exception as ex: await self.delete_marked_update(change_id) raise ex - async def _persist_update( - self, - change_id: str, - is_batch: bool, - resource_inserts: List[Json], - resource_updates: List[Json], - resource_deletes: List[Json], - edge_inserts: Dict[EdgeType, List[Json]], - edge_updates: Dict[EdgeType, List[Json]], - edge_deletes: Dict[EdgeType, List[Json]], - update_history: bool, - ) -> None: + async def _persist_update(self, change_id: str, is_batch: bool, change: GraphChange, update_history: bool) -> None: async def execute_many_async( async_fn: Callable[[str, List[Json]], Any], name: str, array: List[Json], **kwargs: Any ) -> None: @@ -1275,20 +1221,20 @@ async def trafo_many( async def store_to_tmp_collection(temp: StandardCollection) -> None: tmp = temp.name - ri = trafo_many(self.db.insert_many, tmp, resource_inserts, {"action": "node_created"}) - ru = trafo_many(self.db.insert_many, tmp, resource_updates, {"action": "node_updated"}) - rd = trafo_many(self.db.insert_many, tmp, resource_deletes, {"action": "node_deleted"}) + ri = trafo_many(self.db.insert_many, tmp, change.node_inserts, {"action": "node_created"}) + ru = trafo_many(self.db.insert_many, tmp, change.node_updates, {"action": "node_updated"}) + rd = trafo_many(self.db.insert_many, tmp, change.node_deletes, {"action": "node_deleted"}) edge_i = [ trafo_many(self.db.insert_many, tmp, inserts, {"action": "edge_insert", "edge_type": tpe}) - for tpe, inserts in edge_inserts.items() + for tpe, inserts in change.edge_inserts.items() ] edge_u = [ trafo_many(self.db.insert_many, tmp, updates, {"action": "edge_update", "edge_type": tpe}) - for tpe, updates in edge_updates.items() + for tpe, updates in change.edge_updates.items() ] edge_d = [ trafo_many(self.db.insert_many, tmp, deletes, {"action": "edge_delete", "edge_type": tpe}) - for tpe, deletes in edge_deletes.items() + for tpe, deletes in change.edge_deletes.items() ] await asyncio.gather(*([ri, ru, rd] + edge_i + edge_u + edge_d)) @@ -1562,7 +1508,7 @@ async def copy_graph(self, to_graph: GraphName, to_snapshot: bool = False) -> Gr if await self.db.has_graph(to_graph): raise ValueError(f"Graph {to_graph} already exists") - new_graph_db = ArangoGraphDB(db=self.db, name=to_graph, adjust_node=self.node_adjuster, config=self.config) + new_graph_db = ArangoGraphDB(self.db, to_graph, self.node_adjuster, self.config, self.lock_db) # collection creation can't be a part of a transaction so we do that first # we simply reuse the existing create_update_schema method but do not insert any genesis data diff --git a/fixcore/fixcore/db/model.py b/fixcore/fixcore/db/model.py index 081d14f571..5e5675938d 100644 --- a/fixcore/fixcore/db/model.py +++ b/fixcore/fixcore/db/model.py @@ -1,14 +1,16 @@ from __future__ import annotations -from abc import ABC -from typing import Dict, Any, Optional, Tuple, List +from collections import defaultdict +from typing import Dict, Any, Optional, Tuple, List, DefaultDict from attr import define +from attrs import field -from fixcore.model.graph_access import Section +from fixcore.model.graph_access import Section, EdgeTypes from fixcore.model.model import Model, ResolvedPropertyPath, ComplexKind from fixcore.model.resolve_in_graph import GraphResolver from fixcore.query.model import Query +from fixcore.types import Json, EdgeType from fixcore.util import first ancestor_merges = { @@ -49,7 +51,7 @@ def owners(self, path: str) -> List[ComplexKind]: @define(repr=True, eq=True) -class GraphUpdate(ABC): +class GraphUpdate: nodes_created: int = 0 nodes_updated: int = 0 nodes_deleted: int = 0 @@ -76,3 +78,57 @@ def __add__(self, other: GraphUpdate) -> GraphUpdate: self.edges_updated + other.edges_updated, self.edges_deleted + other.edges_deleted, ) + + +@define +class GraphChange: + node_inserts: List[Json] = field(factory=list) + node_updates: List[Json] = field(factory=list) + node_deletes: List[Json] = field(factory=list) + edge_inserts: DefaultDict[EdgeType, List[Json]] = field(factory=lambda: defaultdict(list)) + edge_updates: DefaultDict[EdgeType, List[Json]] = field(factory=lambda: defaultdict(list)) + edge_deletes: DefaultDict[EdgeType, List[Json]] = field(factory=lambda: defaultdict(list)) + + def to_update(self) -> GraphUpdate: + return GraphUpdate( + len(self.node_inserts), + len(self.node_updates), + len(self.node_deletes), + sum(len(edges) for edges in self.edge_inserts.values()), + sum(len(edges) for edges in self.edge_updates.values()), + sum(len(edges) for edges in self.edge_deletes.values()), + ) + + def change_count(self) -> int: + return self.to_update().all_changes() + + def __add__(self, other: GraphChange) -> GraphChange: + update = GraphChange() + # insert + update.node_inserts.extend(self.node_inserts) + update.node_inserts.extend(other.node_inserts) + # update + update.node_updates.extend(self.node_updates) + update.node_updates.extend(other.node_updates) + # delete + update.node_deletes.extend(self.node_deletes) + update.node_deletes.extend(other.node_deletes) + for edge_type in EdgeTypes.all: + # insert + update.edge_inserts[edge_type].extend(self.edge_inserts[edge_type]) + update.edge_inserts[edge_type].extend(other.edge_inserts[edge_type]) + # update + update.edge_updates[edge_type].extend(self.edge_updates[edge_type]) + update.edge_updates[edge_type].extend(other.edge_updates[edge_type]) + # delete + update.edge_deletes[edge_type].extend(self.edge_deletes[edge_type]) + update.edge_deletes[edge_type].extend(other.edge_deletes[edge_type]) + return update + + def clear(self) -> None: + self.node_inserts.clear() + self.node_updates.clear() + self.node_deletes.clear() + self.edge_inserts.clear() + self.edge_updates.clear() + self.edge_deletes.clear() diff --git a/fixcore/fixcore/model/graph_access.py b/fixcore/fixcore/model/graph_access.py index 2276be78b0..ed126fbf6d 100644 --- a/fixcore/fixcore/model/graph_access.py +++ b/fixcore/fixcore/model/graph_access.py @@ -409,8 +409,9 @@ def check_complete(self) -> None: for edge_type in EdgeTypes.all: key = GraphAccess.edge_key(rid, succ, edge_type) if self.graph.has_edge(rid, succ, key): + data = self.graph.get_edge_data(rid, succ, key) self.graph.remove_edge(rid, succ, key) - self.add_edge("root", succ, edge_type) + self.add_edge("root", succ, edge_type, data.get("reported")) self.graph.remove_node(rid) diff --git a/fixcore/fixcore/web/__init__.py b/fixcore/fixcore/web/__init__.py index b626b486e7..315832b0dd 100644 --- a/fixcore/fixcore/web/__init__.py +++ b/fixcore/fixcore/web/__init__.py @@ -3,4 +3,3 @@ from aiohttp.web import Request, StreamResponse RequestHandler = Callable[[Request], Awaitable[StreamResponse]] -Middleware = Callable[[Request, RequestHandler], Awaitable[StreamResponse]] diff --git a/fixcore/fixcore/web/api.py b/fixcore/fixcore/web/api.py index 37ec1d676a..d331eb2dc3 100644 --- a/fixcore/fixcore/web/api.py +++ b/fixcore/fixcore/web/api.py @@ -32,6 +32,7 @@ ) from urllib.parse import urlencode, urlparse, parse_qs, urlunparse +import aiofiles import aiohttp_jinja2 import jinja2 import prometheus_client @@ -44,6 +45,7 @@ MultipartReader, ClientSession, TCPConnector, + BodyPartReader, ) from aiohttp.abc import AbstractStreamWriter from aiohttp.hdrs import METH_ANY @@ -1380,6 +1382,23 @@ def line_to_js(line: ParsedCommandLine) -> Json: @timed("api", "execute") async def execute(self, request: Request, deps: TenantDependencies) -> StreamResponse: temp_dir: Optional[str] = None + + async def write_files(mpr: MultipartReader, tmp_dir: str) -> Dict[str, str]: + files: Dict[str, str] = {} + async for part in mpr: + if isinstance(part, MultipartReader): + files.update(await write_files(part, tmp_dir)) + elif isinstance(part, BodyPartReader): + name = part.filename + if not name: + raise AttributeError("Multipart request: content disposition name is required!") + path = os.path.join(tmp_dir, rnd_str()) # use random local path to avoid clashes + files[name] = path + async with aiofiles.open(path, "wb") as writer: + while not part.at_eof(): + await writer.write(await part.read_chunk()) + return files + try: ctx = self.cli_context_from_request(request) if request.content_type.startswith("text"): @@ -1388,18 +1407,9 @@ async def execute(self, request: Request, deps: TenantDependencies) -> StreamRes command = request.headers["Fix-Shell-Command"].strip() temp = tempfile.mkdtemp() temp_dir = temp - files = {} # for now, we assume that all multi-parts are file uploads - async for part in MultipartReader(request.headers, request.content): - name = part.name - if not name: - raise AttributeError("Multipart request: content disposition name is required!") - path = os.path.join(temp, rnd_str()) # use random local path to avoid clashes - files[name] = path - with open(path, "wb") as writer: - while not part.at_eof(): - writer.write(await part.read_chunk()) - ctx = evolve(ctx, uploaded_files=files) + uploaded = await write_files(MultipartReader(request.headers, request.content), temp) + ctx = evolve(ctx, uploaded_files=uploaded) else: raise AttributeError(f"Not able to handle: {request.content_type}") diff --git a/fixcore/fixcore/web/auth.py b/fixcore/fixcore/web/auth.py index 5a19d0ee68..20738d9155 100644 --- a/fixcore/fixcore/web/auth.py +++ b/fixcore/fixcore/web/auth.py @@ -10,6 +10,7 @@ import jwt from aiohttp import web +from aiohttp.typedefs import Middleware from aiohttp.web import Request, StreamResponse from aiohttp.web import middleware from attr import define @@ -26,7 +27,7 @@ from fixcore.web.certificate_handler import CertificateHandler from fixcore.web.permission import PermissionChecker from fixlib import jwt as ck_jwt -from fixlib.asynchronous.web import RequestHandler, Middleware, TenantRequestHandler +from fixlib.asynchronous.web import RequestHandler, TenantRequestHandler from fixlib.jwt import encode_jwt, create_jwk_dict from fixlib.types import Json from fixlib.utils import utc diff --git a/fixcore/fixcore/web/directives.py b/fixcore/fixcore/web/directives.py index 4c824aa638..c1df258c07 100644 --- a/fixcore/fixcore/web/directives.py +++ b/fixcore/fixcore/web/directives.py @@ -1,8 +1,9 @@ import logging from re import RegexFlag, fullmatch -from typing import Optional, Callable, Awaitable, Tuple +from typing import Optional, Tuple from aiohttp.hdrs import METH_OPTIONS, METH_GET, METH_HEAD, METH_POST, METH_PUT, METH_DELETE, METH_PATCH +from aiohttp.typedefs import Middleware from aiohttp.web import HTTPRedirection, HTTPNotFound, HTTPBadRequest, HTTPException, HTTPNoContent from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPError from aiohttp.web_middlewares import middleware @@ -84,9 +85,7 @@ async def metrics_handler(request: Request, handler: RequestHandler) -> StreamRe RequestInProgress.labels(request.path, request.method).dec() -def error_handler( - config: CoreConfig, event_sender: AnalyticsEventSender -) -> Callable[[Request, RequestHandler], Awaitable[StreamResponse]]: +def error_handler(config: CoreConfig, event_sender: AnalyticsEventSender) -> Middleware: is_debug = (logging.root.level < logging.INFO) or config.runtime.debug def exc_info(ex: Exception) -> Optional[Exception]: @@ -129,7 +128,7 @@ def message_from_error(e: Exception) -> Tuple[str, str, str]: return error_handler_middleware -def default_middleware(api_handler: "api.Api") -> Callable[[Request, RequestHandler], Awaitable[StreamResponse]]: +def default_middleware(api_handler: "api.Api") -> Middleware: @middleware async def default_handler(request: Request, handler: RequestHandler) -> StreamResponse: if api_handler.in_shutdown: diff --git a/fixcore/tests/fixcore/conftest.py b/fixcore/tests/fixcore/conftest.py index c6abd831b6..5c72f2d5ca 100644 --- a/fixcore/tests/fixcore/conftest.py +++ b/fixcore/tests/fixcore/conftest.py @@ -198,8 +198,8 @@ def test_db(local_client: ArangoClient, system_db: StandardDatabase) -> Standard @fixture -async def graph_db(async_db: AsyncArangoDB) -> ArangoGraphDB: - graph_db = ArangoGraphDB(async_db, GraphName("ns"), NoAdjust(), GraphConfig(use_view=False)) +async def graph_db(async_db: AsyncArangoDB, lock_db: LockDB) -> ArangoGraphDB: + graph_db = ArangoGraphDB(async_db, GraphName("ns"), NoAdjust(), GraphConfig(use_view=False), lock_db) await graph_db.create_update_schema() await model_db(async_db, "ns_model").create_update_schema() await async_db.truncate(graph_db.in_progress) diff --git a/fixcore/tests/fixcore/web/api_test.py b/fixcore/tests/fixcore/web/api_test.py index ea0d9208b9..f0b92bd1c8 100644 --- a/fixcore/tests/fixcore/web/api_test.py +++ b/fixcore/tests/fixcore/web/api_test.py @@ -8,7 +8,7 @@ import pytest from _pytest.fixtures import fixture -from aiohttp import ClientSession, MultipartReader +from aiohttp import ClientSession, MultipartReader, BodyPartReader from networkx import MultiDiGraph from datetime import timedelta from fixclient import models as rc @@ -394,7 +394,7 @@ async def test_cli(core_client: FixInventoryClient) -> None: # execute multiple commands response = await core_client.cli_execute_raw("echo foo; echo bar; echo bla") reader: MultipartReader = MultipartReader.from_response(response.undrelying) # type: ignore - assert [await p.text() async for p in reader] == ['"foo"', '"bar"', '"bla"'] + assert [await p.text() async for p in reader if isinstance(p, BodyPartReader)] == ['"foo"', '"bar"', '"bla"'] # list all cli commands info = AccessJson(await core_client.cli_info()) diff --git a/fixlib/fixlib/asynchronous/web/__init__.py b/fixlib/fixlib/asynchronous/web/__init__.py index 687f9067bf..73ece313ec 100644 --- a/fixlib/fixlib/asynchronous/web/__init__.py +++ b/fixlib/fixlib/asynchronous/web/__init__.py @@ -5,4 +5,3 @@ RequestHandler = Callable[[Request], Awaitable[StreamResponse]] TenantRequestHandler = Callable[[Request, TenantDependencies], Awaitable[StreamResponse]] -Middleware = Callable[[Request, RequestHandler], Awaitable[StreamResponse]] diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 06ad46de04..efd85813ed 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1514,6 +1514,31 @@ class BaseManagedKubernetesClusterProvider(BaseResource): endpoint: Optional[str] = field(default=None, metadata={"description": "The kubernetes API endpoint"}) +@define(eq=False, slots=False) +class BaseAIResource(BaseResource): + kind: ClassVar[str] = "ai_resource" + kind_display: ClassVar[str] = "AI resource" + kind_description: ClassVar[str] = "An AI Resource." + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + _categories: ClassVar[List[Category]] = [Category.ai, Category.compute] + + +@define(eq=False, slots=False) +class BaseAIJob(BaseAIResource): + kind: ClassVar[str] = "ai_job" + kind_display: ClassVar[str] = "AI Job" + kind_description: ClassVar[str] = "An AI Job resource." + metadata: ClassVar[Dict[str, Any]] = {"icon": "job", "group": "ai"} + + +@define(eq=False, slots=False) +class BaseAIModel(BaseAIResource): + kind: ClassVar[str] = "ai_model" + kind_display: ClassVar[str] = "AI Model" + kind_description: ClassVar[str] = "An AI Model resource." + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + + @define(eq=False, slots=False) class UnknownCloud(BaseCloud): kind: ClassVar[str] = "unknown_cloud" diff --git a/plugins/aws/fix_plugin_aws/collector.py b/plugins/aws/fix_plugin_aws/collector.py index f4330eb659..5355f44a87 100644 --- a/plugins/aws/fix_plugin_aws/collector.py +++ b/plugins/aws/fix_plugin_aws/collector.py @@ -49,6 +49,7 @@ acm, waf, backup, + bedrock, ) from fix_plugin_aws.resource.base import AwsAccount, AwsApiSpec, AwsRegion, AwsResource, GraphBuilder from fixlib.baseresources import Cloud, EdgeType, BaseOrganizationalRoot, BaseOrganizationalUnit @@ -108,6 +109,7 @@ + redshift.resources + backup.resources + amazonq.resources + + bedrock.resources ) all_resources: List[Type[AwsResource]] = global_resources + regional_resources @@ -264,6 +266,8 @@ def get_last_run() -> Optional[datetime]: # wait for all futures to finish shared_queue.wait_for_submitted_work() + # remove unused nodes + self.remove_unused() self.core_feedback.progress_done(self.account.dname, 1, 1, context=[self.cloud.id]) self.error_accumulator.report_all(global_builder.core_feedback) @@ -335,6 +339,32 @@ def collect_and_set_metrics( builder.submit_work("cloudwatch", collect_and_set_metrics, start, region, queries) + def remove_unused(self) -> None: + remove_nodes = [] + + def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None, check_pred: bool = True) -> None: # type: ignore + for node in self.graph.nodes: + if not isinstance(node, cls): + continue + if check_pred: + nodes = list(self.graph.predecessors(node)) + else: + nodes = list(self.graph.successors(node)) + if ignore_kinds is not None: + nodes = [n for n in nodes if not isinstance(n, ignore_kinds)] + if not nodes: + remove_nodes.append(node) + log.debug(f"Removing {len(remove_nodes)} unreferenced nodes of type {cls}") + removed = set() + for node in remove_nodes: + if node in removed: + continue + removed.add(node) + self.graph.remove_node(node) + remove_nodes.clear() + + rm_nodes(bedrock.AwsBedrockFoundationModel, check_pred=False) + # TODO: move into separate AwsAccountSettings def update_account(self) -> None: log.info(f"Collecting AWS IAM Account Summary in account {self.account.dname}") diff --git a/plugins/aws/fix_plugin_aws/resource/amazonq.py b/plugins/aws/fix_plugin_aws/resource/amazonq.py index 0bd8321a8b..5f4c95d9ca 100644 --- a/plugins/aws/fix_plugin_aws/resource/amazonq.py +++ b/plugins/aws/fix_plugin_aws/resource/amazonq.py @@ -143,7 +143,7 @@ def add_tags(tag_resource: AwsResource) -> None: resourceARN=tag_resource.arn, ) if tags: - tag_resource.tags.update(tags) + tag_resource.tags.update(tags[0]) else: tags = builder.client.list( service_name, @@ -356,9 +356,9 @@ class AwsQBusinessDataSource(AmazonQTaggable, AwsResource): metadata: ClassVar[Dict[str, Any]] = {"icon": "bucket", "group": "ai"} # Collected via AwsQBusinessApplication() aws_metadata: ClassVar[Dict[str, Any]] = { - "provider_link_tpl": "https://{region_id}.console.aws.amazon.com/amazonq/business/applications/{app_id}/indices/{indice_id}/datasources/{id}/details?region={region}", # fmt: skip + "provider_link_tpl": "https://{region_id}.console.aws.amazon.com/amazonq/business/applications/{application_id}/indices/{indice_id}/datasources/{id}/details?region={region}", # fmt: skip "arn_tpl": "arn:{partition}:qbusiness:{region}:{account}:application/{application_id}/index/{indice_id}/data-source/{id}", - "extra_args": ["application_id", "indice_id"], + "extra_args_for_arn": ["application_id", "indice_id"], } mapping: ClassVar[Dict[str, Bender]] = { "id": S("dataSourceId"), @@ -528,7 +528,7 @@ class AwsQBusinessIndice(AmazonQTaggable, AwsResource): metadata: ClassVar[Dict[str, Any]] = {"icon": "config", "group": "ai"} aws_metadata: ClassVar[Dict[str, Any]] = { "arn_tpl": "arn:{partition}:qbusiness:{region}:{account}:application/{application_id}/index/{id}", - "extra_args": ["application_id"], + "extra_args_for_arn": ["application_id"], } # Collected via AwsQBusinessApplication() mapping: ClassVar[Dict[str, Bender]] = { @@ -753,7 +753,7 @@ class AwsQBusinessPlugin(AmazonQTaggable, AwsResource): metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} aws_metadata: ClassVar[Dict[str, Any]] = { "arn_tpl": "arn:{partition}:qbusiness:{region}:{account}:application/{application_id}/plugin/{id}", - "extra_args": ["application_id"], + "extra_args_for_arn": ["application_id"], } # Collected via AwsQBusinessApplication() mapping: ClassVar[Dict[str, Bender]] = { @@ -819,7 +819,7 @@ class AwsQBusinessRetriever(AmazonQTaggable, AwsResource): metadata: ClassVar[Dict[str, Any]] = {"icon": "application", "group": "ai"} aws_metadata: ClassVar[Dict[str, Any]] = { "arn_tpl": "arn:{partition}:qbusiness:{region}:{account}:application/{application_id}/retriever/{id}", - "extra_args": ["application_id"], + "extra_args_for_arn": ["application_id"], } # Collected via AwsQBusinessApplication() mapping: ClassVar[Dict[str, Bender]] = { @@ -876,7 +876,7 @@ class AwsQBusinessWebExperience(AmazonQTaggable, AwsResource): metadata: ClassVar[Dict[str, Any]] = {"icon": "application", "group": "ai"} aws_metadata: ClassVar[Dict[str, Any]] = { "arn_tpl": "arn:{partition}:qbusiness:{region}:{account}:application/{application_id}/web-experience/{id}", - "extra_args": ["application_id"], + "extra_args_for_arn": ["application_id"], } # Collected via AwsQBusinessApplication() mapping: ClassVar[Dict[str, Bender]] = { diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py index 793dda32f9..5417191853 100644 --- a/plugins/aws/fix_plugin_aws/resource/base.py +++ b/plugins/aws/fix_plugin_aws/resource/base.py @@ -530,7 +530,7 @@ def add_node( } # Add any additional dynamic arguments from the metadata (if they exist) - if extra_args := meta.get("extra_args"): + if extra_args := meta.get("extra_args_for_arn"): for extra_arg in extra_args: args[extra_arg] = getattr(node, extra_arg) diff --git a/plugins/aws/fix_plugin_aws/resource/bedrock.py b/plugins/aws/fix_plugin_aws/resource/bedrock.py new file mode 100644 index 0000000000..460917a133 --- /dev/null +++ b/plugins/aws/fix_plugin_aws/resource/bedrock.py @@ -0,0 +1,2048 @@ +import logging +from datetime import datetime +from typing import ClassVar, Dict, Optional, List, Type, Any + +from attrs import define, field + +from fix_plugin_aws.aws_client import AwsClient +from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder +from fix_plugin_aws.resource.ec2 import AwsEc2Subnet, AwsEc2SecurityGroup +from fix_plugin_aws.resource.iam import AwsIamRole +from fix_plugin_aws.resource.kms import AwsKmsKey +from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction +from fix_plugin_aws.resource.s3 import AwsS3Bucket +from fix_plugin_aws.resource.rds import AwsRdsCluster, AwsRdsInstance +from fixlib.baseresources import BaseAIJob, ModelReference, BaseAIModel +from fixlib.graph import Graph +from fixlib.json_bender import Bender, S, ForallBend, Bend +from fixlib.types import Json + +log = logging.getLogger("fix.plugins.aws") +service_name = "bedrock" + + +class BedrockTaggable: + def update_resource_tag(self, client: AwsClient, key: str, value: str) -> bool: + if isinstance(self, AwsResource): + if self.service_name() == "bedrock": + client.call( + aws_service=self.service_name(), + action="tag-resource", + result_name=None, + resourceARN=self.arn, + tags=[{"key": key, "value": value}], + ) + else: + client.call( + aws_service=self.service_name(), + action="tag-resource", + result_name=None, + resourceArn=self.arn, + tags={key: value}, + ) + return True + return False + + def delete_resource_tag(self, client: AwsClient, key: str) -> bool: + if isinstance(self, AwsResource): + if self.service_name() == "bedrock": + client.call( + aws_service=self.service_name(), + action="untag-resource", + result_name=None, + resourceARN=self.arn, + tagKeys=[key], + ) + else: + client.call( + aws_service=self.service_name(), + action="untag-resource", + result_name=None, + resourceArn=self.arn, + tagKeys=[key], + ) + return True + return False + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return [ + AwsApiSpec(cls.service_name(), "list-tags-for-resource"), + ] + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return [ + AwsApiSpec(cls.service_name(), "tag-resource"), + AwsApiSpec(cls.service_name(), "untag-resource"), + ] + + @classmethod + def service_name(cls) -> str: + return service_name + + +@define(eq=False, slots=False) +class AwsBedrockFoundationModel(BaseAIModel, AwsResource): + kind: ClassVar[str] = "aws_bedrock_foundation_model" + kind_display: ClassVar[str] = "AWS Bedrock Foundation Model" + kind_description: ClassVar[str] = ( + "AWS Bedrock Foundation Model represents the base machine learning models provided by AWS Bedrock. " + "These models are pre-trained and can be used as a starting point for various machine learning tasks." + ) + kind_service: ClassVar[Optional[str]] = service_name + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/providers?model={id}"} # fmt: skip + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("bedrock", "list-foundation-models", "modelSummaries") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("modelId"), + "name": S("modelName"), + "arn": S("modelArn"), + "model_arn": S("modelArn"), + "model_id": S("modelId"), + "model_name": S("modelName"), + "model_provider_name": S("providerName"), + "input_modalities": S("inputModalities", default=[]), + "output_modalities": S("outputModalities", default=[]), + "response_streaming_supported": S("responseStreamingSupported"), + "customizations_supported": S("customizationsSupported", default=[]), + "inference_types_supported": S("inferenceTypesSupported", default=[]), + "model_lifecycle_status": S("modelLifecycle", "status"), + } + model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the foundation model."}) # fmt: skip + model_id: Optional[str] = field(default=None, metadata={"description": "The model ID of the foundation model."}) # fmt: skip + model_name: Optional[str] = field(default=None, metadata={"description": "The name of the model."}) # fmt: skip + model_provider_name: Optional[str] = field(default=None, metadata={"description": "The model's provider name."}) # fmt: skip + input_modalities: Optional[List[str]] = field(factory=list, metadata={"description": "The input modalities that the model supports."}) # fmt: skip + output_modalities: Optional[List[str]] = field(factory=list, metadata={"description": "The output modalities that the model supports."}) # fmt: skip + response_streaming_supported: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the model supports streaming."}) # fmt: skip + customizations_supported: Optional[List[str]] = field(factory=list, metadata={"description": "Whether the model supports fine-tuning or continual pre-training."}) # fmt: skip + inference_types_supported: Optional[List[str]] = field(factory=list, metadata={"description": "The inference types that the model supports."}) # fmt: skip + model_lifecycle_status: Optional[str] = field(default=None, metadata={"description": "Contains details about whether a model version is available or deprecated."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockValidationDataConfig: + kind: ClassVar[str] = "aws_bedrock_validation_data_config" + mapping: ClassVar[Dict[str, Bender]] = {"validators": S("validators", default=[]) >> ForallBend(S("s3Uri"))} + validators: Optional[List[str]] = field(factory=list, metadata={"description": "Information about the validators."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockCustomModel(BedrockTaggable, BaseAIModel, AwsResource): + kind: ClassVar[str] = "aws_bedrock_custom_model" + kind_display: ClassVar[str] = "AWS Bedrock Custom Model" + kind_description: ClassVar[str] = ( + "AWS Bedrock Custom Model enables users to create and train machine learning models " + "tailored to specific needs, providing flexibility in model customization for a variety of applications." + ) + kind_service: ClassVar[Optional[str]] = service_name + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/custom-models/{name}"} # fmt: skip + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + reference_kinds: ClassVar[ModelReference] = { + "successors": {"default": ["aws_bedrock_model_customization_job", AwsKmsKey.kind]}, + "predecessors": {"default": [AwsBedrockFoundationModel.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("bedrock", "list-custom-models", "modelSummaries") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("modelArn"), + "name": S("modelName"), + "ctime": S("creationTime"), + "arn": S("modelArn"), + "model_arn": S("modelArn"), + "model_name": S("modelName"), + "job_name": S("jobName"), + "job_arn": S("jobArn"), + "base_model_arn": S("baseModelArn"), + "customization_type": S("customizationType"), + "model_kms_key_arn": S("modelKmsKeyArn"), + "hyper_parameters": S("hyperParameters"), + "training_data_config": S("trainingDataConfig", "s3Uri"), + "validation_data_config": S("validationDataConfig") >> Bend(AwsBedrockValidationDataConfig.mapping), + "output_data_config": S("outputDataConfig", "s3Uri"), + "training_metrics": S("trainingMetrics", "trainingLoss"), + "validation_metrics": S("validationMetrics", default=[]) >> ForallBend(S("validationLoss")), + "creation_time": S("creationTime"), + } + model_arn: Optional[str] = field(default=None, metadata={"description": "Amazon Resource Name (ARN) associated with this model."}) # fmt: skip + model_name: Optional[str] = field(default=None, metadata={"description": "Model name associated with this model."}) # fmt: skip + job_name: Optional[str] = field(default=None, metadata={"description": "Job name associated with this model."}) # fmt: skip + job_arn: Optional[str] = field(default=None, metadata={"description": "Job Amazon Resource Name (ARN) associated with this model."}) # fmt: skip + base_model_arn: Optional[str] = field(default=None, metadata={"description": "Amazon Resource Name (ARN) of the base model."}) # fmt: skip + customization_type: Optional[str] = field(default=None, metadata={"description": "The type of model customization."}) # fmt: skip + model_kms_key_arn: Optional[str] = field(default=None, metadata={"description": "The custom model is encrypted at rest using this key."}) # fmt: skip + hyper_parameters: Optional[Dict[str, str]] = field(default=None, metadata={"description": "Hyperparameter values associated with this model. For details on the format for different models, see Custom model hyperparameters."}) # fmt: skip + training_data_config: Optional[str] = field(default=None, metadata={"description": "Contains information about the training dataset."}) # fmt: skip + validation_data_config: Optional[AwsBedrockValidationDataConfig] = field(default=None, metadata={"description": "Contains information about the validation dataset."}) # fmt: skip + output_data_config: Optional[str] = field(default=None, metadata={"description": "Output data configuration associated with this custom model."}) # fmt: skip + training_metrics: Optional[float] = field(default=None, metadata={"description": "Contains training metrics from the job creation."}) # fmt: skip + validation_metrics: Optional[List[float]] = field(factory=list, metadata={"description": "The validation metrics from the job creation."}) # fmt: skip + creation_time: Optional[datetime] = field(default=None, metadata={"description": "Creation time of the model."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if job_arn := self.job_arn: + builder.add_edge(self, clazz=AwsBedrockModelCustomizationJob, id=job_arn) + if base_model_arn := self.base_model_arn: + builder.add_edge(self, reverse=True, clazz=AwsBedrockFoundationModel, arn=base_model_arn) + if model_kms_key_arn := self.model_kms_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=model_kms_key_arn) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service=service_name, + action="delete-custom-model", + result_name=None, + modelIdentifier=self.name, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec(service_name, "delete-custom-model")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec(service_name, "get-custom-model")] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(job: AwsResource) -> None: + tags = builder.client.list( + service_name, + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceARN=job.arn, + ) + if tags: + for tag in tags: + job.tags.update({tag.get("key"): tag.get("value")}) + + for js in json: + for result in builder.client.list( + service_name, + "get-custom-model", + modelIdentifier=js["modelArn"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, add_tags, instance) + + +@define(eq=False, slots=False) +class AwsBedrockProvisionedModelThroughput(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_provisioned_model_throughput" + kind_display: ClassVar[str] = "AWS Bedrock Provisioned Model Throughput" + kind_description: ClassVar[str] = ( + "AWS Bedrock Provisioned Model Throughput manages the throughput capacity for machine learning models, " + "ensuring consistent performance and scalability to handle varying workloads." + ) + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/provisioned-throughput/{name}"} # fmt: skip + kind_service: ClassVar[Optional[str]] = service_name + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsBedrockCustomModel.kind, AwsBedrockFoundationModel.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock", "list-provisioned-model-throughputs", "provisionedModelSummaries" + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("provisionedModelArn"), + "name": S("provisionedModelName"), + "ctime": S("creationTime"), + "mtime": S("lastModifiedTime"), + "provisioned_model_name": S("provisionedModelName"), + "provisioned_model_arn": S("provisionedModelArn"), + "model_arn": S("modelArn"), + "desired_model_arn": S("desiredModelArn"), + "foundation_model_arn": S("foundationModelArn"), + "model_units": S("modelUnits"), + "desired_model_units": S("desiredModelUnits"), + "status": S("status"), + "commitment_duration": S("commitmentDuration"), + "commitment_expiration_time": S("commitmentExpirationTime"), + "creation_time": S("creationTime"), + "last_modified_time": S("lastModifiedTime"), + } + provisioned_model_name: Optional[str] = field(default=None, metadata={"description": "The name of the Provisioned Throughput."}) # fmt: skip + provisioned_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the Provisioned Throughput."}) # fmt: skip + model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the model associated with the Provisioned Throughput."}) # fmt: skip + desired_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the model requested to be associated to this Provisioned Throughput. This value differs from the modelArn if updating hasn't completed."}) # fmt: skip + foundation_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the base model for which the Provisioned Throughput was created, or of the base model that the custom model for which the Provisioned Throughput was created was customized."}) # fmt: skip + model_units: Optional[int] = field(default=None, metadata={"description": "The number of model units allocated to the Provisioned Throughput."}) # fmt: skip + desired_model_units: Optional[int] = field(default=None, metadata={"description": "The number of model units that was requested to be allocated to the Provisioned Throughput."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the Provisioned Throughput."}) # fmt: skip + commitment_duration: Optional[str] = field(default=None, metadata={"description": "The duration for which the Provisioned Throughput was committed."}) # fmt: skip + commitment_expiration_time: Optional[datetime] = field(default=None, metadata={"description": "The timestamp for when the commitment term of the Provisioned Throughput expires."}) # fmt: skip + creation_time: Optional[datetime] = field(default=None, metadata={"description": "The time that the Provisioned Throughput was created."}) # fmt: skip + last_modified_time: Optional[datetime] = field(default=None, metadata={"description": "The time that the Provisioned Throughput was last modified."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if model_arn := self.model_arn: + builder.add_edge(self, reverse=True, clazz=AwsBedrockCustomModel, id=model_arn) + if foundation_model_arn := self.foundation_model_arn: + builder.add_edge(self, reverse=True, clazz=AwsBedrockFoundationModel, arn=foundation_model_arn) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service=service_name, + action="delete-provisioned-model-throughput", + result_name=None, + provisionedModelId=self.safe_name, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec(service_name, "delete-provisioned-model-throughput")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec] + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailTopic: + kind: ClassVar[str] = "aws_bedrock_guardrail_topic" + mapping: ClassVar[Dict[str, Bender]] = { + "name": S("name"), + "definition": S("definition"), + "examples": S("examples", default=[]), + "type": S("type"), + } + name: Optional[str] = field(default=None, metadata={"description": "The name of the topic to deny."}) # fmt: skip + definition: Optional[str] = field(default=None, metadata={"description": "A definition of the topic to deny."}) # fmt: skip + examples: Optional[List[str]] = field(factory=list, metadata={"description": "A list of prompts, each of which is an example of a prompt that can be categorized as belonging to the topic."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "Specifies to deny the topic."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailTopicPolicy: + kind: ClassVar[str] = "aws_bedrock_guardrail_topic_policy" + mapping: ClassVar[Dict[str, Bender]] = { + "topics": S("topics", default=[]) >> ForallBend(AwsBedrockGuardrailTopic.mapping) + } + topics: Optional[List[AwsBedrockGuardrailTopic]] = field(factory=list, metadata={"description": "A list of policies related to topics that the guardrail should deny."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailContentFilter: + kind: ClassVar[str] = "aws_bedrock_guardrail_content_filter" + mapping: ClassVar[Dict[str, Bender]] = { + "type": S("type"), + "input_strength": S("inputStrength"), + "output_strength": S("outputStrength"), + } + type: Optional[str] = field(default=None, metadata={"description": "The harmful category that the content filter is applied to."}) # fmt: skip + input_strength: Optional[str] = field(default=None, metadata={"description": "The strength of the content filter to apply to prompts. As you increase the filter strength, the likelihood of filtering harmful content increases and the probability of seeing harmful content in your application reduces."}) # fmt: skip + output_strength: Optional[str] = field(default=None, metadata={"description": "The strength of the content filter to apply to model responses. As you increase the filter strength, the likelihood of filtering harmful content increases and the probability of seeing harmful content in your application reduces."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailContentPolicy: + kind: ClassVar[str] = "aws_bedrock_guardrail_content_policy" + mapping: ClassVar[Dict[str, Bender]] = { + "filters": S("filters", default=[]) >> ForallBend(AwsBedrockGuardrailContentFilter.mapping) + } + filters: Optional[List[AwsBedrockGuardrailContentFilter]] = field(factory=list, metadata={"description": "Contains the type of the content filter and how strongly it should apply to prompts and model responses."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailWordPolicy: + kind: ClassVar[str] = "aws_bedrock_guardrail_word_policy" + mapping: ClassVar[Dict[str, Bender]] = { + "words": S("words", default=[]) >> ForallBend(S("text")), + "managed_word_lists": S("managedWordLists", default=[]) >> ForallBend(S("type")), + } + words: Optional[List[str]] = field(factory=list, metadata={"description": "A list of words configured for the guardrail."}) # fmt: skip + managed_word_lists: Optional[List[str]] = field(factory=list, metadata={"description": "A list of managed words configured for the guardrail."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailPiiEntity: + kind: ClassVar[str] = "aws_bedrock_guardrail_pii_entity" + mapping: ClassVar[Dict[str, Bender]] = {"type": S("type"), "action": S("action")} + type: Optional[str] = field(default=None, metadata={"description": "The type of PII entity. For example, Social Security Number."}) # fmt: skip + action: Optional[str] = field(default=None, metadata={"description": "The configured guardrail action when PII entity is detected."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailRegex: + kind: ClassVar[str] = "aws_bedrock_guardrail_regex" + mapping: ClassVar[Dict[str, Bender]] = { + "name": S("name"), + "description": S("description"), + "pattern": S("pattern"), + "action": S("action"), + } + name: Optional[str] = field(default=None, metadata={"description": "The name of the regular expression for the guardrail."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the regular expression for the guardrail."}) # fmt: skip + pattern: Optional[str] = field(default=None, metadata={"description": "The pattern of the regular expression configured for the guardrail."}) # fmt: skip + action: Optional[str] = field(default=None, metadata={"description": "The action taken when a match to the regular expression is detected."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailSensitiveInformationPolicy: + kind: ClassVar[str] = "aws_bedrock_guardrail_sensitive_information_policy" + mapping: ClassVar[Dict[str, Bender]] = { + "pii_entities": S("piiEntities", default=[]) >> ForallBend(AwsBedrockGuardrailPiiEntity.mapping), + "regexes": S("regexes", default=[]) >> ForallBend(AwsBedrockGuardrailRegex.mapping), + } + pii_entities: Optional[List[AwsBedrockGuardrailPiiEntity]] = field(factory=list, metadata={"description": "The list of PII entities configured for the guardrail."}) # fmt: skip + regexes: Optional[List[AwsBedrockGuardrailRegex]] = field(factory=list, metadata={"description": "The list of regular expressions configured for the guardrail."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailContextualGroundingFilter: + kind: ClassVar[str] = "aws_bedrock_guardrail_contextual_grounding_filter" + mapping: ClassVar[Dict[str, Bender]] = {"type": S("type"), "threshold": S("threshold")} + type: Optional[str] = field(default=None, metadata={"description": "The filter type details for the guardrails contextual grounding filter."}) # fmt: skip + threshold: Optional[float] = field(default=None, metadata={"description": "The threshold details for the guardrails contextual grounding filter."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailContextualGroundingPolicy: + kind: ClassVar[str] = "aws_bedrock_guardrail_contextual_grounding_policy" + mapping: ClassVar[Dict[str, Bender]] = { + "filters": S("filters", default=[]) >> ForallBend(AwsBedrockGuardrailContextualGroundingFilter.mapping) + } + filters: Optional[List[AwsBedrockGuardrailContextualGroundingFilter]] = field(factory=list, metadata={"description": "The filter details for the guardrails contextual grounding policy."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockGuardrail(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_guardrail" + kind_display: ClassVar[str] = "AWS Bedrock Guardrail" + kind_description: ClassVar[str] = ( + "AWS Bedrock Guardrail provides safety mechanisms and policies to ensure responsible use of machine learning models, " + "helping to enforce compliance, security, and ethical standards." + ) + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock", "list-guardrails", "guardrails", expected_errors=["AccessDeniedException"] + ) + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/guardrails/guardrail/{name}/{id}"} # fmt: skip + metadata: ClassVar[Dict[str, Any]] = {"icon": "config", "group": "ai"} + kind_service: ClassVar[Optional[str]] = service_name + reference_kinds: ClassVar[ModelReference] = { + "successors": {"default": [AwsKmsKey.kind]}, + } + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("guardrailId"), + "name": S("name"), + "ctime": S("createdAt"), + "mtime": S("updatedAt"), + "arn": S("guardrailArn"), + "description": S("description"), + "guardrail_id": S("guardrailId"), + "guardrail_arn": S("guardrailArn"), + "version": S("version"), + "status": S("status"), + "topic_policy": S("topicPolicy") >> Bend(AwsBedrockGuardrailTopicPolicy.mapping), + "content_policy": S("contentPolicy") >> Bend(AwsBedrockGuardrailContentPolicy.mapping), + "word_policy": S("wordPolicy") >> Bend(AwsBedrockGuardrailWordPolicy.mapping), + "sensitive_information_policy": S("sensitiveInformationPolicy") + >> Bend(AwsBedrockGuardrailSensitiveInformationPolicy.mapping), + "contextual_grounding_policy": S("contextualGroundingPolicy") + >> Bend(AwsBedrockGuardrailContextualGroundingPolicy.mapping), + "created_at": S("createdAt"), + "updated_at": S("updatedAt"), + "status_reasons": S("statusReasons", default=[]), + "failure_recommendations": S("failureRecommendations", default=[]), + "blocked_input_messaging": S("blockedInputMessaging"), + "blocked_outputs_messaging": S("blockedOutputsMessaging"), + "kms_key_arn": S("kmsKeyArn"), + } + description: Optional[str] = field(default=None, metadata={"description": "The description of the guardrail."}) # fmt: skip + guardrail_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the guardrail."}) # fmt: skip + guardrail_arn: Optional[str] = field(default=None, metadata={"description": "The ARN of the guardrail."}) # fmt: skip + version: Optional[str] = field(default=None, metadata={"description": "The version of the guardrail."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the guardrail."}) # fmt: skip + topic_policy: Optional[AwsBedrockGuardrailTopicPolicy] = field(default=None, metadata={"description": "The topic policy that was configured for the guardrail."}) # fmt: skip + content_policy: Optional[AwsBedrockGuardrailContentPolicy] = field(default=None, metadata={"description": "The content policy that was configured for the guardrail."}) # fmt: skip + word_policy: Optional[AwsBedrockGuardrailWordPolicy] = field(default=None, metadata={"description": "The word policy that was configured for the guardrail."}) # fmt: skip + sensitive_information_policy: Optional[AwsBedrockGuardrailSensitiveInformationPolicy] = field(default=None, metadata={"description": "The sensitive information policy that was configured for the guardrail."}) # fmt: skip + contextual_grounding_policy: Optional[AwsBedrockGuardrailContextualGroundingPolicy] = field(default=None, metadata={"description": "The contextual grounding policy used in the guardrail."}) # fmt: skip + created_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time at which the guardrail was created."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time at which the guardrail was updated."}) # fmt: skip + status_reasons: Optional[List[str]] = field(factory=list, metadata={"description": "Appears if the status is FAILED. A list of reasons for why the guardrail failed to be created, updated, versioned, or deleted."}) # fmt: skip + failure_recommendations: Optional[List[str]] = field(factory=list, metadata={"description": "Appears if the status of the guardrail is FAILED. A list of recommendations to carry out before retrying the request."}) # fmt: skip + blocked_input_messaging: Optional[str] = field(default=None, metadata={"description": "The message that the guardrail returns when it blocks a prompt."}) # fmt: skip + blocked_outputs_messaging: Optional[str] = field(default=None, metadata={"description": "The message that the guardrail returns when it blocks a model response."}) # fmt: skip + kms_key_arn: Optional[str] = field(default=None, metadata={"description": "The ARN of the KMS key that encrypts the guardrail."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if kms_key_arn := self.kms_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=kms_key_arn) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service=service_name, + action="delete-guardrail", + result_name=None, + guardrailIdentifier=self.id or self.arn, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec(service_name, "delete-guardrail")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec(service_name, "get-guardrail")] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(job: AwsResource) -> None: + tags = builder.client.list( + service_name, + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceARN=job.arn, + ) + if tags: + for tag in tags: + job.tags.update({tag.get("key"): tag.get("value")}) + + for js in json: + for result in builder.client.list( + service_name, + "get-guardrail", + guardrailIdentifier=js["id"], + guardrailVersion=js["version"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, add_tags, instance) + + +@define(eq=False, slots=False) +class AwsBedrockVpcConfig: + kind: ClassVar[str] = "aws_bedrock_vpc_config" + mapping: ClassVar[Dict[str, Bender]] = { + "subnet_ids": S("subnetIds", default=[]), + "security_group_ids": S("securityGroupIds", default=[]), + } + subnet_ids: Optional[List[str]] = field(factory=list, metadata={"description": "VPC configuration subnets."}) # fmt: skip + security_group_ids: Optional[List[str]] = field(factory=list, metadata={"description": "VPC configuration security group Ids."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockModelCustomizationJob(BedrockTaggable, BaseAIJob, AwsResource): + kind: ClassVar[str] = "aws_bedrock_model_customization_job" + kind_display: ClassVar[str] = "AWS Bedrock Model Customization Job" + kind_description: ClassVar[str] = ( + "AWS Bedrock Model Customization Job is responsible for executing the processes involved in tailoring a machine learning model " + "to specific datasets and tasks, optimizing the model for unique use cases." + ) + kind_service: ClassVar[Optional[str]] = service_name + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/custom-models/item/?arn={arn}"} # fmt: skip + metadata: ClassVar[Dict[str, Any]] = {"icon": "job", "group": "ai"} + reference_kinds: ClassVar[ModelReference] = { + "predecessors": { + "default": [AwsEc2Subnet.kind, AwsEc2SecurityGroup.kind, AwsIamRole.kind, AwsBedrockFoundationModel.kind] + }, + "successors": {"default": [AwsKmsKey.kind, AwsS3Bucket.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock", + "list-model-customization-jobs", + "modelCustomizationJobSummaries", + expected_errors=["ValidationException"], + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("jobArn"), + "name": S("jobName"), + "arn": S("jobArn"), + "ctime": S("creationTime"), + "mtime": S("lastModifiedTime"), + "job_arn": S("jobArn"), + "job_name": S("jobName"), + "output_model_name": S("outputModelName"), + "output_model_arn": S("outputModelArn"), + "client_request_token": S("clientRequestToken"), + "role_arn": S("roleArn"), + "status": S("status"), + "failure_message": S("failureMessage"), + "creation_time": S("creationTime"), + "last_modified_time": S("lastModifiedTime"), + "end_time": S("endTime"), + "base_model_arn": S("baseModelArn"), + "hyper_parameters": S("hyperParameters"), + "training_data_config": S("trainingDataConfig", "s3Uri"), + "validation_data_config": S("validationDataConfig") >> Bend(AwsBedrockValidationDataConfig.mapping), + "output_data_config": S("outputDataConfig", "s3Uri"), + "customization_type": S("customizationType"), + "output_model_kms_key_arn": S("outputModelKmsKeyArn"), + "training_metrics": S("trainingMetrics", "trainingLoss"), + "validation_metrics": S("validationMetrics", default=[]) >> ForallBend(S("validationLoss")), + "vpc_config": S("vpcConfig") >> Bend(AwsBedrockVpcConfig.mapping), + } + job_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the customization job."}) # fmt: skip + job_name: Optional[str] = field(default=None, metadata={"description": "The name of the customization job."}) # fmt: skip + output_model_name: Optional[str] = field(default=None, metadata={"description": "The name of the output model."}) # fmt: skip + output_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the output model."}) # fmt: skip + client_request_token: Optional[str] = field(default=None, metadata={"description": "The token that you specified in the CreateCustomizationJob request."}) # fmt: skip + role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM role."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the job. A successful job transitions from in-progress to completed when the output model is ready to use. If the job failed, the failure message contains information about why the job failed."}) # fmt: skip + failure_message: Optional[str] = field(default=None, metadata={"description": "Information about why the job failed."}) # fmt: skip + creation_time: Optional[datetime] = field(default=None, metadata={"description": "Time that the resource was created."}) # fmt: skip + last_modified_time: Optional[datetime] = field(default=None, metadata={"description": "Time that the resource was last modified."}) # fmt: skip + end_time: Optional[datetime] = field(default=None, metadata={"description": "Time that the resource transitioned to terminal state."}) # fmt: skip + base_model_arn: Optional[str] = field(default=None, metadata={"description": "Amazon Resource Name (ARN) of the base model."}) # fmt: skip + hyper_parameters: Optional[Dict[str, str]] = field(default=None, metadata={"description": "The hyperparameter values for the job. For details on the format for different models, see Custom model hyperparameters."}) # fmt: skip + training_data_config: Optional[str] = field(default=None, metadata={"description": "Contains information about the training dataset."}) # fmt: skip + validation_data_config: Optional[AwsBedrockValidationDataConfig] = field(default=None, metadata={"description": "Contains information about the validation dataset."}) # fmt: skip + output_data_config: Optional[str] = field(default=None, metadata={"description": "Output data configuration"}) # fmt: skip + customization_type: Optional[str] = field(default=None, metadata={"description": "The type of model customization."}) # fmt: skip + output_model_kms_key_arn: Optional[str] = field(default=None, metadata={"description": "The custom model is encrypted at rest using this key."}) # fmt: skip + training_metrics: Optional[float] = field(default=None, metadata={"description": "Contains training metrics from the job creation."}) # fmt: skip + validation_metrics: Optional[List[float]] = field(factory=list, metadata={"description": "The loss metric for each validator that you provided in the createjob request."}) # fmt: skip + vpc_config: Optional[AwsBedrockVpcConfig] = field(default=None, metadata={"description": "VPC configuration for the custom model job."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if base_model_arn := self.base_model_arn: + builder.add_edge(self, reverse=True, clazz=AwsBedrockFoundationModel, arn=base_model_arn) + if model_kms_key_arn := self.output_model_kms_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=model_kms_key_arn) + if output_data_config := self.output_data_config: + bucket_name = AwsS3Bucket.name_from_path(output_data_config) + builder.add_edge(self, clazz=AwsS3Bucket, name=bucket_name) + if training_data_config := self.training_data_config: + bucket_name = AwsS3Bucket.name_from_path(training_data_config) + builder.add_edge(self, clazz=AwsS3Bucket, name=bucket_name) + if config := self.vpc_config: + if subnet_ids := config.subnet_ids: + for subnet_id in subnet_ids: + builder.add_edge(self, reverse=True, clazz=AwsEc2Subnet, id=subnet_id) + if security_group_ids := config.security_group_ids: + for security_group_id in security_group_ids: + builder.add_edge(self, reverse=True, clazz=AwsEc2SecurityGroup, id=security_group_id) + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec(service_name, "get-model-customization-job")] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(job: AwsResource) -> None: + tags = builder.client.list( + service_name, + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceARN=job.arn, + ) + if tags: + for tag in tags: + job.tags.update({tag.get("key"): tag.get("value")}) + + for js in json: + for result in builder.client.list( + service_name, + "get-model-customization-job", + jobIdentifier=js["jobArn"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, add_tags, instance) + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationDataset: + kind: ClassVar[str] = "aws_bedrock_evaluation_dataset" + mapping: ClassVar[Dict[str, Bender]] = {"name": S("name"), "dataset_location": S("datasetLocation", "s3Uri")} + name: Optional[str] = field(default=None, metadata={"description": "Used to specify supported built-in prompt datasets. Valid values are Builtin.Bold, Builtin.BoolQ, Builtin.NaturalQuestions, Builtin.Gigaword, Builtin.RealToxicityPrompts, Builtin.TriviaQa, Builtin.T-Rex, Builtin.WomensEcommerceClothingReviews and Builtin.Wikitext2."}) # fmt: skip + dataset_location: Optional[str] = field(default=None, metadata={"description": "For custom prompt datasets, you must specify the location in Amazon S3 where the prompt dataset is saved."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationDatasetMetricConfig: + kind: ClassVar[str] = "aws_bedrock_evaluation_dataset_metric_config" + mapping: ClassVar[Dict[str, Bender]] = { + "task_type": S("taskType"), + "dataset": S("dataset") >> Bend(AwsBedrockEvaluationDataset.mapping), + "metric_names": S("metricNames", default=[]), + } + task_type: Optional[str] = field(default=None, metadata={"description": "The task type you want the model to carry out."}) # fmt: skip + dataset: Optional[AwsBedrockEvaluationDataset] = field(default=None, metadata={"description": "Specifies the prompt dataset."}) # fmt: skip + metric_names: Optional[List[str]] = field(factory=list, metadata={"description": "The names of the metrics used."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockAutomatedEvaluationConfig: + kind: ClassVar[str] = "aws_bedrock_automated_evaluation_config" + mapping: ClassVar[Dict[str, Bender]] = { + "dataset_metric_configs": S("datasetMetricConfigs", default=[]) + >> ForallBend(AwsBedrockEvaluationDatasetMetricConfig.mapping) + } + dataset_metric_configs: Optional[List[AwsBedrockEvaluationDatasetMetricConfig]] = field(factory=list, metadata={"description": "Specifies the required elements for an automatic model evaluation job."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockHumanWorkflowConfig: + kind: ClassVar[str] = "aws_bedrock_human_workflow_config" + mapping: ClassVar[Dict[str, Bender]] = { + "flow_definition_arn": S("flowDefinitionArn"), + "instructions": S("instructions"), + } + flow_definition_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Number (ARN) for the flow definition"}) # fmt: skip + instructions: Optional[str] = field(default=None, metadata={"description": "Instructions for the flow definition"}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockHumanEvaluationCustomMetric: + kind: ClassVar[str] = "aws_bedrock_human_evaluation_custom_metric" + mapping: ClassVar[Dict[str, Bender]] = { + "name": S("name"), + "description": S("description"), + "rating_method": S("ratingMethod"), + } + name: Optional[str] = field(default=None, metadata={"description": "The name of the metric. Your human evaluators will see this name in the evaluation UI."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "An optional description of the metric. Use this parameter to provide more details about the metric."}) # fmt: skip + rating_method: Optional[str] = field(default=None, metadata={"description": "Choose how you want your human workers to evaluation your model. Valid values for rating methods are ThumbsUpDown, IndividualLikertScale,ComparisonLikertScale, ComparisonChoice, and ComparisonRank"}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockHumanEvaluationConfig: + kind: ClassVar[str] = "aws_bedrock_human_evaluation_config" + mapping: ClassVar[Dict[str, Bender]] = { + "human_workflow_config": S("humanWorkflowConfig") >> Bend(AwsBedrockHumanWorkflowConfig.mapping), + "custom_metrics": S("customMetrics", default=[]) >> ForallBend(AwsBedrockHumanEvaluationCustomMetric.mapping), + "dataset_metric_configs": S("datasetMetricConfigs", default=[]) + >> ForallBend(AwsBedrockEvaluationDatasetMetricConfig.mapping), + } + human_workflow_config: Optional[AwsBedrockHumanWorkflowConfig] = field(default=None, metadata={"description": "The parameters of the human workflow."}) # fmt: skip + custom_metrics: Optional[List[AwsBedrockHumanEvaluationCustomMetric]] = field(factory=list, metadata={"description": "A HumanEvaluationCustomMetric object. It contains the names the metrics, how the metrics are to be evaluated, an optional description."}) # fmt: skip + dataset_metric_configs: Optional[List[AwsBedrockEvaluationDatasetMetricConfig]] = field(factory=list, metadata={"description": "Use to specify the metrics, task, and prompt dataset to be used in your model evaluation job."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationConfig: + kind: ClassVar[str] = "aws_bedrock_evaluation_config" + mapping: ClassVar[Dict[str, Bender]] = { + "automated": S("automated") >> Bend(AwsBedrockAutomatedEvaluationConfig.mapping), + "human": S("human") >> Bend(AwsBedrockHumanEvaluationConfig.mapping), + } + automated: Optional[AwsBedrockAutomatedEvaluationConfig] = field(default=None, metadata={"description": "Used to specify an automated model evaluation job. See AutomatedEvaluationConfig to view the required parameters."}) # fmt: skip + human: Optional[AwsBedrockHumanEvaluationConfig] = field(default=None, metadata={"description": "Used to specify a model evaluation job that uses human workers.See HumanEvaluationConfig to view the required parameters."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationBedrockModel: + kind: ClassVar[str] = "aws_bedrock_evaluation_bedrock_model" + mapping: ClassVar[Dict[str, Bender]] = { + "model_identifier": S("modelIdentifier"), + "inference_params": S("inferenceParams"), + } + model_identifier: Optional[str] = field(default=None, metadata={"description": "The ARN of the Amazon Bedrock model specified."}) # fmt: skip + inference_params: Optional[str] = field(default=None, metadata={"description": "Each Amazon Bedrock support different inference parameters that change how the model behaves during inference."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationModelConfig: + kind: ClassVar[str] = "aws_bedrock_evaluation_model_config" + mapping: ClassVar[Dict[str, Bender]] = { + "bedrock_model": S("bedrockModel") >> Bend(AwsBedrockEvaluationBedrockModel.mapping) + } + bedrock_model: Optional[AwsBedrockEvaluationBedrockModel] = field(default=None, metadata={"description": "Defines the Amazon Bedrock model and inference parameters you want used."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationInferenceConfig: + kind: ClassVar[str] = "aws_bedrock_evaluation_inference_config" + mapping: ClassVar[Dict[str, Bender]] = { + "models": S("models", default=[]) >> ForallBend(AwsBedrockEvaluationModelConfig.mapping) + } + models: Optional[List[AwsBedrockEvaluationModelConfig]] = field(factory=list, metadata={"description": "Used to specify the models."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockEvaluationJob(BedrockTaggable, BaseAIJob, AwsResource): + kind: ClassVar[str] = "aws_bedrock_evaluation_job" + kind_display: ClassVar[str] = "AWS Bedrock Evaluation Job" + kind_description: ClassVar[str] = ( + "AWS Bedrock Evaluation Job assesses the performance of machine learning models, running test datasets and producing metrics " + "that indicate the effectiveness of the model's predictions." + ) + aws_metadata: ClassVar[Dict[str, Any]] = {} + metadata: ClassVar[Dict[str, Any]] = {"icon": "job", "group": "ai"} + kind_service: ClassVar[Optional[str]] = service_name + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsIamRole.kind]}, + "successors": {"default": [AwsS3Bucket.kind, AwsKmsKey.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock", "list-evaluation-jobs", "jobSummaries", expected_errors=["AccessDeniedException"] + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("jobArn"), + "name": S("jobName"), + "arn": S("jobArn"), + "ctime": S("creationTime"), + "mtime": S("lastModifiedTime"), + "job_name": S("jobName"), + "status": S("status"), + "job_arn": S("jobArn"), + "job_description": S("jobDescription"), + "role_arn": S("roleArn"), + "customer_encryption_key_arn": S("customerEncryptionKeyId"), + "job_type": S("jobType"), + "evaluation_config": S("evaluationConfig") >> Bend(AwsBedrockEvaluationConfig.mapping), + "job_inference_config": S("inferenceConfig") >> Bend(AwsBedrockEvaluationInferenceConfig.mapping), + "output_data_config": S("outputDataConfig", "s3Uri"), + "creation_time": S("creationTime"), + "last_modified_time": S("lastModifiedTime"), + "failure_messages": S("failureMessages", default=[]), + } + job_name: Optional[str] = field(default=None, metadata={"description": "The name of the model evaluation job."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the model evaluation job."}) # fmt: skip + job_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the model evaluation job."}) # fmt: skip + job_description: Optional[str] = field(default=None, metadata={"description": "The description of the model evaluation job."}) # fmt: skip + role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM service role used in the model evaluation job."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the customer managed key specified when the model evaluation job was created."}) # fmt: skip + job_type: Optional[str] = field(default=None, metadata={"description": "The type of model evaluation job."}) # fmt: skip + evaluation_config: Optional[AwsBedrockEvaluationConfig] = field(default=None, metadata={"description": "Contains details about the type of model evaluation job, the metrics used, the task type selected, the datasets used, and any custom metrics you defined."}) # fmt: skip + job_inference_config: Optional[AwsBedrockEvaluationInferenceConfig] = field(default=None, metadata={"description": "Details about the models you specified in your model evaluation job."}) # fmt: skip + output_data_config: Optional[str] = field(default=None, metadata={"description": "Amazon S3 location for where output data is saved."}) # fmt: skip + creation_time: Optional[datetime] = field(default=None, metadata={"description": "When the model evaluation job was created."}) # fmt: skip + last_modified_time: Optional[datetime] = field(default=None, metadata={"description": "When the model evaluation job was last modified."}) # fmt: skip + failure_messages: Optional[List[str]] = field(factory=list, metadata={"description": "An array of strings the specify why the model evaluation job has failed."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + if output_data_config := self.output_data_config: + bucket_name = AwsS3Bucket.name_from_path(output_data_config) + builder.add_edge(self, clazz=AwsS3Bucket, name=bucket_name) + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec(service_name, "get-evaluation-job")] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(job: AwsResource) -> None: + tags = builder.client.list( + service_name, + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceARN=job.arn, + ) + if tags: + for tag in tags: + job.tags.update({tag.get("key"): tag.get("value")}) + + for js in json: + for result in builder.client.list( + service_name, + "get-evaluation-job", + jobIdentifier=js["jobArn"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, add_tags, instance) + + +@define(eq=False, slots=False) +class AwsBedrockGuardrailConfiguration: + kind: ClassVar[str] = "aws_bedrock_guardrail_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "guardrail_identifier": S("guardrailIdentifier"), + "guardrail_version": S("guardrailVersion"), + } + guardrail_identifier: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the guardrail."}) # fmt: skip + guardrail_version: Optional[str] = field(default=None, metadata={"description": "The version of the guardrail."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockMemoryConfiguration: + kind: ClassVar[str] = "aws_bedrock_memory_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "enabled_memory_types": S("enabledMemoryTypes", default=[]), + "storage_days": S("storageDays"), + } + enabled_memory_types: Optional[List[str]] = field(factory=list, metadata={"description": "The type of memory that is stored."}) # fmt: skip + storage_days: Optional[int] = field(default=None, metadata={"description": "The number of days the agent is configured to retain the conversational context."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockInferenceConfiguration: + kind: ClassVar[str] = "aws_bedrock_inference_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "maximum_length": S("maximumLength"), + "stop_sequences": S("stopSequences", default=[]), + "temperature": S("temperature"), + "top_k": S("topK"), + "top_p": S("topP"), + } + maximum_length: Optional[int] = field(default=None, metadata={"description": "The maximum number of tokens to allow in the generated response."}) # fmt: skip + stop_sequences: Optional[List[str]] = field(factory=list, metadata={"description": "A list of stop sequences. A stop sequence is a sequence of characters that causes the model to stop generating the response."}) # fmt: skip + temperature: Optional[float] = field(default=None, metadata={"description": "The likelihood of the model selecting higher-probability options while generating a response. A lower value makes the model more likely to choose higher-probability options, while a higher value makes the model more likely to choose lower-probability options."}) # fmt: skip + top_k: Optional[int] = field(default=None, metadata={"description": "While generating a response, the model determines the probability of the following token at each point of generation. The value that you set for topK is the number of most-likely candidates from which the model chooses the next token in the sequence. For example, if you set topK to 50, the model selects the next token from among the top 50 most likely choices."}) # fmt: skip + top_p: Optional[float] = field(default=None, metadata={"description": "While generating a response, the model determines the probability of the following token at each point of generation. The value that you set for Top P determines the number of most-likely candidates from which the model chooses the next token in the sequence. For example, if you set topP to 80, the model only selects the next token from the top 80% of the probability distribution of next tokens."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "base_prompt_template": S("basePromptTemplate"), + "inference_configuration": S("inferenceConfiguration") >> Bend(AwsBedrockInferenceConfiguration.mapping), + "parser_mode": S("parserMode"), + "prompt_creation_mode": S("promptCreationMode"), + "prompt_state": S("promptState"), + "prompt_type": S("promptType"), + } + base_prompt_template: Optional[str] = field(default=None, metadata={"description": "Defines the prompt template with which to replace the default prompt template. You can use placeholder variables in the base prompt template to customize the prompt. For more information, see Prompt template placeholder variables. For more information, see Configure the prompt templates."}) # fmt: skip + inference_configuration: Optional[AwsBedrockInferenceConfiguration] = field(default=None, metadata={"description": "Contains inference parameters to use when the agent invokes a foundation model in the part of the agent sequence defined by the promptType. For more information, see Inference parameters for foundation models."}) # fmt: skip + parser_mode: Optional[str] = field(default=None, metadata={"description": "Specifies whether to override the default parser Lambda function when parsing the raw foundation model output in the part of the agent sequence defined by the promptType. If you set the field as OVERRIDEN, the overrideLambda field in the PromptOverrideConfiguration must be specified with the ARN of a Lambda function."}) # fmt: skip + prompt_creation_mode: Optional[str] = field(default=None, metadata={"description": "Specifies whether to override the default prompt template for this promptType. Set this value to OVERRIDDEN to use the prompt that you provide in the basePromptTemplate. If you leave it as DEFAULT, the agent uses a default prompt template."}) # fmt: skip + prompt_state: Optional[str] = field(default=None, metadata={"description": "Specifies whether to allow the agent to carry out the step specified in the promptType. If you set this value to DISABLED, the agent skips that step. The default state for each promptType is as follows. PRE_PROCESSING – ENABLED ORCHESTRATION – ENABLED KNOWLEDGE_BASE_RESPONSE_GENERATION – ENABLED POST_PROCESSING – DISABLED"}) # fmt: skip + prompt_type: Optional[str] = field(default=None, metadata={"description": "The step in the agent sequence that this prompt configuration applies to."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptOverrideConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_override_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "override_lambda": S("overrideLambda"), + "prompt_configurations": S("promptConfigurations", default=[]) + >> ForallBend(AwsBedrockPromptConfiguration.mapping), + } + override_lambda: Optional[str] = field(default=None, metadata={"description": "The ARN of the Lambda function to use when parsing the raw foundation model output in parts of the agent sequence. If you specify this field, at least one of the promptConfigurations must contain a parserMode value that is set to OVERRIDDEN. For more information, see Parser Lambda function in Agents for Amazon Bedrock."}) # fmt: skip + prompt_configurations: Optional[List[AwsBedrockPromptConfiguration]] = field(factory=list, metadata={"description": "Contains configurations to override a prompt template in one part of an agent sequence. For more information, see Advanced prompts."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockAgent(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent" + kind_display: ClassVar[str] = "AWS Bedrock Agent" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent is an intelligent service designed to facilitate communication with machine learning models, " + "acting as a mediator between applications and model execution workflows." + ) + aws_metadata: ClassVar[Dict[str, Any]] = { + "provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/agents/{id}" + } + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + reference_kinds: ClassVar[ModelReference] = { + "successors": { + "default": [ + AwsBedrockGuardrail.kind, + AwsKmsKey.kind, + "aws_bedrock_agent_version", + "aws_bedrock_agent_knowledge_base", + ] + }, + "predecessors": {"default": [AwsIamRole.kind, AwsBedrockFoundationModel.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("bedrock-agent", "list-agents", "agentSummaries") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("agent", "agentId"), + "name": S("agent", "agentName"), + "ctime": S("agent", "createdAt"), + "mtime": S("agent", "updatedAt"), + "arn": S("agent", "agentArn"), + "agent_arn": S("agent", "agentArn"), + "agent_id": S("agent", "agentId"), + "agent_name": S("agent", "agentName"), + "agent_resource_role_arn": S("agent", "agentResourceRoleArn"), + "agent_status": S("agent", "agentStatus"), + "agent_version": S("agent", "agentVersion").or_else(S("latestAgentVersion")), + "client_token": S("agent", "clientToken"), + "created_at": S("agent", "createdAt"), + "customer_encryption_key_arn": S("agent", "customerEncryptionKeyArn"), + "description": S("agent", "description"), + "failure_reasons": S("agent", "failureReasons", default=[]), + "foundation_model": S("agent", "foundationModel"), + "guardrail_configuration": S("agent", "guardrailConfiguration") + >> Bend(AwsBedrockGuardrailConfiguration.mapping), + "idle_session_ttl_in_seconds": S("agent", "idleSessionTTLInSeconds"), + "instruction": S("agent", "instruction"), + "memory_configuration": S("agent", "memoryConfiguration") >> Bend(AwsBedrockMemoryConfiguration.mapping), + "prepared_at": S("agent", "preparedAt"), + "prompt_override_configuration": S("agent", "promptOverrideConfiguration") + >> Bend(AwsBedrockPromptOverrideConfiguration.mapping), + "agent_recommended_actions": S("agent", "recommendedActions", default=[]), + "updated_at": S("agent", "updatedAt"), + } + agent_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the agent."}) # fmt: skip + agent_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the agent."}) # fmt: skip + agent_name: Optional[str] = field(default=None, metadata={"description": "The name of the agent."}) # fmt: skip + agent_resource_role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM role with permissions to invoke API operations on the agent."}) # fmt: skip + agent_status: Optional[str] = field(default=None, metadata={"description": "The status of the agent and whether it is ready for use. The following statuses are possible: CREATING – The agent is being created. PREPARING – The agent is being prepared. PREPARED – The agent is prepared and ready to be invoked. NOT_PREPARED – The agent has been created but not yet prepared. FAILED – The agent API operation failed. UPDATING – The agent is being updated. DELETING – The agent is being deleted."}) # fmt: skip + agent_version: Optional[str] = field(default=None, metadata={"description": "The version of the agent."}) # fmt: skip + client_token: Optional[str] = field(default=None, metadata={"description": "A unique, case-sensitive identifier to ensure that the API request completes no more than one time. If this token matches a previous request, Amazon Bedrock ignores the request, but does not return an error. For more information, see Ensuring idempotency."}) # fmt: skip + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the agent was created."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the KMS key that encrypts the agent."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the agent."}) # fmt: skip + failure_reasons: Optional[List[str]] = field(factory=list, metadata={"description": "Contains reasons that the agent-related API that you invoked failed."}) # fmt: skip + foundation_model: Optional[str] = field(default=None, metadata={"description": "The foundation model used for orchestration by the agent."}) # fmt: skip + guardrail_configuration: Optional[AwsBedrockGuardrailConfiguration] = field(default=None, metadata={"description": "Details about the guardrail associated with the agent."}) # fmt: skip + idle_session_ttl_in_seconds: Optional[int] = field(default=None, metadata={"description": "The number of seconds for which Amazon Bedrock keeps information about a user's conversation with the agent. A user interaction remains active for the amount of time specified. If no conversation occurs during this time, the session expires and Amazon Bedrock deletes any data provided before the timeout."}) # fmt: skip + instruction: Optional[str] = field(default=None, metadata={"description": "Instructions that tell the agent what it should do and how it should interact with users."}) # fmt: skip + memory_configuration: Optional[AwsBedrockMemoryConfiguration] = field(default=None, metadata={"description": "Contains memory configuration for the agent."}) # fmt: skip + prepared_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the agent was last prepared."}) # fmt: skip + prompt_override_configuration: Optional[AwsBedrockPromptOverrideConfiguration] = field(default=None, metadata={"description": "Contains configurations to override prompt templates in different parts of an agent sequence. For more information, see Advanced prompts."}) # fmt: skip + agent_recommended_actions: Optional[List[str]] = field(factory=list, metadata={"description": "Contains recommended actions to take for the agent-related API that you invoked to succeed."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the agent was last updated."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.agent_resource_role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + if (g_configuration := self.guardrail_configuration) and (g_id := g_configuration.guardrail_identifier): + builder.add_edge(self, clazz=AwsBedrockGuardrail, id=g_id) + if foundation_model_name := self.foundation_model: + builder.add_edge(self, reverse=True, clazz=AwsBedrockFoundationModel, id=foundation_model_name) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", + action="delete-agent", + result_name=None, + agentId=self.id, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-agent")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec("bedrock-agent", "get-agent")] + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(agent: AwsResource) -> None: + tags = builder.client.list( + "bedrock-agent", + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceArn=agent.arn, + ) + if tags: + agent.tags.update(tags[0]) + + def collect_agent_versions(agent: AwsBedrockAgent) -> None: + if not agent.agent_version or agent.agent_version == "DRAFT": + return + for result in builder.client.list( + "bedrock-agent", + "get-agent-version", + agentId=agent.id, + agentVersion=agent.agent_version, + ): + if instance := AwsBedrockAgentVersion.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work("bedrock-agent", add_tags, instance) + + for js in json: + for result in builder.client.list( + "bedrock-agent", + "get-agent", + agentId=js["agentId"], + ): + if instance := AwsBedrockAgent.from_api(result, builder): + instance.agent_version = js["latestAgentVersion"] + builder.add_node(instance, js) + builder.submit_work("bedrock-agent", add_tags, instance) + builder.submit_work("bedrock-agent", collect_agent_versions, instance) + + +@define(eq=False, slots=False) +class AwsBedrockAgentVersion(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent_version" + kind_display: ClassVar[str] = "AWS Bedrock Agent Version" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent Version refers to a specific iteration of the Bedrock Agent, ensuring compatibility " + "and enhancements across different versions of the agent for improved functionality." + ) + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + aws_metadata: ClassVar[Dict[str, Any]] = { + "provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/agents/{id}/versions/{version}" + } + metadata: ClassVar[Dict[str, Any]] = {"icon": "version", "group": "ai"} + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsIamRole.kind, AwsBedrockFoundationModel.kind]}, + "successors": {"default": [AwsBedrockGuardrail.kind, AwsKmsKey.kind]}, + } + # Collected via AwsBedrockAgent() + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("agentVersion", "agentId"), + "name": S("agentVersion", "agentName"), + "ctime": S("agentVersion", "createdAt"), + "mtime": S("agentVersion", "updatedAt"), + "arn": S("agentVersion", "agentArn"), + "agent_arn": S("agentVersion", "agentArn"), + "agent_id": S("agentVersion", "agentId"), + "agent_name": S("agentVersion", "agentName"), + "agent_resource_role_arn": S("agentVersion", "agentResourceRoleArn"), + "agent_status": S("agentVersion", "agentStatus"), + "created_at": S("agentVersion", "createdAt"), + "customer_encryption_key_arn": S("agentVersion", "customerEncryptionKeyArn"), + "description": S("agentVersion", "description"), + "failure_reasons": S("agentVersion", "failureReasons", default=[]), + "foundation_model": S("agentVersion", "foundationModel"), + "guardrail_configuration": S("agentVersion", "guardrailConfiguration") + >> Bend(AwsBedrockGuardrailConfiguration.mapping), + "idle_session_ttl_in_seconds": S("agentVersion", "idleSessionTTLInSeconds"), + "instruction": S("agentVersion", "instruction"), + "memory_configuration": S("agentVersion", "memoryConfiguration") >> Bend(AwsBedrockMemoryConfiguration.mapping), + "prompt_override_configuration": S("agentVersion", "promptOverrideConfiguration") + >> Bend(AwsBedrockPromptOverrideConfiguration.mapping), + "agent_recommended_actions": S("agentVersion", "recommendedActions", default=[]), + "updated_at": S("agentVersion", "updatedAt"), + "version": S("agentVersion", "version"), + } + agent_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the agent that the version belongs to."}) # fmt: skip + agent_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the agent that the version belongs to."}) # fmt: skip + agent_name: Optional[str] = field(default=None, metadata={"description": "The name of the agent that the version belongs to."}) # fmt: skip + agent_resource_role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM role with permissions to invoke API operations on the agent."}) # fmt: skip + agent_status: Optional[str] = field(default=None, metadata={"description": "The status of the agent that the version belongs to."}) # fmt: skip + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the version was created."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the KMS key that encrypts the agent."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the version."}) # fmt: skip + failure_reasons: Optional[List[str]] = field(factory=list, metadata={"description": "A list of reasons that the API operation on the version failed."}) # fmt: skip + foundation_model: Optional[str] = field(default=None, metadata={"description": "The foundation model that the version invokes."}) # fmt: skip + guardrail_configuration: Optional[AwsBedrockGuardrailConfiguration] = field(default=None, metadata={"description": "Details about the guardrail associated with the agent."}) # fmt: skip + idle_session_ttl_in_seconds: Optional[int] = field(default=None, metadata={"description": "The number of seconds for which Amazon Bedrock keeps information about a user's conversation with the agent. A user interaction remains active for the amount of time specified. If no conversation occurs during this time, the session expires and Amazon Bedrock deletes any data provided before the timeout."}) # fmt: skip + instruction: Optional[str] = field(default=None, metadata={"description": "The instructions provided to the agent."}) # fmt: skip + memory_configuration: Optional[AwsBedrockMemoryConfiguration] = field(default=None, metadata={"description": "Contains details of the memory configuration on the version of the agent."}) # fmt: skip + prompt_override_configuration: Optional[AwsBedrockPromptOverrideConfiguration] = field(default=None, metadata={"description": "Contains configurations to override prompt templates in different parts of an agent sequence. For more information, see Advanced prompts."}) # fmt: skip + agent_recommended_actions: Optional[List[str]] = field(factory=list, metadata={"description": "A list of recommended actions to take for the failed API operation on the version to succeed."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the version was last updated."}) # fmt: skip + version: Optional[str] = field(default=None, metadata={"description": "The version number."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.agent_resource_role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + if (g_configuration := self.guardrail_configuration) and (g_id := g_configuration.guardrail_identifier): + builder.add_edge(self, clazz=AwsBedrockGuardrail, id=g_id) + if foundation_model_name := self.foundation_model: + builder.add_edge(self, reverse=True, clazz=AwsBedrockFoundationModel, id=foundation_model_name) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", + action="delete-agent-version", + result_name=None, + agentId=self.agent_id, + agentVersion=self.version, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-agent-version")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [AwsApiSpec("bedrock-agent", "get-agent-version")] + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + +@define(eq=False, slots=False) +class AwsBedrockEmbeddingModelConfiguration: + kind: ClassVar[str] = "aws_bedrock_embedding_model_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "bedrock_embedding_model_configuration": S("bedrockEmbeddingModelConfiguration", "dimensions") + } + bedrock_embedding_model_configuration: Optional[int] = field(default=None, metadata={"description": "The vector configuration details on the Bedrock embeddings model."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockVectorKnowledgeBaseConfiguration: + kind: ClassVar[str] = "aws_bedrock_vector_knowledge_base_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "embedding_model_arn": S("embeddingModelArn"), + "embedding_model_configuration": S("embeddingModelConfiguration") + >> Bend(AwsBedrockEmbeddingModelConfiguration.mapping), + } + embedding_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the model used to create vector embeddings for the knowledge base."}) # fmt: skip + embedding_model_configuration: Optional[AwsBedrockEmbeddingModelConfiguration] = field(default=None, metadata={"description": "The embeddings model configuration details for the vector model used in Knowledge Base."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockKnowledgeBaseConfiguration: + kind: ClassVar[str] = "aws_bedrock_knowledge_base_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "type": S("type"), + "vector_knowledge_base_configuration": S("vectorKnowledgeBaseConfiguration") + >> Bend(AwsBedrockVectorKnowledgeBaseConfiguration.mapping), + } + type: Optional[str] = field(default=None, metadata={"description": "The type of data that the data source is converted into for the knowledge base."}) # fmt: skip + vector_knowledge_base_configuration: Optional[AwsBedrockVectorKnowledgeBaseConfiguration] = field(default=None, metadata={"description": "Contains details about the embeddings model that'sused to convert the data source."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockMongoDbAtlasFieldMapping: + kind: ClassVar[str] = "aws_bedrock_mongo_db_atlas_field_mapping" + mapping: ClassVar[Dict[str, Bender]] = { + "metadata_field": S("metadataField"), + "text_field": S("textField"), + "vector_field": S("vectorField"), + } + metadata_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores metadata about the vector store."}) # fmt: skip + text_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the raw text from your data. The text is split according to the chunking strategy you choose."}) # fmt: skip + vector_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the vector embeddings for your data sources."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockMongoDbAtlasConfiguration: + kind: ClassVar[str] = "aws_bedrock_mongo_db_atlas_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "collection_name": S("collectionName"), + "credentials_secret_arn": S("credentialsSecretArn"), + "database_name": S("databaseName"), + "endpoint": S("endpoint"), + "endpoint_service_name": S("endpointServiceName"), + "field_mapping": S("fieldMapping") >> Bend(AwsBedrockMongoDbAtlasFieldMapping.mapping), + "vector_index_name": S("vectorIndexName"), + } + collection_name: Optional[str] = field(default=None, metadata={"description": "The collection name of the knowledge base in MongoDB Atlas."}) # fmt: skip + credentials_secret_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the secret that you created in Secrets Manager that contains user credentials for your MongoDB Atlas cluster."}) # fmt: skip + database_name: Optional[str] = field(default=None, metadata={"description": "The database name in your MongoDB Atlas cluster for your knowledge base."}) # fmt: skip + endpoint: Optional[str] = field(default=None, metadata={"description": "The endpoint URL of your MongoDB Atlas cluster for your knowledge base."}) # fmt: skip + endpoint_service_name: Optional[str] = field(default=None, metadata={"description": "The name of the VPC endpoint service in your account that is connected to your MongoDB Atlas cluster."}) # fmt: skip + field_mapping: Optional[AwsBedrockMongoDbAtlasFieldMapping] = field(default=None, metadata={"description": "Contains the names of the fields to which to map information about the vector store."}) # fmt: skip + vector_index_name: Optional[str] = field(default=None, metadata={"description": "The name of the MongoDB Atlas vector search index."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockOpenSearchServerlessFieldMapping: + kind: ClassVar[str] = "aws_bedrock_open_search_serverless_field_mapping" + mapping: ClassVar[Dict[str, Bender]] = { + "metadata_field": S("metadataField"), + "text_field": S("textField"), + "vector_field": S("vectorField"), + } + metadata_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores metadata about the vector store."}) # fmt: skip + text_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the raw text from your data. The text is split according to the chunking strategy you choose."}) # fmt: skip + vector_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the vector embeddings for your data sources."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockOpenSearchServerlessConfiguration: + kind: ClassVar[str] = "aws_bedrock_open_search_serverless_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "collection_arn": S("collectionArn"), + "field_mapping": S("fieldMapping") >> Bend(AwsBedrockOpenSearchServerlessFieldMapping.mapping), + "vector_index_name": S("vectorIndexName"), + } + collection_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the OpenSearch Service vector store."}) # fmt: skip + field_mapping: Optional[AwsBedrockOpenSearchServerlessFieldMapping] = field(default=None, metadata={"description": "Contains the names of the fields to which to map information about the vector store."}) # fmt: skip + vector_index_name: Optional[str] = field(default=None, metadata={"description": "The name of the vector store."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPineconeFieldMapping: + kind: ClassVar[str] = "aws_bedrock_pinecone_field_mapping" + mapping: ClassVar[Dict[str, Bender]] = {"metadata_field": S("metadataField"), "text_field": S("textField")} + metadata_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores metadata about the vector store."}) # fmt: skip + text_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the raw text from your data. The text is split according to the chunking strategy you choose."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPineconeConfiguration: + kind: ClassVar[str] = "aws_bedrock_pinecone_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "connection_string": S("connectionString"), + "credentials_secret_arn": S("credentialsSecretArn"), + "field_mapping": S("fieldMapping") >> Bend(AwsBedrockPineconeFieldMapping.mapping), + "namespace": S("namespace"), + } + connection_string: Optional[str] = field(default=None, metadata={"description": "The endpoint URL for your index management page."}) # fmt: skip + credentials_secret_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the secret that you created in Secrets Manager that is linked to your Pinecone API key."}) # fmt: skip + field_mapping: Optional[AwsBedrockPineconeFieldMapping] = field(default=None, metadata={"description": "Contains the names of the fields to which to map information about the vector store."}) # fmt: skip + namespace: Optional[str] = field(default=None, metadata={"description": "The namespace to be used to write new data to your database."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRdsFieldMapping: + kind: ClassVar[str] = "aws_bedrock_rds_field_mapping" + mapping: ClassVar[Dict[str, Bender]] = { + "metadata_field": S("metadataField"), + "primary_key_field": S("primaryKeyField"), + "text_field": S("textField"), + "vector_field": S("vectorField"), + } + metadata_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores metadata about the vector store."}) # fmt: skip + primary_key_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the ID for each entry."}) # fmt: skip + text_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the raw text from your data. The text is split according to the chunking strategy you choose."}) # fmt: skip + vector_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the vector embeddings for your data sources."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRdsConfiguration: + kind: ClassVar[str] = "aws_bedrock_rds_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "credentials_secret_arn": S("credentialsSecretArn"), + "database_name": S("databaseName"), + "field_mapping": S("fieldMapping") >> Bend(AwsBedrockRdsFieldMapping.mapping), + "resource_arn": S("resourceArn"), + "table_name": S("tableName"), + } + credentials_secret_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the secret that you created in Secrets Manager that is linked to your Amazon RDS database."}) # fmt: skip + database_name: Optional[str] = field(default=None, metadata={"description": "The name of your Amazon RDS database."}) # fmt: skip + field_mapping: Optional[AwsBedrockRdsFieldMapping] = field(default=None, metadata={"description": "Contains the names of the fields to which to map information about the vector store."}) # fmt: skip + resource_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the vector store."}) # fmt: skip + table_name: Optional[str] = field(default=None, metadata={"description": "The name of the table in the database."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRedisEnterpriseCloudFieldMapping: + kind: ClassVar[str] = "aws_bedrock_redis_enterprise_cloud_field_mapping" + mapping: ClassVar[Dict[str, Bender]] = { + "metadata_field": S("metadataField"), + "text_field": S("textField"), + "vector_field": S("vectorField"), + } + metadata_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores metadata about the vector store."}) # fmt: skip + text_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the raw text from your data. The text is split according to the chunking strategy you choose."}) # fmt: skip + vector_field: Optional[str] = field(default=None, metadata={"description": "The name of the field in which Amazon Bedrock stores the vector embeddings for your data sources."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRedisEnterpriseCloudConfiguration: + kind: ClassVar[str] = "aws_bedrock_redis_enterprise_cloud_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "credentials_secret_arn": S("credentialsSecretArn"), + "endpoint": S("endpoint"), + "field_mapping": S("fieldMapping") >> Bend(AwsBedrockRedisEnterpriseCloudFieldMapping.mapping), + "vector_index_name": S("vectorIndexName"), + } + credentials_secret_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the secret that you created in Secrets Manager that is linked to your Redis Enterprise Cloud database."}) # fmt: skip + endpoint: Optional[str] = field(default=None, metadata={"description": "The endpoint URL of the Redis Enterprise Cloud database."}) # fmt: skip + field_mapping: Optional[AwsBedrockRedisEnterpriseCloudFieldMapping] = field(default=None, metadata={"description": "Contains the names of the fields to which to map information about the vector store."}) # fmt: skip + vector_index_name: Optional[str] = field(default=None, metadata={"description": "The name of the vector index."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockStorageConfiguration: + kind: ClassVar[str] = "aws_bedrock_storage_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "mongo_db_atlas_configuration": S("mongoDbAtlasConfiguration") + >> Bend(AwsBedrockMongoDbAtlasConfiguration.mapping), + "opensearch_serverless_configuration": S("opensearchServerlessConfiguration") + >> Bend(AwsBedrockOpenSearchServerlessConfiguration.mapping), + "pinecone_configuration": S("pineconeConfiguration") >> Bend(AwsBedrockPineconeConfiguration.mapping), + "rds_configuration": S("rdsConfiguration") >> Bend(AwsBedrockRdsConfiguration.mapping), + "redis_enterprise_cloud_configuration": S("redisEnterpriseCloudConfiguration") + >> Bend(AwsBedrockRedisEnterpriseCloudConfiguration.mapping), + "type": S("type"), + } + mongo_db_atlas_configuration: Optional[AwsBedrockMongoDbAtlasConfiguration] = field(default=None, metadata={"description": "Contains the storage configuration of the knowledge base in MongoDB Atlas."}) # fmt: skip + opensearch_serverless_configuration: Optional[AwsBedrockOpenSearchServerlessConfiguration] = field(default=None, metadata={"description": "Contains the storage configuration of the knowledge base in Amazon OpenSearch Service."}) # fmt: skip + pinecone_configuration: Optional[AwsBedrockPineconeConfiguration] = field(default=None, metadata={"description": "Contains the storage configuration of the knowledge base in Pinecone."}) # fmt: skip + rds_configuration: Optional[AwsBedrockRdsConfiguration] = field(default=None, metadata={"description": "Contains details about the storage configuration of the knowledge base in Amazon RDS. For more information, see Create a vector index in Amazon RDS."}) # fmt: skip + redis_enterprise_cloud_configuration: Optional[AwsBedrockRedisEnterpriseCloudConfiguration] = field(default=None, metadata={"description": "Contains the storage configuration of the knowledge base in Redis Enterprise Cloud."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "The vector store service in which the knowledge base is stored."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockAgentKnowledgeBase(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent_knowledge_base" + kind_display: ClassVar[str] = "AWS Bedrock Agent Knowledge Base" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent Knowledge Base contains the knowledge resources that the Bedrock Agent relies on " + "to provide accurate responses and perform tasks based on learned information." + ) + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + aws_metadata: ClassVar[Dict[str, Any]] = { + "provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/knowledge-bases/knowledge-base/{name}/{id}/0" + } + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsIamRole.kind]}, + "successors": { + "default": [ + AwsRdsCluster.kind, + AwsRdsInstance.kind, + ] + }, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("bedrock-agent", "list-knowledge-bases", "knowledgeBaseSummaries") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("knowledgeBase", "knowledgeBaseId"), + "name": S("knowledgeBase", "name"), + "arn": S("knowledgeBase", "knowledgeBaseArn"), + "ctime": S("knowledgeBase", "createdAt"), + "mtime": S("knowledgeBase", "updatedAt"), + "created_at": S("knowledgeBase", "createdAt"), + "description": S("knowledgeBase", "description"), + "failure_reasons": S("knowledgeBase", "failureReasons", default=[]), + "knowledge_base_arn": S("knowledgeBase", "knowledgeBaseArn"), + "knowledge_base_configuration": S("knowledgeBase", "knowledgeBaseConfiguration") + >> Bend(AwsBedrockKnowledgeBaseConfiguration.mapping), + "knowledge_base_id": S("knowledgeBase", "knowledgeBaseId"), + "role_arn": S("knowledgeBase", "roleArn"), + "status": S("knowledgeBase", "status"), + "storage_configuration": S("knowledgeBase", "storageConfiguration") + >> Bend(AwsBedrockStorageConfiguration.mapping), + "updated_at": S("knowledgeBase", "updatedAt"), + } + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the knowledge base was created."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the knowledge base."}) # fmt: skip + failure_reasons: Optional[List[str]] = field(factory=list, metadata={"description": "A list of reasons that the API operation on the knowledge base failed."}) # fmt: skip + knowledge_base_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the knowledge base."}) # fmt: skip + knowledge_base_configuration: Optional[AwsBedrockKnowledgeBaseConfiguration] = field(default=None, metadata={"description": "Contains details about the embeddings configuration of the knowledge base."}) # fmt: skip + knowledge_base_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the knowledge base."}) # fmt: skip + role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM role with permissions to invoke API operations on the knowledge base."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the knowledge base. The following statuses are possible: CREATING – The knowledge base is being created. ACTIVE – The knowledge base is ready to be queried. DELETING – The knowledge base is being deleted. UPDATING – The knowledge base is being updated. FAILED – The knowledge base API operation failed."}) # fmt: skip + storage_configuration: Optional[AwsBedrockStorageConfiguration] = field(default=None, metadata={"description": "Contains details about the storage configuration of the knowledge base."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the knowledge base was last updated."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if storage_config := self.storage_configuration: + if rds_config := storage_config.rds_configuration: + builder.add_edge(self, clazz=AwsRdsCluster, rds_database_name=rds_config.database_name) + builder.add_edge(self, clazz=AwsRdsInstance, name=rds_config.database_name) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", action="delete-knowledge-base", result_name=None, knowledgeBaseId=self.id + ) + return True + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(knowledge_base: AwsResource) -> None: + tags = builder.client.list( + "bedrock-agent", + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceArn=knowledge_base.arn, + ) + if tags: + knowledge_base.tags.update(tags[0]) + + for js in json: + for result in builder.client.list( + "bedrock-agent", + "get-knowledge-base", + knowledgeBaseId=js["knowledgeBaseId"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work(service_name, add_tags, instance) + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-knowledge-base")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [ + AwsApiSpec("bedrock-agent", "list-knowledge-bases"), + AwsApiSpec("bedrock-agent", "get-knowledge-base"), + ] + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + +@define(eq=False, slots=False) +class AwsBedrockPromptModelInferenceConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_model_inference_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "max_tokens": S("maxTokens"), + "stop_sequences": S("stopSequences", default=[]), + "temperature": S("temperature"), + "top_k": S("topK"), + "top_p": S("topP"), + } + max_tokens: Optional[int] = field(default=None, metadata={"description": "The maximum number of tokens to return in the response."}) # fmt: skip + stop_sequences: Optional[List[str]] = field(factory=list, metadata={"description": "A list of strings that define sequences after which the model will stop generating."}) # fmt: skip + temperature: Optional[float] = field(default=None, metadata={"description": "Controls the randomness of the response. Choose a lower value for more predictable outputs and a higher value for more surprising outputs."}) # fmt: skip + top_k: Optional[int] = field(default=None, metadata={"description": "The number of most-likely candidates that the model considers for the next token during generation."}) # fmt: skip + top_p: Optional[float] = field(default=None, metadata={"description": "The percentage of most-likely candidates that the model considers for the next token."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptInferenceConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_inference_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "text": S("text") >> Bend(AwsBedrockPromptModelInferenceConfiguration.mapping) + } + text: Optional[AwsBedrockPromptModelInferenceConfiguration] = field(default=None, metadata={"description": "Contains inference configurations for a text prompt."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockTextPromptTemplateConfiguration: + kind: ClassVar[str] = "aws_bedrock_text_prompt_template_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "input_variables": S("inputVariables", default=[]) >> ForallBend(S("name")), + "text": S("text"), + } + input_variables: Optional[List[str]] = field(factory=list, metadata={"description": "An array of the variables in the prompt template."}) # fmt: skip + text: Optional[str] = field(default=None, metadata={"description": "The message for the prompt."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptTemplateConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_template_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "text": S("text") >> Bend(AwsBedrockTextPromptTemplateConfiguration.mapping) + } + text: Optional[AwsBedrockTextPromptTemplateConfiguration] = field(default=None, metadata={"description": "Contains configurations for the text in a message for a prompt."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptVariant: + kind: ClassVar[str] = "aws_bedrock_prompt_variant" + mapping: ClassVar[Dict[str, Bender]] = { + "inference_configuration": S("inferenceConfiguration") >> Bend(AwsBedrockPromptInferenceConfiguration.mapping), + "model_id": S("modelId"), + "name": S("name"), + "template_configuration": S("templateConfiguration") >> Bend(AwsBedrockPromptTemplateConfiguration.mapping), + "template_type": S("templateType"), + } + inference_configuration: Optional[AwsBedrockPromptInferenceConfiguration] = field(default=None, metadata={"description": "Contains inference configurations for the prompt variant."}) # fmt: skip + model_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the model with which to run inference on the prompt."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "The name of the prompt variant."}) # fmt: skip + template_configuration: Optional[AwsBedrockPromptTemplateConfiguration] = field(default=None, metadata={"description": "Contains configurations for the prompt template."}) # fmt: skip + template_type: Optional[str] = field(default=None, metadata={"description": "The type of prompt template to use."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockAgentPrompt(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent_prompt" + kind_display: ClassVar[str] = "AWS Bedrock Agent Prompt" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent Prompt defines the input that the agent uses to generate responses or actions, " + "guiding the interaction between users and machine learning models through structured prompts." + ) + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/prompt-management/{id}"} # fmt: skip + reference_kinds: ClassVar[ModelReference] = { + "successors": {"default": [AwsBedrockCustomModel.kind, AwsKmsKey.kind]}, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock-agent", "list-prompts", "promptSummaries", expected_errors=["AccessDeniedException"] + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("id"), + "name": S("name"), + "ctime": S("createdAt"), + "mtime": S("updatedAt"), + "arn": S("arn"), + "created_at": S("createdAt"), + "customer_encryption_key_arn": S("customerEncryptionKeyArn"), + "default_variant": S("defaultVariant"), + "description": S("description"), + "updated_at": S("updatedAt"), + "prompt_variants": S("variants", default=[]) >> ForallBend(AwsBedrockPromptVariant.mapping), + "version": S("version"), + } + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the prompt was created."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the KMS key that the prompt is encrypted with."}) # fmt: skip + default_variant: Optional[str] = field(default=None, metadata={"description": "The name of the default variant for the prompt. This value must match the name field in the relevant PromptVariant object."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The descriptino of the prompt."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the prompt was last updated."}) # fmt: skip + prompt_variants: Optional[List[AwsBedrockPromptVariant]] = field(factory=list, metadata={"description": "A list of objects, each containing details about a variant of the prompt."}) # fmt: skip + version: Optional[str] = field(default=None, metadata={"description": "The version of the prompt."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if variants := self.prompt_variants: + for variant in variants: + builder.add_edge(self, clazz=AwsBedrockCustomModel, id=variant.model_id) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", + action="delete-prompt", + result_name=None, + promptIdentifier=self.id, + ) + return True + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-prompt")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec("bedrock-agent", "get-prompt")] + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(prompt: AwsResource) -> None: + tags = builder.client.list( + "bedrock-agent", + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceArn=prompt.arn, + ) + if tags: + prompt.tags.update(tags[0]) + + for js in json: + for result in builder.client.list( + "bedrock-agent", + "get-prompt", + promptIdentifier=js["id"], + promptVersion=js["version"], + ): + if instance := cls.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work("bedrock-agent", add_tags, instance) + + +@define(eq=False, slots=False) +class AwsBedrockFlowDataConnectionConfiguration: + kind: ClassVar[str] = "aws_bedrock_flow_data_connection_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"source_output": S("sourceOutput"), "target_input": S("targetInput")} + source_output: Optional[str] = field(default=None, metadata={"description": "The name of the output in the source node that the connection begins from."}) # fmt: skip + target_input: Optional[str] = field(default=None, metadata={"description": "The name of the input in the target node that the connection ends at."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowConnectionConfiguration: + kind: ClassVar[str] = "aws_bedrock_flow_connection_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "conditional": S("conditional", "condition"), + "data": S("data") >> Bend(AwsBedrockFlowDataConnectionConfiguration.mapping), + } + conditional: Optional[str] = field(default=None, metadata={"description": "The configuration of a connection originating from a Condition node."}) # fmt: skip + data: Optional[AwsBedrockFlowDataConnectionConfiguration] = field(default=None, metadata={"description": "The configuration of a connection originating from a node that isn't a Condition node."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowConnection: + kind: ClassVar[str] = "aws_bedrock_flow_connection" + mapping: ClassVar[Dict[str, Bender]] = { + "configuration": S("configuration") >> Bend(AwsBedrockFlowConnectionConfiguration.mapping), + "name": S("name"), + "source": S("source"), + "target": S("target"), + "type": S("type"), + } + configuration: Optional[AwsBedrockFlowConnectionConfiguration] = field(default=None, metadata={"description": "The configuration of the connection."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "A name for the connection that you can reference."}) # fmt: skip + source: Optional[str] = field(default=None, metadata={"description": "The node that the connection starts at."}) # fmt: skip + target: Optional[str] = field(default=None, metadata={"description": "The node that the connection ends at."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "Whether the source node that the connection begins from is a condition node (Conditional) or not (Data)."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowCondition: + kind: ClassVar[str] = "aws_bedrock_flow_condition" + mapping: ClassVar[Dict[str, Bender]] = {"expression": S("expression"), "name": S("name")} + expression: Optional[str] = field(default=None, metadata={"description": "Defines the condition. You must refer to at least one of the inputs in the condition. For more information, expand the Condition node section in Node types in prompt flows."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "A name for the condition that you can reference."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockConditionFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_condition_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "conditions": S("conditions", default=[]) >> ForallBend(AwsBedrockFlowCondition.mapping) + } + conditions: Optional[List[AwsBedrockFlowCondition]] = field(factory=list, metadata={"description": "An array of conditions. Each member contains the name of a condition and an expression that defines the condition."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockKnowledgeBaseFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_knowledge_base_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"knowledge_base_id": S("knowledgeBaseId"), "model_id": S("modelId")} + knowledge_base_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the knowledge base to query."}) # fmt: skip + model_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the model to use to generate a response from the query results. Omit this field if you want to return the retrieved results as an array."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockLexFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_lex_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"bot_alias_arn": S("botAliasArn"), "locale_id": S("localeId")} + bot_alias_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the Amazon Lex bot alias to invoke."}) # fmt: skip + locale_id: Optional[str] = field(default=None, metadata={"description": "The Region to invoke the Amazon Lex bot in."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptFlowNodeInlineConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_flow_node_inline_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "inference_configuration": S("inferenceConfiguration") >> Bend(AwsBedrockPromptInferenceConfiguration.mapping), + "model_id": S("modelId"), + "template_configuration": S("templateConfiguration") >> Bend(AwsBedrockPromptTemplateConfiguration.mapping), + "template_type": S("templateType"), + } + inference_configuration: Optional[AwsBedrockPromptInferenceConfiguration] = field(default=None, metadata={"description": "Contains inference configurations for the prompt."}) # fmt: skip + model_id: Optional[str] = field(default=None, metadata={"description": "The unique identifier of the model to run inference with."}) # fmt: skip + template_configuration: Optional[AwsBedrockPromptTemplateConfiguration] = field(default=None, metadata={"description": "Contains a prompt and variables in the prompt that can be replaced with values at runtime."}) # fmt: skip + template_type: Optional[str] = field(default=None, metadata={"description": "The type of prompt template."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptFlowNodeSourceConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_flow_node_source_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "inline": S("inline") >> Bend(AwsBedrockPromptFlowNodeInlineConfiguration.mapping), + "resource": S("resource", "promptArn"), + } + inline: Optional[AwsBedrockPromptFlowNodeInlineConfiguration] = field(default=None, metadata={"description": "Contains configurations for a prompt that is defined inline"}) # fmt: skip + resource: Optional[str] = field(default=None, metadata={"description": "Contains configurations for a prompt from Prompt management."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockPromptFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_prompt_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "source_configuration": S("sourceConfiguration") >> Bend(AwsBedrockPromptFlowNodeSourceConfiguration.mapping) + } + source_configuration: Optional[AwsBedrockPromptFlowNodeSourceConfiguration] = field(default=None, metadata={"description": "Specifies whether the prompt is from Prompt management or defined inline."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRetrievalFlowNodeServiceConfiguration: + kind: ClassVar[str] = "aws_bedrock_retrieval_flow_node_service_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"s3": S("s3", "bucketName")} + s3: Optional[str] = field(default=None, metadata={"description": "Contains configurations for the Amazon S3 location from which to retrieve data to return as the output from the node."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockRetrievalFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_retrieval_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "service_configuration": S("serviceConfiguration") + >> Bend(AwsBedrockRetrievalFlowNodeServiceConfiguration.mapping) + } + service_configuration: Optional[AwsBedrockRetrievalFlowNodeServiceConfiguration] = field(default=None, metadata={"description": "Contains configurations for the service to use for retrieving data to return as the output from the node."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockStorageFlowNodeServiceConfiguration: + kind: ClassVar[str] = "aws_bedrock_storage_flow_node_service_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"s3": S("s3", "bucketName")} + s3: Optional[str] = field(default=None, metadata={"description": "Contains configurations for the Amazon S3 location in which to store the input into the node."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockStorageFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_storage_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "service_configuration": S("serviceConfiguration") + >> Bend(AwsBedrockStorageFlowNodeServiceConfiguration.mapping) + } + service_configuration: Optional[AwsBedrockStorageFlowNodeServiceConfiguration] = field(default=None, metadata={"description": "Contains configurations for the service to use for storing the input into the node."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowNodeConfiguration: + kind: ClassVar[str] = "aws_bedrock_flow_node_configuration" + mapping: ClassVar[Dict[str, Bender]] = { + "agent": S("agent", "agentAliasArn"), + "condition": S("condition") >> Bend(AwsBedrockConditionFlowNodeConfiguration.mapping), + "knowledge_base": S("knowledgeBase") >> Bend(AwsBedrockKnowledgeBaseFlowNodeConfiguration.mapping), + "lambda_function": S("lambdaFunction", "lambdaArn"), + "lex": S("lex") >> Bend(AwsBedrockLexFlowNodeConfiguration.mapping), + "prompt": S("prompt") >> Bend(AwsBedrockPromptFlowNodeConfiguration.mapping), + "retrieval": S("retrieval") >> Bend(AwsBedrockRetrievalFlowNodeConfiguration.mapping), + "storage": S("storage") >> Bend(AwsBedrockStorageFlowNodeConfiguration.mapping), + } + agent: Optional[str] = field(default=None, metadata={"description": "Contains configurations for an agent node in your flow. Invokes an alias of an agent and returns the response."}) # fmt: skip + condition: Optional[AwsBedrockConditionFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a Condition node in your flow. Defines conditions that lead to different branches of the flow."}) # fmt: skip + knowledge_base: Optional[AwsBedrockKnowledgeBaseFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a knowledge base node in your flow. Queries a knowledge base and returns the retrieved results or generated response."}) # fmt: skip + lambda_function: Optional[str] = field(default=None, metadata={"description": "Contains configurations for a Lambda function node in your flow. Invokes an Lambda function."}) # fmt: skip + lex: Optional[AwsBedrockLexFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a Lex node in your flow. Invokes an Amazon Lex bot to identify the intent of the input and return the intent as the output."}) # fmt: skip + prompt: Optional[AwsBedrockPromptFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a prompt node in your flow. Runs a prompt and generates the model response as the output. You can use a prompt from Prompt management or you can configure one in this node."}) # fmt: skip + retrieval: Optional[AwsBedrockRetrievalFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a Retrieval node in your flow. Retrieves data from an Amazon S3 location and returns it as the output."}) # fmt: skip + storage: Optional[AwsBedrockStorageFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for a Storage node in your flow. Stores an input in an Amazon S3 location."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowNodeInput: + kind: ClassVar[str] = "aws_bedrock_flow_node_input" + mapping: ClassVar[Dict[str, Bender]] = {"expression": S("expression"), "name": S("name"), "type": S("type")} + expression: Optional[str] = field(default=None, metadata={"description": "An expression that formats the input for the node. For an explanation of how to create expressions, see Expressions in Prompt flows in Amazon Bedrock."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "A name for the input that you can reference."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "The data type of the input. If the input doesn't match this type at runtime, a validation error will be thrown."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowNodeOutput: + kind: ClassVar[str] = "aws_bedrock_flow_node_output" + mapping: ClassVar[Dict[str, Bender]] = {"name": S("name"), "type": S("type")} + name: Optional[str] = field(default=None, metadata={"description": "A name for the output that you can reference."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "The data type of the output. If the output doesn't match this type at runtime, a validation error will be thrown."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowNode: + kind: ClassVar[str] = "aws_bedrock_flow_node" + mapping: ClassVar[Dict[str, Bender]] = { + "configuration": S("configuration") >> Bend(AwsBedrockFlowNodeConfiguration.mapping), + "inputs": S("inputs", default=[]) >> ForallBend(AwsBedrockFlowNodeInput.mapping), + "name": S("name"), + "outputs": S("outputs", default=[]) >> ForallBend(AwsBedrockFlowNodeOutput.mapping), + "type": S("type"), + } + configuration: Optional[AwsBedrockFlowNodeConfiguration] = field(default=None, metadata={"description": "Contains configurations for the node."}) # fmt: skip + inputs: Optional[List[AwsBedrockFlowNodeInput]] = field(factory=list, metadata={"description": "An array of objects, each of which contains information about an input into the node."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "A name for the node."}) # fmt: skip + outputs: Optional[List[AwsBedrockFlowNodeOutput]] = field(factory=list, metadata={"description": "A list of objects, each of which contains information about an output from the node."}) # fmt: skip + type: Optional[str] = field(default=None, metadata={"description": "The type of node. This value must match the name of the key that you provide in the configuration you provide in the FlowNodeConfiguration field."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowDefinition: + kind: ClassVar[str] = "aws_bedrock_flow_definition" + mapping: ClassVar[Dict[str, Bender]] = { + "connections": S("connections", default=[]) >> ForallBend(AwsBedrockFlowConnection.mapping), + "nodes": S("nodes", default=[]) >> ForallBend(AwsBedrockFlowNode.mapping), + } + connections: Optional[List[AwsBedrockFlowConnection]] = field(factory=list, metadata={"description": "An array of connection definitions in the flow."}) # fmt: skip + nodes: Optional[List[AwsBedrockFlowNode]] = field(factory=list, metadata={"description": "An array of node definitions in the flow."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockFlowValidation: + kind: ClassVar[str] = "aws_bedrock_flow_validation" + mapping: ClassVar[Dict[str, Bender]] = {"message": S("message"), "severity": S("severity")} + message: Optional[str] = field(default=None, metadata={"description": "A message describing the validation error."}) # fmt: skip + severity: Optional[str] = field(default=None, metadata={"description": "The severity of the issue described in the message."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsBedrockAgentFlow(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent_flow" + kind_display: ClassVar[str] = "AWS Bedrock Agent Flow" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent Flow outlines the logical sequence of interactions between the agent and the model, " + "defining the steps to be followed in executing tasks or answering queries." + ) + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "ai"} + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/prompt-flows/{id}"} # fmt: skip + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsIamRole.kind]}, + "successors": { + "default": [ + "aws_bedrock_agent_flow_version", + AwsKmsKey.kind, + AwsS3Bucket.kind, + AwsLambdaFunction.kind, + ] + }, + } + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( + "bedrock-agent", "list-flows", "flowSummaries", expected_errors=["AccessDeniedException"] + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("id"), + "name": S("name"), + "ctime": S("createdAt"), + "mtime": S("updatedAt"), + "arn": S("arn"), + "created_at": S("createdAt"), + "customer_encryption_key_arn": S("customerEncryptionKeyArn"), + "definition": S("definition") >> Bend(AwsBedrockFlowDefinition.mapping), + "description": S("description"), + "execution_role_arn": S("executionRoleArn"), + "status": S("status"), + "updated_at": S("updatedAt"), + "validations": S("validations", default=[]) >> ForallBend(AwsBedrockFlowValidation.mapping), + "version": S("version"), + } + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the flow was created."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the KMS key that the flow is encrypted with."}) # fmt: skip + definition: Optional[AwsBedrockFlowDefinition] = field(default=None, metadata={"description": "The definition of the nodes and connections between the nodes in the flow."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the flow."}) # fmt: skip + execution_role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the service role with permissions to create a flow. For more information, see Create a service row for flows in the Amazon Bedrock User Guide."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the flow. The following statuses are possible: NotPrepared – The flow has been created or updated, but hasn't been prepared. If you just created the flow, you can't test it. If you updated the flow, the DRAFT version won't contain the latest changes for testing. Send a PrepareFlow request to package the latest changes into the DRAFT version. Preparing – The flow is being prepared so that the DRAFT version contains the latest changes for testing. Prepared – The flow is prepared and the DRAFT version contains the latest changes for testing. Failed – The last API operation that you invoked on the flow failed. Send a GetFlow request and check the error message in the validations field."}) # fmt: skip + updated_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the flow was last updated."}) # fmt: skip + validations: Optional[List[AwsBedrockFlowValidation]] = field(factory=list, metadata={"description": "A list of validation error messages related to the last failed operation on the flow."}) # fmt: skip + version: Optional[str] = field(default=None, metadata={"description": "The version of the flow for which information was retrieved."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.execution_role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + if (definition := self.definition) and (nodes := definition.nodes): + for node in nodes: + if node_config := node.configuration: + if lambda_arn := node_config.lambda_function: + builder.add_edge(self, clazz=AwsLambdaFunction, arn=lambda_arn) + if retrieval_config := node_config.retrieval: + if retrieval_s3_config := retrieval_config.service_configuration: + if bucket_name := retrieval_s3_config.s3: + builder.add_edge(self, clazz=AwsS3Bucket, name=bucket_name) + if storage_config := node_config.storage: + if storage_s3_config := storage_config.service_configuration: + if bucket_name := storage_s3_config.s3: + builder.add_edge(self, clazz=AwsS3Bucket, name=bucket_name) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", + action="delete-flow", + result_name=None, + flowIdentifier=self.id, + ) + return True + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-flow")] + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [cls.api_spec, AwsApiSpec("bedrock-agent", "get-flow")] + + @classmethod + def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def add_tags(flow: AwsResource) -> None: + tags = builder.client.list( + "bedrock-agent", + "list-tags-for-resource", + "tags", + expected_errors=["ResourceNotFoundException"], + resourceArn=flow.arn, + ) + if tags: + flow.tags.update(tags[0]) + + def collect_flow_versions(flow: AwsBedrockAgentFlow) -> None: + if not flow.version or flow.version == "DRAFT": + return + for result in builder.client.list( + "bedrock-agent", + "get-flow-version", + flowIdentifier=flow.id, + flowVersion=flow.version, + ): + if instance := AwsBedrockAgentFlowVersion.from_api(result, builder): + builder.add_node(instance, js) + builder.submit_work("bedrock-agent", add_tags, instance) + + for js in json: + for result in builder.client.list( + "bedrock-agent", + "get-flow", + flowIdentifier=js["id"], + ): + if instance := AwsBedrockAgentFlow.from_api(result, builder): + if not instance.version: + instance.version = js["version"] + builder.add_node(instance, js) + builder.submit_work("bedrock-agent", add_tags, instance) + builder.submit_work("bedrock-agent", collect_flow_versions, instance) + + +@define(eq=False, slots=False) +class AwsBedrockAgentFlowVersion(BedrockTaggable, AwsResource): + kind: ClassVar[str] = "aws_bedrock_agent_flow_version" + kind_display: ClassVar[str] = "AWS Bedrock Agent Flow Version" + kind_description: ClassVar[str] = ( + "AWS Bedrock Agent Flow Version tracks the version history of agent workflows, ensuring compatibility and stability " + "as workflows evolve and improve over time." + ) + kind_service: ClassVar[Optional[str]] = "bedrock-agent" + metadata: ClassVar[Dict[str, Any]] = {"icon": "version", "group": "ai"} + aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/bedrock/home?region={region_id}#/prompt-flows/{id}/versions/{version}"} # fmt: skip + reference_kinds: ClassVar[ModelReference] = { + "predecessors": {"default": [AwsIamRole.kind]}, + "successors": {"default": [AwsKmsKey.kind]}, + } + # Collected via AwsBedrockAgentFlow() + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("id"), + "name": S("name"), + "ctime": S("createdAt"), + "arn": S("arn"), + "created_at": S("createdAt"), + "customer_encryption_key_arn": S("customerEncryptionKeyArn"), + "definition": S("definition") >> Bend(AwsBedrockFlowDefinition.mapping), + "description": S("description"), + "execution_role_arn": S("executionRoleArn"), + "status": S("status"), + "version": S("version"), + } + created_at: Optional[datetime] = field(default=None, metadata={"description": "The time at which the flow was created."}) # fmt: skip + customer_encryption_key_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the KMS key that the version of the flow is encrypted with."}) # fmt: skip + definition: Optional[AwsBedrockFlowDefinition] = field(default=None, metadata={"description": "The definition of the nodes and connections between nodes in the flow."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The description of the flow."}) # fmt: skip + execution_role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the service role with permissions to create a flow. For more information, see Create a service role for flows in Amazon Bedrock in the Amazon Bedrock User Guide."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the flow."}) # fmt: skip + version: Optional[str] = field(default=None, metadata={"description": "The version of the flow for which information was retrieved."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if role_arn := self.execution_role_arn: + builder.add_edge(self, reverse=True, clazz=AwsIamRole, arn=role_arn) + if encryption_key_arn := self.customer_encryption_key_arn: + builder.add_edge(self, clazz=AwsKmsKey, arn=encryption_key_arn) + + def delete_resource(self, client: AwsClient, graph: Graph) -> bool: + client.call( + aws_service="bedrock-agent", + action="delete-flow-version", + result_name=None, + flowIdentifier=self.id, + flowVersion=self.version, + ) + return True + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return super().called_collect_apis() + [AwsApiSpec("bedrock-agent", "get-flow-version")] + + @classmethod + def called_mutator_apis(cls) -> List[AwsApiSpec]: + return super().called_mutator_apis() + [AwsApiSpec("bedrock-agent", "delete-flow-version")] + + @classmethod + def service_name(cls) -> str: + return "bedrock-agent" + + +resources: List[Type[AwsResource]] = [ + AwsBedrockFoundationModel, + AwsBedrockCustomModel, + AwsBedrockProvisionedModelThroughput, + AwsBedrockGuardrail, + AwsBedrockModelCustomizationJob, + AwsBedrockEvaluationJob, + AwsBedrockAgent, + AwsBedrockAgentVersion, + AwsBedrockAgentKnowledgeBase, + AwsBedrockAgentPrompt, + AwsBedrockAgentFlow, + AwsBedrockAgentFlowVersion, +] diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index 7c1146db3d..51f7e78b37 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -437,7 +437,7 @@ def called_mutator_apis(cls) -> List[AwsApiSpec]: def name_from_path(path_or_uri: str) -> str: # https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html # Accessing a bucket using S3:// - if path_or_uri.startswith("s3://") or path_or_uri.startswith("S3://"): + if path_or_uri.lower().startswith("s3://"): return path_or_uri.split("/")[2] # Path-style access if path_or_uri.startswith("https://s3"): diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index 2d668b14a8..e784201685 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -34,8 +34,8 @@ def count_kind(clazz: Type[AwsResource]) -> int: # make sure all threads have been joined assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges - assert count_kind(AwsResource) == 248 - assert len(account_collector.graph.edges) == 564 + assert count_kind(AwsResource) == 261 + assert len(account_collector.graph.edges) == 579 assert len(account_collector.graph.deferred_edges) == 2 for node in account_collector.graph.nodes: if isinstance(node, AwsRegion): diff --git a/plugins/aws/test/resources/__init__.py b/plugins/aws/test/resources/__init__.py index f4b8d58dfc..47fe04c6ea 100644 --- a/plugins/aws/test/resources/__init__.py +++ b/plugins/aws/test/resources/__init__.py @@ -171,6 +171,7 @@ def check_single_node(node: AwsResource) -> None: def round_trip_for( cls: Type[AwsResourceType], *ignore_props: str, + ignore_checking_props: bool = False, region_name: Optional[str] = None, collect_also: Optional[List[Type[AwsResource]]] = None, ) -> Tuple[AwsResourceType, GraphBuilder]: @@ -181,5 +182,6 @@ def round_trip_for( node.connect_in_graph(builder, data.get("source", {})) check_single_node(node) first = next(iter(builder.resources_of(cls))) - all_props_set(first, set(ignore_props)) + if not ignore_checking_props: + all_props_set(first, set(ignore_props)) return first, builder diff --git a/plugins/aws/test/resources/bedrock_test.py b/plugins/aws/test/resources/bedrock_test.py new file mode 100644 index 0000000000..c261e166af --- /dev/null +++ b/plugins/aws/test/resources/bedrock_test.py @@ -0,0 +1,48 @@ +from fix_plugin_aws.resource.bedrock import ( + AwsBedrockCustomModel, + AwsBedrockProvisionedModelThroughput, + AwsBedrockGuardrail, + AwsBedrockModelCustomizationJob, + AwsBedrockEvaluationJob, + AwsBedrockAgent, + AwsBedrockAgentPrompt, + AwsBedrockAgentFlow, + AwsBedrockFoundationModel, +) +from test.resources import round_trip_for + + +def test_bedrock_custom_models() -> None: + round_trip_for(AwsBedrockCustomModel, ignore_checking_props=True) + + +def test_bedrock_provisioned_model_throughputs() -> None: + round_trip_for(AwsBedrockProvisionedModelThroughput, ignore_checking_props=True) + + +def test_bedrock_guardrails() -> None: + round_trip_for(AwsBedrockGuardrail, ignore_checking_props=True) + + +def test_bedrock_model_customization_jobs() -> None: + round_trip_for(AwsBedrockModelCustomizationJob, ignore_checking_props=True) + + +def test_bedrock_evaluation_jobs() -> None: + round_trip_for(AwsBedrockEvaluationJob, ignore_checking_props=True) + + +def test_bedrock_agents() -> None: + round_trip_for(AwsBedrockAgent, ignore_checking_props=True) + + +def test_bedrock_agent_prompts() -> None: + round_trip_for(AwsBedrockAgentPrompt, ignore_checking_props=True) + + +def test_bedrock_agent_flows() -> None: + round_trip_for(AwsBedrockAgentFlow, ignore_checking_props=True) + + +def test_bedrock_foundation_model() -> None: + round_trip_for(AwsBedrockFoundationModel, ignore_checking_props=True) diff --git a/plugins/aws/test/resources/files/bedrock/get-custom-model__foo.json b/plugins/aws/test/resources/files/bedrock/get-custom-model__foo.json new file mode 100644 index 0000000000..cb49420e3f --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/get-custom-model__foo.json @@ -0,0 +1,12 @@ +{ + "id": "foo", + "name": "foo", + "status": "ACTIVE", + "createdAt": "2024-09-12T10:00:00Z", + "lastModifiedAt": "2024-09-12T12:00:00Z", + "description": "A custom model created for specific use cases", + "trainingData": { + "source": "s3://bucket/training-data" + } + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/get-evaluation-job__foo.json b/plugins/aws/test/resources/files/bedrock/get-evaluation-job__foo.json new file mode 100644 index 0000000000..33aaf28fe8 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/get-evaluation-job__foo.json @@ -0,0 +1,14 @@ +{ + "id": "foo", + "name": "foo", + "status": "IN_PROGRESS", + "modelId": "foo", + "evaluationMetrics": { + "accuracy": 0.95, + "precision": 0.92, + "recall": 0.90 + }, + "startedAt": "2024-09-12T10:30:00Z", + "completedAt": null + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/get-guardrail__foo_foo.json b/plugins/aws/test/resources/files/bedrock/get-guardrail__foo_foo.json new file mode 100644 index 0000000000..b6df817b8a --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/get-guardrail__foo_foo.json @@ -0,0 +1,19 @@ +{ + "id": "foo", + "name": "foo", + "status": "ENABLED", + "description": "A guardrail to enforce responsible AI usage", + "rules": [ + { + "id": "rule1", + "name": "Bias Mitigation", + "action": "MONITOR" + }, + { + "id": "rule2", + "name": "Fairness Check", + "action": "ALERT" + } + ] + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/get-model-customization-job__foo.json b/plugins/aws/test/resources/files/bedrock/get-model-customization-job__foo.json new file mode 100644 index 0000000000..cef5fb2b01 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/get-model-customization-job__foo.json @@ -0,0 +1,12 @@ +{ + "id": "foo", + "name": "foo", + "status": "COMPLETED", + "modelId": "foo", + "startedAt": "2024-09-12T09:00:00Z", + "completedAt": "2024-09-12T12:00:00Z", + "customizationData": { + "source": "s3://bucket/customization-data" + } + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-custom-models.json b/plugins/aws/test/resources/files/bedrock/list-custom-models.json new file mode 100644 index 0000000000..e356d6ee17 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-custom-models.json @@ -0,0 +1,13 @@ +{ + "nextToken": "foo", + "modelSummaries": [ + { + "modelArn": "foo", + "modelName": "foo", + "creationTime": "2024-09-13T12:32:05Z", + "baseModelArn": "foo", + "baseModelName": "foo", + "customizationType": "CONTINUED_PRE_TRAINING" + } + ] + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-evaluation-jobs.json b/plugins/aws/test/resources/files/bedrock/list-evaluation-jobs.json new file mode 100644 index 0000000000..0b356be024 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-evaluation-jobs.json @@ -0,0 +1,22 @@ +{ + "nextToken": "foo", + "jobSummaries": [ + { + "jobArn": "foo", + "jobName": "foo", + "status": "Completed", + "creationTime": "2024-09-13T12:32:05Z", + "jobType": "Automated", + "evaluationTaskTypes": [ + "Classification", + "Classification", + "Classification" + ], + "modelIdentifiers": [ + "foo", + "foo", + "foo" + ] + } + ] + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-foundation-models.json b/plugins/aws/test/resources/files/bedrock/list-foundation-models.json new file mode 100644 index 0000000000..72f173c548 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-foundation-models.json @@ -0,0 +1,34 @@ +{ + "modelSummaries": [ + { + "modelArn": "foo", + "modelId": "foo", + "modelName": "foo", + "providerName": "foo", + "inputModalities": [ + "IMAGE", + "IMAGE", + "IMAGE" + ], + "outputModalities": [ + "IMAGE", + "IMAGE", + "IMAGE" + ], + "responseStreamingSupported": true, + "customizationsSupported": [ + "CONTINUED_PRE_TRAINING", + "CONTINUED_PRE_TRAINING", + "CONTINUED_PRE_TRAINING" + ], + "inferenceTypesSupported": [ + "PROVISIONED", + "PROVISIONED", + "PROVISIONED" + ], + "modelLifecycle": { + "status": "LEGACY" + } + } + ] + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-guardrails.json b/plugins/aws/test/resources/files/bedrock/list-guardrails.json new file mode 100644 index 0000000000..d2d07be90e --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-guardrails.json @@ -0,0 +1,15 @@ +{ + "guardrails": [ + { + "id": "foo", + "arn": "foo", + "status": "UPDATING", + "name": "foo", + "description": "foo", + "version": "foo", + "createdAt": "2024-09-13T12:32:05Z", + "updatedAt": "2024-09-13T12:32:05Z" + } + ], + "nextToken": "foo" + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-model-customization-jobs.json b/plugins/aws/test/resources/files/bedrock/list-model-customization-jobs.json new file mode 100644 index 0000000000..5e9edff154 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-model-customization-jobs.json @@ -0,0 +1,17 @@ +{ + "nextToken": "foo", + "modelCustomizationJobSummaries": [ + { + "jobArn": "foo", + "baseModelArn": "foo", + "jobName": "foo", + "status": "Completed", + "lastModifiedTime": "2024-09-13T12:32:05Z", + "creationTime": "2024-09-13T12:32:05Z", + "endTime": "2024-09-13T12:32:05Z", + "customModelArn": "foo", + "customModelName": "foo", + "customizationType": "CONTINUED_PRE_TRAINING" + } + ] + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock/list-provisioned-model-throughputs.json b/plugins/aws/test/resources/files/bedrock/list-provisioned-model-throughputs.json new file mode 100644 index 0000000000..c96473490b --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock/list-provisioned-model-throughputs.json @@ -0,0 +1,33 @@ +{ + "nextToken": "foo", + "provisionedModelSummaries": [ + { + "provisionedModelName": "foo", + "provisionedModelArn": "foo", + "modelArn": "foo", + "desiredModelArn": "foo", + "foundationModelArn": "foo", + "modelUnits": 123, + "desiredModelUnits": 123, + "status": "InService", + "commitmentDuration": "SixMonths", + "commitmentExpirationTime": "2024-09-13T12:32:05Z", + "creationTime": "2024-09-13T12:32:05Z", + "lastModifiedTime": "2024-09-13T12:32:05Z" + }, + { + "provisionedModelName": "foo", + "provisionedModelArn": "foo", + "modelArn": "foo", + "desiredModelArn": "foo", + "foundationModelArn": "foo", + "modelUnits": 123, + "desiredModelUnits": 123, + "status": "InService", + "commitmentDuration": "SixMonths", + "commitmentExpirationTime": "2024-09-13T12:32:05Z", + "creationTime": "2024-09-13T12:32:05Z", + "lastModifiedTime": "2024-09-13T12:32:05Z" + } + ] + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-agent-version__None_1_0.json b/plugins/aws/test/resources/files/bedrock_agent/get-agent-version__None_1_0.json new file mode 100644 index 0000000000..dc8a0df49a --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-agent-version__None_1_0.json @@ -0,0 +1,10 @@ +{ + "agentId": "foo", + "agentName": "foo", + "version": "1.0", + "status": "ACTIVE", + "releaseNotes": "Initial release with basic features.", + "createdAt": "2024-09-12T08:00:00Z", + "lastModifiedAt": "2024-09-12T09:00:00Z" + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-agent__foo.json b/plugins/aws/test/resources/files/bedrock_agent/get-agent__foo.json new file mode 100644 index 0000000000..ba5f6220d0 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-agent__foo.json @@ -0,0 +1,12 @@ +{ + "agent": { + "id": "foo", + "name": "foo", + "agentVersion": "1.0", + "status": "ACTIVE", + "createdAt": "2024-09-12T11:00:00Z", + "lastModifiedAt": "2024-09-12T13:00:00Z", + "knowledgeBaseId": "foo" + } + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-flow-version__foo_1_0.json b/plugins/aws/test/resources/files/bedrock_agent/get-flow-version__foo_1_0.json new file mode 100644 index 0000000000..172bac6237 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-flow-version__foo_1_0.json @@ -0,0 +1,27 @@ +{ + "id": "foo", + "name": "foo", + "version": "1.0", + "status": "ACTIVE", + "steps": [ + { + "id": "step1", + "name": "Initial Request", + "action": "Gather Input" + }, + { + "id": "step2", + "name": "Process Data", + "action": "Run Model" + }, + { + "id": "step3", + "name": "Return Result", + "action": "Respond to User" + } + ], + "createdAt": "2024-09-12T09:00:00Z", + "lastModifiedAt": "2024-09-12T10:30:00Z", + "description": "Flow version 1.0 for processing user requests." + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-flow__foo.json b/plugins/aws/test/resources/files/bedrock_agent/get-flow__foo.json new file mode 100644 index 0000000000..f47e50b26e --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-flow__foo.json @@ -0,0 +1,26 @@ +{ + "id": "foo", + "name": "foo", + "status": "ACTIVE", + "version": "1.0", + "steps": [ + { + "id": "step1", + "name": "Initial Request", + "action": "Gather Input" + }, + { + "id": "step2", + "name": "Process Data", + "action": "Run Model" + }, + { + "id": "step3", + "name": "Return Result", + "action": "Respond to User" + } + ], + "createdAt": "2024-09-12T09:00:00Z", + "lastModifiedAt": "2024-09-12T10:30:00Z" + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-knowledge-base__foo.json b/plugins/aws/test/resources/files/bedrock_agent/get-knowledge-base__foo.json new file mode 100644 index 0000000000..fa561e8029 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-knowledge-base__foo.json @@ -0,0 +1,7 @@ +{ + "description": "foo", + "knowledgeBaseId": "foo", + "name": "foo", + "status": "ACTIVE", + "updatedAt": "2024-09-17T12:11:47Z" +} \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/get-prompt__foo_foo.json b/plugins/aws/test/resources/files/bedrock_agent/get-prompt__foo_foo.json new file mode 100644 index 0000000000..ec88a09209 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/get-prompt__foo_foo.json @@ -0,0 +1,9 @@ +{ + "id": "foo", + "name": "foo", + "text": "Generate a response based on the following input.", + "createdAt": "2024-09-12T08:00:00Z", + "lastModifiedAt": "2024-09-12T10:00:00Z", + "agentId": "foo" +} + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/list-agents.json b/plugins/aws/test/resources/files/bedrock_agent/list-agents.json new file mode 100644 index 0000000000..22e673f74f --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/list-agents.json @@ -0,0 +1,17 @@ +{ + "agentSummaries": [ + { + "agentId": "foo", + "agentName": "foo", + "agentStatus": "PREPARING", + "description": "foo", + "guardrailConfiguration": { + "guardrailIdentifier": "foo", + "guardrailVersion": "foo" + }, + "latestAgentVersion": "foo", + "updatedAt": "2024-09-13T12:32:05Z" + } + ], + "nextToken": "foo" + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/list-flows.json b/plugins/aws/test/resources/files/bedrock_agent/list-flows.json new file mode 100644 index 0000000000..643808e881 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/list-flows.json @@ -0,0 +1,15 @@ +{ + "flowSummaries": [ + { + "arn": "foo", + "createdAt": "2024-09-13T12:32:05Z", + "description": "foo", + "id": "foo", + "name": "foo", + "status": "Prepared", + "updatedAt": "2024-09-13T12:32:05Z", + "version": "foo" + } + ], + "nextToken": "foo" + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases.json b/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases.json new file mode 100644 index 0000000000..cc40da7b1b --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases.json @@ -0,0 +1,12 @@ +{ + "knowledgeBaseSummaries": [ + { + "description": "foo", + "knowledgeBaseId": "foo", + "name": "foo", + "status": "ACTIVE", + "updatedAt": "2024-09-17T12:11:47Z" + } + ], + "nextToken": "foo" + } \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases__None_1_0.json b/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases__None_1_0.json new file mode 100644 index 0000000000..e13a043289 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/list-knowledge-bases__None_1_0.json @@ -0,0 +1,12 @@ +{ + "agentKnowledgeBaseSummaries": [ + { + "id": "foo", + "name": "foo", + "description": "A knowledge base for general queries.", + "createdAt": "2024-09-12T07:00:00Z", + "lastModifiedAt": "2024-09-12T08:00:00Z" + } + ] + } + \ No newline at end of file diff --git a/plugins/aws/test/resources/files/bedrock_agent/list-prompts.json b/plugins/aws/test/resources/files/bedrock_agent/list-prompts.json new file mode 100644 index 0000000000..5e577d4f17 --- /dev/null +++ b/plugins/aws/test/resources/files/bedrock_agent/list-prompts.json @@ -0,0 +1,14 @@ +{ + "nextToken": "foo", + "promptSummaries": [ + { + "arn": "foo", + "createdAt": "2024-09-13T12:32:05Z", + "description": "foo", + "id": "foo", + "name": "foo", + "updatedAt": "2024-09-13T12:32:05Z", + "version": "foo" + } + ] + } \ No newline at end of file diff --git a/plugins/aws/tools/aws_model_gen.py b/plugins/aws/tools/aws_model_gen.py index 2e4d6c59a8..90ba7e827e 100644 --- a/plugins/aws/tools/aws_model_gen.py +++ b/plugins/aws/tools/aws_model_gen.py @@ -944,20 +944,20 @@ def default_imports() -> str: # AwsFixModel("get-logging-configuration", "LoggingConfigurations", "LoggingConfiguration", prefix="Waf") ], "qbusiness": [ - AwsFixModel( - api_action="list-applications", - result_property="applications", - result_shape="Applications", - prefix="QBusiness", - ), + # AwsFixModel( + # api_action="list-applications", + # result_property="applications", + # result_shape="Applications", + # prefix="QBusiness", + # ), ], "qapps": [ - AwsFixModel( - api_action="list-qapps", - result_property="apps", - result_shape="ListQAppsOutput", - prefix="QApps", - ), + # AwsFixModel( + # api_action="list-qapps", + # result_property="apps", + # result_shape="ListQAppsOutput", + # prefix="QApps", + # ), ], "backup": [ # AwsFixModel( @@ -967,12 +967,28 @@ def default_imports() -> str: # prefix="Backup", # ), ], + "bedrock": [ + # AwsFixModel( + # api_action="list-foundation-models", + # result_property="modelSummaries", + # result_shape="ListFoundationModelsResponse", + # prefix="Bedrock", + # ) + ], + "bedrock-agent": [ + # AwsFixModel( + # api_action="get-agent", + # result_property="Agents", + # result_shape=None, + # prefix="Bedrock", + # ) + ], } if __name__ == "__main__": """print some test data""" - print(json.dumps(create_test_response("dynamodb", "describe-continuous-backups"), indent=2)) + print(json.dumps(create_test_response("bedrock-agent", "get-knowledge-base"), indent=2)) """print the class models""" # print(default_imports()) diff --git a/plugins/azure/fix_plugin_azure/resource/machinelearning.py b/plugins/azure/fix_plugin_azure/resource/machinelearning.py index 74eb7e04e3..38481c6831 100644 --- a/plugins/azure/fix_plugin_azure/resource/machinelearning.py +++ b/plugins/azure/fix_plugin_azure/resource/machinelearning.py @@ -25,7 +25,7 @@ from fix_plugin_azure.resource.network import AzureNetworkSubnet, AzureNetworkVirtualNetwork from fix_plugin_azure.resource.storage import AzureStorageAccount from fix_plugin_azure.resource.web import AzureWebApp -from fixlib.baseresources import BaseInstanceType, ModelReference +from fixlib.baseresources import BaseInstanceType, ModelReference, BaseAIJob, BaseAIModel from fixlib.graph import BySearchCriteria from fixlib.json_bender import Bender, S, ForallBend, Bend, K from fixlib.types import Json @@ -1344,7 +1344,7 @@ class AzureJobService: @define(eq=False, slots=False) -class AzureMachineLearningJob(MicrosoftResource, AzureProxyResource): +class AzureMachineLearningJob(BaseAIJob, MicrosoftResource, AzureProxyResource): kind: ClassVar[str] = "azure_machine_learning_job" kind_display: ClassVar[str] = "Azure Machine Learning Job" kind_service: ClassVar[Optional[str]] = service_name @@ -1513,7 +1513,7 @@ class AzureStatusMessage: @define(eq=False, slots=False) -class AzureMachineLearningLabelingJob(MicrosoftResource): +class AzureMachineLearningLabelingJob(BaseAIJob, MicrosoftResource): kind: ClassVar[str] = "azure_machine_learning_labeling_job" kind_display: ClassVar[str] = "Azure Machine Learning Labeling Job" kind_service: ClassVar[Optional[str]] = service_name @@ -1581,7 +1581,9 @@ class AzureMachineLearningModelContainerBase(AzureProxyResource): @define(eq=False, slots=False) -class AzureMachineLearningWorkspaceModelContainer(AzureMachineLearningModelContainerBase, MicrosoftResource): +class AzureMachineLearningWorkspaceModelContainer( + BaseAIModel, AzureMachineLearningModelContainerBase, MicrosoftResource +): # Defined to split registry and workspace resource kind: ClassVar[str] = "azure_machine_learning_workspace_model_container" @@ -1618,7 +1620,9 @@ def collect_versions() -> None: @define(eq=False, slots=False) -class AzureMachineLearningRegistryModelContainer(AzureMachineLearningModelContainerBase, MicrosoftResource): +class AzureMachineLearningRegistryModelContainer( + BaseAIModel, AzureMachineLearningModelContainerBase, MicrosoftResource +): # Defined to split registry and workspace resource kind: ClassVar[str] = "azure_machine_learning_registry_model_container" diff --git a/requirements-all.txt b/requirements-all.txt index 8e79828f9f..0d908eae80 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,8 +1,9 @@ aiodns==3.2.0 aiofiles==24.1.0 -aiohttp[speedups]==3.9.5 +aiohappyeyeballs==2.4.0 +aiohttp[speedups]==3.10.5 aiohttp-jinja2==1.6 -aiohttp-swagger3==0.8.0 +aiohttp-swagger3==0.9.0 aiosignal==1.3.1 aiostream==0.6.2 appdirs==1.4.4 @@ -12,7 +13,7 @@ astroid==3.2.4 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.30.2 +azure-core==1.31.0 azure-identity==1.17.1 azure-mgmt-core==1.4.0 azure-mgmt-resource==23.1.1 @@ -20,15 +21,15 @@ backoff==2.2.1 backports-tarfile==1.2.0 bcrypt==4.2.0 black==24.8.0 -boto3==1.35.10 -botocore==1.35.10 +boto3==1.35.22 +botocore==1.35.22 brotli==1.1.0 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -cattrs==24.1.0 +cattrs==24.1.1 cerberus==1.3.5 certifi==2024.8.30 -cffi==1.17.0 +cffi==1.17.1 chardet==5.2.0 charset-normalizer==3.3.2 cheroot==10.0.1 @@ -36,7 +37,7 @@ cherrypy==18.10.0 click==8.1.7 colorama==0.4.6 coverage[toml]==7.6.1 -cryptography==43.0.0 +cryptography==43.0.1 deepdiff==8.0.1 defusedxml==0.7.1 deprecated==1.2.14 @@ -44,7 +45,7 @@ detect-secrets==1.5.0 dill==0.3.8 distlib==0.3.8 fastjsonschema==2.19.1 -filelock==3.15.4 +filelock==3.16.1 fixcompliance==0.4.34 fixdatalink[extra]==2.0.2 fixinventoryclient==2.0.1 @@ -55,18 +56,19 @@ flexparser==0.3.1 frozendict==2.4.4 frozenlist==1.4.1 google-api-core==2.19.2 -google-api-python-client==2.143.0 +google-api-python-client==2.146.0 google-auth==2.34.0 google-auth-httplib2==0.2.0 google-cloud-core==2.4.1 google-cloud-storage==2.18.2 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 googleapis-common-protos==1.65.0 +hcloud==2.2.0 httplib2==0.22.0 -hypothesis==6.111.2 -idna==3.8 -importlib-metadata==8.4.0 +hypothesis==6.112.1 +idna==3.10 +importlib-metadata==8.5.0 iniconfig==2.0.0 isodate==0.6.1 isort==5.13.2 @@ -84,20 +86,20 @@ markupsafe==2.1.5 mccabe==0.7.0 mdurl==0.1.2 monotonic==1.6 -more-itertools==10.4.0 -msal==1.30.0 +more-itertools==10.5.0 +msal==1.31.0 msal-extensions==1.2.0 -multidict==6.0.5 +multidict==6.1.0 mypy==1.11.2 mypy-extensions==1.0.0 networkx==3.3 -numpy==2.1.0 +numpy==2.1.1 oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 packaging==24.1 -paramiko==3.4.1 +paramiko==3.5.0 parsy==2.1 pathspec==0.12.1 pep8-naming==0.14.1 @@ -105,20 +107,20 @@ pint==0.24.3 pip==24.2 pip-tools==7.4.1 plantuml==0.3.0 -platformdirs==4.2.2 +platformdirs==4.3.6 pluggy==1.5.0 portalocker==2.10.1 portend==3.2.0 -posthog==3.6.0 +posthog==3.6.6 prometheus-client==0.20.0 prompt-toolkit==3.0.47 proto-plus==1.24.0 -protobuf==5.28.0 +protobuf==5.28.2 psutil==6.0.0 psycopg2-binary==2.9.9 pyarrow==17.0.0 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycares==4.4.0 pycodestyle==2.12.1 pycparser==2.22 @@ -131,13 +133,13 @@ pymysql==1.1.1 pynacl==1.5.0 pyopenssl==24.2.1 pyparsing==3.1.4 -pyproject-api==1.7.1 +pyproject-api==1.8.0 pyproject-hooks==1.1.0 -pytest==8.3.2 +pytest==8.3.3 pytest-asyncio==0.24.0 pytest-cov==5.0.0 pytest-runner==6.0.1 -python-arango==8.1.0 +python-arango==8.1.1 python-dateutil==2.9.0.post0 pytz==2024.1 pyvmomi==8.0.3.0.1 @@ -147,30 +149,30 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.8.0 +rich==13.8.1 rsa==4.9 s3transfer==0.10.2 -setuptools==74.0.0 +setuptools==75.1.0 six==1.16.0 -slack-sdk==3.31.0 -snowflake-connector-python==3.12.1 +slack-sdk==3.33.0 +snowflake-connector-python==3.12.2 snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 -sqlalchemy==1.4.53 +sqlalchemy==1.4.54 tempora==5.7.0 tenacity==9.0.0 toml==0.10.2 tomlkit==0.13.2 toolz==0.12.1 -tox==4.18.0 +tox==4.20.0 transitions==0.9.2 typeguard==4.3.0 types-aiofiles==24.1.0.20240626 -types-python-dateutil==2.9.0.20240821 -types-pytz==2024.1.0.20240417 -types-pyyaml==6.0.12.20240808 +types-python-dateutil==2.9.0.20240906 +types-pytz==2024.2.0.20240913 +types-pyyaml==6.0.12.20240917 types-requests==2.31.0.6 -types-setuptools==74.0.0.20240831 +types-setuptools==75.1.0.20240917 types-six==1.16.21.20240513 types-tzlocal==5.1.0.1 types-urllib3==1.26.25.14 @@ -181,11 +183,11 @@ tzlocal==5.2 uritemplate==4.1.1 urllib3==1.26.20 ustache==0.1.5 -virtualenv==20.26.3 +virtualenv==20.26.5 wcwidth==0.2.13 websocket-client==1.8.0 wheel==0.44.0 wrapt==1.16.0 -yarl==1.9.7 +yarl==1.11.1 zc-lockfile==3.0.post1 -zipp==3.20.1 +zipp==3.20.2 diff --git a/requirements-extra.txt b/requirements-extra.txt index 5fd1019a19..bbf4b2bf3d 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -1,8 +1,9 @@ aiodns==3.2.0 aiofiles==24.1.0 -aiohttp[speedups]==3.9.5 +aiohappyeyeballs==2.4.0 +aiohttp[speedups]==3.10.5 aiohttp-jinja2==1.6 -aiohttp-swagger3==0.8.0 +aiohttp-swagger3==0.9.0 aiosignal==1.3.1 aiostream==0.6.2 appdirs==1.4.4 @@ -11,31 +12,31 @@ asn1crypto==1.5.1 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.30.2 +azure-core==1.31.0 azure-identity==1.17.1 azure-mgmt-core==1.4.0 azure-mgmt-resource==23.1.1 backoff==2.2.1 backports-tarfile==1.2.0 bcrypt==4.2.0 -boto3==1.35.10 -botocore==1.35.10 +boto3==1.35.22 +botocore==1.35.22 brotli==1.1.0 cachetools==5.5.0 -cattrs==24.1.0 +cattrs==24.1.1 cerberus==1.3.5 certifi==2024.8.30 -cffi==1.17.0 +cffi==1.17.1 charset-normalizer==3.3.2 cheroot==10.0.1 cherrypy==18.10.0 -cryptography==43.0.0 +cryptography==43.0.1 deepdiff==8.0.1 defusedxml==0.7.1 deprecated==1.2.14 detect-secrets==1.5.0 fastjsonschema==2.19.1 -filelock==3.15.4 +filelock==3.16.1 fixcompliance==0.4.34 fixdatalink[extra]==2.0.2 fixinventoryclient==2.0.1 @@ -45,17 +46,18 @@ flexparser==0.3.1 frozendict==2.4.4 frozenlist==1.4.1 google-api-core==2.19.2 -google-api-python-client==2.143.0 +google-api-python-client==2.146.0 google-auth==2.34.0 google-auth-httplib2==0.2.0 google-cloud-core==2.4.1 google-cloud-storage==2.18.2 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 googleapis-common-protos==1.65.0 +hcloud==2.2.0 httplib2==0.22.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 isodate==0.6.1 jaraco-collections==5.1.0 jaraco-context==6.0.1 @@ -70,34 +72,34 @@ markdown-it-py==3.0.0 markupsafe==2.1.5 mdurl==0.1.2 monotonic==1.6 -more-itertools==10.4.0 -msal==1.30.0 +more-itertools==10.5.0 +msal==1.31.0 msal-extensions==1.2.0 -multidict==6.0.5 +multidict==6.1.0 networkx==3.3 -numpy==2.1.0 +numpy==2.1.1 oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 packaging==24.1 -paramiko==3.4.1 +paramiko==3.5.0 parsy==2.1 pint==0.24.3 plantuml==0.3.0 -platformdirs==4.2.2 +platformdirs==4.3.6 portalocker==2.10.1 portend==3.2.0 -posthog==3.6.0 +posthog==3.6.6 prometheus-client==0.20.0 prompt-toolkit==3.0.47 proto-plus==1.24.0 -protobuf==5.28.0 +protobuf==5.28.2 psutil==6.0.0 psycopg2-binary==2.9.9 pyarrow==17.0.0 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycares==4.4.0 pycparser==2.22 pygithub==2.4.0 @@ -107,7 +109,7 @@ pymysql==1.1.1 pynacl==1.5.0 pyopenssl==24.2.1 pyparsing==3.1.4 -python-arango==8.1.0 +python-arango==8.1.1 python-dateutil==2.9.0.post0 pytz==2024.1 pyvmomi==8.0.3.0.1 @@ -117,16 +119,16 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.8.0 +rich==13.8.1 rsa==4.9 s3transfer==0.10.2 -setuptools==74.0.0 +setuptools==75.1.0 six==1.16.0 -slack-sdk==3.31.0 -snowflake-connector-python==3.12.1 +slack-sdk==3.33.0 +snowflake-connector-python==3.12.2 snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 -sqlalchemy==1.4.53 +sqlalchemy==1.4.54 tempora==5.7.0 tenacity==9.0.0 tomlkit==0.13.2 @@ -143,6 +145,6 @@ ustache==0.1.5 wcwidth==0.2.13 websocket-client==1.8.0 wrapt==1.16.0 -yarl==1.9.7 +yarl==1.11.1 zc-lockfile==3.0.post1 -zipp==3.20.1 +zipp==3.20.2 diff --git a/requirements.txt b/requirements.txt index 7abdac98bb..3336e360b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ aiodns==3.2.0 aiofiles==24.1.0 -aiohttp[speedups]==3.9.5 +aiohappyeyeballs==2.4.0 +aiohttp[speedups]==3.10.5 aiohttp-jinja2==1.6 -aiohttp-swagger3==0.8.0 +aiohttp-swagger3==0.9.0 aiosignal==1.3.1 aiostream==0.6.2 appdirs==1.4.4 @@ -10,25 +11,25 @@ apscheduler==3.10.4 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.30.2 +azure-core==1.31.0 azure-identity==1.17.1 azure-mgmt-core==1.4.0 azure-mgmt-resource==23.1.1 backoff==2.2.1 backports-tarfile==1.2.0 bcrypt==4.2.0 -boto3==1.35.10 -botocore==1.35.10 +boto3==1.35.22 +botocore==1.35.22 brotli==1.1.0 cachetools==5.5.0 -cattrs==24.1.0 +cattrs==24.1.1 cerberus==1.3.5 certifi==2024.8.30 -cffi==1.17.0 +cffi==1.17.1 charset-normalizer==3.3.2 cheroot==10.0.1 cherrypy==18.10.0 -cryptography==43.0.0 +cryptography==43.0.1 deepdiff==8.0.1 defusedxml==0.7.1 deprecated==1.2.14 @@ -43,13 +44,14 @@ flexparser==0.3.1 frozendict==2.4.4 frozenlist==1.4.1 google-api-core==2.19.2 -google-api-python-client==2.143.0 +google-api-python-client==2.146.0 google-auth==2.34.0 google-auth-httplib2==0.2.0 googleapis-common-protos==1.65.0 +hcloud==2.2.0 httplib2==0.22.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 isodate==0.6.1 jaraco-collections==5.1.0 jaraco-context==6.0.1 @@ -64,30 +66,30 @@ markdown-it-py==3.0.0 markupsafe==2.1.5 mdurl==0.1.2 monotonic==1.6 -more-itertools==10.4.0 -msal==1.30.0 +more-itertools==10.5.0 +msal==1.31.0 msal-extensions==1.2.0 -multidict==6.0.5 +multidict==6.1.0 networkx==3.3 oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 packaging==24.1 -paramiko==3.4.1 +paramiko==3.5.0 parsy==2.1 pint==0.24.3 plantuml==0.3.0 portalocker==2.10.1 portend==3.2.0 -posthog==3.6.0 +posthog==3.6.6 prometheus-client==0.20.0 prompt-toolkit==3.0.47 proto-plus==1.24.0 -protobuf==5.28.0 +protobuf==5.28.2 psutil==6.0.0 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycares==4.4.0 pycparser==2.22 pygithub==2.4.0 @@ -95,7 +97,7 @@ pygments==2.18.0 pyjwt[crypto]==2.9.0 pynacl==1.5.0 pyparsing==3.1.4 -python-arango==8.1.0 +python-arango==8.1.1 python-dateutil==2.9.0.post0 pytz==2024.1 pyvmomi==8.0.3.0.1 @@ -105,13 +107,13 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.8.0 +rich==13.8.1 rsa==4.9 s3transfer==0.10.2 -setuptools==74.0.0 +setuptools==75.1.0 six==1.16.0 -slack-sdk==3.31.0 -sqlalchemy==1.4.53 +slack-sdk==3.33.0 +sqlalchemy==1.4.54 tempora==5.7.0 tenacity==9.0.0 toolz==0.12.1 @@ -127,6 +129,6 @@ ustache==0.1.5 wcwidth==0.2.13 websocket-client==1.8.0 wrapt==1.16.0 -yarl==1.9.7 +yarl==1.11.1 zc-lockfile==3.0.post1 -zipp==3.20.1 +zipp==3.20.2