Feature store marketplace operator

.github/workflows/run-unittests-py38-cov-report.yml

@@ -71,7 +71,9 @@ jobs:
       - name: "Install Forecasting dependencies"
         run: |
           pip install -e ".[forecast]"
-
+      - name: "Install featurestore marketplace dependencies"
+        run: |
+          pip install -e ".[feature-store-marketplace]"
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 40
         shell: bash

ads/common/decorator/runtime_dependency.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """
@@ -65,6 +65,7 @@ class OptionalDependency:
     SPARK = "oracle-ads[spark]"
     HUGGINGFACE = "oracle-ads[huggingface]"
     FORECAST = "oracle-ads[forecast]"
+    FEATURE_STORE_MARKETPLACE = "oracle-ads[feature-store-marketplace]"
     PII = "oracle-ads[pii]"
     FEATURE_STORE = "oracle-ads[feature-store]"
     GRAPHVIZ = "oracle-ads[graphviz]"

ads/common/extended_enum.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -70,4 +70,4 @@ def values(cls):
 
     @classmethod
     def keys(cls):
-        return sorted(map(lambda c: c.name, cls))
+        return sorted(map(lambda c: c.name, cls))

ads/common/oci_client.py

@@ -1,18 +1,21 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
 
+import oci.artifacts
 from oci.ai_language import AIServiceLanguageClient
+from oci.artifacts import ArtifactsClient
 from oci.data_catalog import DataCatalogClient
 from oci.data_flow import DataFlowClient
 from oci.data_labeling_service import DataLabelingManagementClient
 from oci.data_labeling_service_dataplane import DataLabelingClient
 from oci.data_science import DataScienceClient
 from oci.identity import IdentityClient
+from oci.marketplace import MarketplaceClient
 from oci.object_storage import ObjectStorageClient
 from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
@@ -25,7 +28,7 @@ class OCIClientFactory:
 
     """
     A factory class to create OCI client objects. The constructor takes in config, signer and client_kwargs. `client_kwargs` is passed
-    to the client constructor as key word argutments.
+    to the client constructor as key word arguments.
 
     Examples
     --------
@@ -48,12 +51,15 @@ class OCIClientFactory:
     oc.OCIClientFactory(**auth).object_storage # Creates Object storage client using instance principal authentication
     """
 
-    def __init__(self, config={}, signer=None, client_kwargs=None):
+    def __init__(self, config=None, signer=None, client_kwargs=None):
+        if not config:
+            config = {}
         self.config = config
         self.signer = signer
         self.client_kwargs = client_kwargs
 
-    def _client_impl(self, client):
+    @staticmethod
+    def _client_impl(client):
         client_map = {
             "object_storage": ObjectStorageClient,
             "data_science": DataScienceClient,
@@ -66,21 +72,17 @@ def _client_impl(self, client):
             "data_labeling_cp": DataLabelingManagementClient,
             "resource_search": ResourceSearchClient,
             "data_catalog": DataCatalogClient,
+            "marketplace": MarketplaceClient,
+            "artifacts": ArtifactsClient,
         }
-        try:
-            from oci.feature_store import FeatureStoreClient
-
-            client_map["feature_store"] = FeatureStoreClient
-        except ImportError:
-            logger.debug("OCI SDK with feature store support is not installed")
-            pass
 
         assert (
             client in client_map
         ), f"Invalid client name. Client name not found in {client_map.keys()}"
         return client_map[client]
 
-    def _validate_auth_param(self, auth):
+    @staticmethod
+    def _validate_auth_param(auth):
         if not isinstance(auth, dict):
             raise ValueError("auth parameter should be of type dictionary")
         if "config" in auth and not isinstance(auth["config"], dict):
@@ -135,10 +137,6 @@ def ai_language(self):
     def data_labeling_cp(self):
         return self.create_client("data_labeling_cp")
 
-    @property
-    def feature_store(self):
-        return self.create_client("feature_store")
-
     @property
     def data_labeling_dp(self):
         return self.create_client("data_labeling_dp")
@@ -150,3 +148,11 @@ def resource_search(self):
     @property
     def data_catalog(self):
         return self.create_client("data_catalog")
+
+    @property
+    def marketplace(self):
+        return self.create_client("marketplace")
+
+    @property
+    def artifacts(self) -> oci.artifacts.ArtifactsClient:
+        return self.create_client("artifacts")

ads/feature_store/README.md

@@ -0,0 +1,65 @@
+# Oracle Feature Store (ADS)
+
+[![Python](https://img.shields.io/badge/python-3.8-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/oracle-ads/) [![PysparkConda](https://img.shields.io/badge/fspyspark32_p38_cpu_v2-1.0-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://docs.oracle.com/en-us/iaas/data-science/using/conda-pyspark-fam.htm) [![Notebook Examples](https://img.shields.io/badge/docs-notebook--examples-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples) [![Delta](https://img.shields.io/badge/delta-2.0.1-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://delta.io/) [![PySpark](https://img.shields.io/badge/pyspark-3.2.1-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://spark.apache.org/docs/3.2.1/api/python/index.html) [![Great Expectations](https://img.shields.io/badge/greatexpectations-0.17.19-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://greatexpectations.io/) [![Pandas](https://img.shields.io/badge/pandas-1.5.3-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://pandas.pydata.org/) [![PyArrow](https://img.shields.io/badge/pyarrow-11.0.0-blue?style=for-the-badge&logo=pypi&logoColor=white)](https://arrow.apache.org/docs/python/index.html)
+
+Managing many datasets, data sources, and transformations for machine learning is complex and costly. Poorly cleaned data, data issues, bugs in transformations, data drift, and training serving skew all lead to increased model development time and poor model performance. Feature store solves many of the problems because it is a centralized way to transform and access data for training and serving time, Feature stores help define a standardised pipeline for ingestion of data and querying of data.
+
+ADS feature store is a stack-based solution that is deployed in your tenancy using OCI Resource Manager.
+
+Following are brief descriptions of key concepts and the main components of ADS feature store.
+
+- ``Feature Vector``:  Set of feature values for any one primary and identifier key. For example, all and a subset of features of customer ID 2536 can be called as one feature vector .
+- ``Feature``: A feature is an individual measurable property or characteristic of an event being observed.
+- ``Entity``: An entity is a group of semantically related features. The first step a consumer of features would typically do when accessing the feature store service is to list the entities and the entities associated with features. Another way to look at it is that an entity is an object or concept that's described by its features. Examples of entities are customer, product, transaction, review, image, document, and so on.
+- ``Feature Group``: A feature group in a feature store is a collection of related features that are often used together in ML models. It serves as an organizational unit within the feature store for users to manage, version, and share features across different ML projects. By organizing features into groups, data scientists and ML engineers can efficiently discover, reuse, and collaborate on features reducing the redundant work and ensuring consistency in feature engineering.
+- ``Feature Group Job``: Feature group jobs are the processing instance of a feature group. Each feature group job includes validation results and statistics results.
+- ``Dataset``: 	A dataset is a collection of features that are used together to either train a model or perform model inference.
+- ``Dataset Job``: A dataset job is the processing instance of a dataset. Each dataset job includes validation results and statistics results.
+
+## Documentation
+
+  - [Oracle Feature Store SDK (ADS) Documentation](https://feature-store-accelerated-data-science.readthedocs.io/en/latest/)
+  - [OCI Data Science and AI services Examples](https://github.com/oracle/oci-data-science-ai-samples)
+  - [Oracle AI & Data Science Blog](https://blogs.oracle.com/ai-and-datascience/)
+  - [OCI Documentation](https://docs.oracle.com/en-us/iaas/data-science/using/data-science.htm)
+
+## Examples
+
+### Quick start examples
+
+| Jupyter Notebook                                                                                                                                                          | Description                                                                                                                                                                                                                                                                                                 |
+|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Feature store querying](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_querying.ipynb)                          | - Ingestion, querying and exploration of data.                                                                                                                                                                                                                                                              |
+| [Feature store quickstart](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_quickstart.ipynb)                      | - Ingestion, querying and exploration of data.                                                                                                                                                                                                                                                              |
+| [Schema enforcement and schema evolution](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_schema_evolution.ipynb) | - `Schema evolution` allows you to easily change a table's current schema to accommodate data that is changing over time. `Schema enforcement`, also known as schema validation, is a safeguard in Delta Lake that ensures data quality by rejecting writes to a table that don't match the table's schema. |
+| [Storage of medical records in feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_ehr_data.ipynb)     | Example to demonstrate storage of medical records in feature store                                                                                                                                                                                                                                          |
+
+### Big data operations using OCI DataFlow
+
+| Jupyter Notebook                                                                                                                                                    | Description                                                                                |
+|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|
+| [Big data operations with feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_spark_magic.ipynb) | - Ingestion of data using Spark Magic, querying and exploration of data using Spark Magic. |
+
+### LLM Use cases
+
+| Jupyter Notebook                                                                                                                                                                                                                        | Description                                                                                                                                                                                                                                                                                                              |
+|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Embeddings in Feature Store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_embeddings.ipynb)                                                                                 | - `Embedding feature stores` are optimized for fast and efficient retrieval of embeddings. This is important because embeddings can be high-dimensional and computationally expensive to calculate. By storing them in a dedicated store, you can avoid the need to recalculate embeddings for the same data repeatedly. |
+| [Synthetic data generation in feature store using OpenAI and FewShotPromptTemplate](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_medical_synthetic_data_openai.ipynb)        | - `Synthetic data` is artificially generated data, rather than data collected from real-world events. It's used to simulate real data without compromising privacy or encountering real-world limitations.                                                                                                               |
+| [PII Data redaction, Summarise Content and Translate content using doctran and open AI](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_pii_redaction_and_transformation.ipynb) | - One way to think of Doctran is a LLM-powered black box where messy strings go in and nice, clean, labelled strings come out. Another way to think about it is a modular, declarative wrapper over OpenAI's functional calling feature that significantly improves the developer experience.                            |
+| [OpenAI embeddings in feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_embeddings_openai.ipynb)                                                                   | - `Embedding feature stores` are optimized for fast and efficient retrieval of embeddings. This is important because embeddings can be high-dimensional and computationally expensive to calculate. By storing them in a dedicated store, you can avoid the need to recalculate embeddings for the same data repeatedly. |
+
+
+## Contributing
+
+This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./../../CONTRIBUTING.md)
+
+Find Getting Started instructions for developers in [README-development.md](https://github.com/oracle/accelerated-data-science/blob/main/README-development.md)
+
+## Security
+
+Consult the security guide [SECURITY.md](https://github.com/oracle/accelerated-data-science/blob/main/SECURITY.md) for our responsible security vulnerability disclosure process.
+
+## License
+
+Copyright (c) 2020, 2022 Oracle and/or its affiliates. Licensed under the [Universal Permissive License v1.0](https://oss.oracle.com/licenses/upl/)

ads/feature_store/common/utils/transformation_utils.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 import json
 
 # Copyright (c) 2023 Oracle and/or its affiliates.
@@ -52,9 +56,7 @@ def apply_transformation(
         # Execute the function under namespace
         execution_namespace = {}
         exec(transformation_function, execution_namespace)
-        transformation_function_caller = execution_namespace.get(
-            transformation.display_name
-        )
+        transformation_function_caller = execution_namespace.get(transformation.name)
         transformed_data = None
 
         transformation_kwargs_dict = json.loads(transformation_kwargs)

ads/feature_store/dataset.py

@@ -50,7 +50,7 @@
 )
 from ads.feature_store.validation_output import ValidationOutput
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 logger = logging.getLogger(__name__)
@@ -422,8 +422,8 @@ def with_expectation_suite(
 
         Returns
         -------
-        Pipeline
-            The Expectation instance (self).
+        Dataset
+            The Dataset instance (self).
         """
         return self.set_spec(
             self.CONST_EXPECTATION_DETAILS,

ads/feature_store/dataset_job.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
-# -*- coding: utf-8; -*-
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
 import logging
 from copy import deepcopy
 from typing import Dict, List, Any, Union
@@ -54,7 +54,7 @@ class DatasetJob(Builder):
     >>> dataset_run = dataset_run.DatasetJob()
     >>>     .with_compartment_id(os.environ["PROJECT_COMPARTMENT_OCID"])
     >>>     .with_dataset_id("dataset_id")
-    >>>     .with_ingestion_mode(IngestionMode.SQL)
+    >>>     .with_ingestion_mode(BatchIngestionMode.SQL)
     >>> dataset_run.create()
     """
 

ads/feature_store/docs/requirements.txt

@@ -1,12 +1,11 @@
-autodoc
-nbsphinx
+autodoc nbsphinx
 sphinx
 sphinxcontrib-napoleon
 sphinx_copybutton
 sphinx_code_tabs
 sphinx-autobuild
 sphinx-autorun
-oracle_ads==2.9.0rc0
+oracle_ads
 furo
 IPython
 pandoc