LA-41 Update POST endpoints for data categories, subjects and uses fo…

…r new taxonomy functionality (#5468)

Co-authored-by: Adrian Galvan <[email protected]>

CHANGELOG.md

@@ -30,6 +30,7 @@ The types of changes are:
 
 ### Changed
 - Allow hiding systems via a `hidden` parameter and add two flags on the `/system` api endpoint; `show_hidden` and `dnd_relevant`, to display only systems with integrations [#5484](https://github.com/ethyca/fides/pull/5484)
+- Updated POST taxonomy endpoints to handle creating resources without specifying fides_key [#5468](https://github.com/ethyca/fides/pull/5468)
 
 ### Developer Experience
 - Fixing BigQuery integration tests [#5491](https://github.com/ethyca/fides/pull/5491)

src/fides/api/api/v1/endpoints/generic_overrides.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Type, Union
 
 from fastapi import APIRouter, Depends, Query, Security
 from fastapi_pagination import Page, Params
@@ -7,15 +7,23 @@
 from sqlalchemy import not_
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.sql.expression import select
+from starlette import status
 
+from fides.api.db.base_class import get_key_from_data
 from fides.api.db.crud import list_resource_query
 from fides.api.db.ctl_session import get_async_db
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.datasetconfig import DatasetConfig
 from fides.api.oauth.utils import verify_oauth_client
 from fides.api.schemas.filter_params import FilterParams
+from fides.api.schemas.taxonomy_extensions import DataCategory, DataSubject, DataUse
 from fides.api.util.filter_utils import apply_filters_to_query
-from fides.common.api.scope_registry import DATASET_READ
+from fides.common.api.scope_registry import (
+    DATA_CATEGORY_CREATE,
+    DATA_SUBJECT_CREATE,
+    DATA_USE_CREATE,
+    DATASET_READ,
+)
 from fides.common.api.v1.urn_registry import V1_URL_PREFIX
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
@@ -26,6 +34,9 @@
 # when we need more custom implementations for only some of the methods in a router.
 
 dataset_router = APIRouter(tags=["Dataset"], prefix=V1_URL_PREFIX)
+data_use_router = APIRouter(tags=["DataUse"], prefix=V1_URL_PREFIX)
+data_category_router = APIRouter(tags=["DataCategory"], prefix=V1_URL_PREFIX)
+data_subject_router = APIRouter(tags=["DataSubject"], prefix=V1_URL_PREFIX)
 
 
 @dataset_router.get(
@@ -85,5 +96,92 @@ async def list_dataset_paginated(
     return await async_paginate(db, filtered_query, pagination_params)
 
 
+async def create_with_key(
+    data: Union[DataUse, DataCategory, DataSubject],
+    model: Type[Union[DataUse, DataCategory, DataSubject]],
+    db: AsyncSession,
+) -> Dict:
+    """
+    Helper to create taxonomy resource when not given a fides_key.
+    Automatically re-enables disabled resources with the same name.
+    """
+    # If data with same name exists but is disabled, re-enable it
+    disabled_resource_with_name = db.query(model).filter(
+        model.key == data.name,  # type: ignore[union-attr]
+        model.active is False,
+    )
+    if disabled_resource_with_name:
+        return model.update(db=db, data=data, active=True)  # type: ignore[union-attr]
+    data.fides_key = get_key_from_data(
+        {"key": data.fides_key, "name": data.name}, model.__name__
+    )
+    return model.create(db=db, data=data.model_dump(mode="json"))  # type: ignore[union-attr]
+
+
+@data_use_router.post(
+    "/data_use",
+    dependencies=[Security(verify_oauth_client, scopes=[DATA_USE_CREATE])],
+    response_model=DataUse,
+    status_code=status.HTTP_201_CREATED,
+    name="Create",
+)
+async def create_data_use(
+    data_use: DataUse,
+    db: AsyncSession = Depends(get_async_db),
+) -> Dict:
+    """
+    Create a data use. Updates existing data use if data use with name already exists and is disabled.
+    """
+    if data_use.fides_key is None:
+        await create_with_key(data_use, DataUse, db)
+
+    return await DataUse.create(db=db, data=data_use.model_dump(mode="json"))  # type: ignore[attr-defined]
+
+
+@data_category_router.post(
+    "/data_category",
+    dependencies=[Security(verify_oauth_client, scopes=[DATA_CATEGORY_CREATE])],
+    response_model=DataCategory,
+    status_code=status.HTTP_201_CREATED,
+    name="Create",
+)
+async def create_data_category(
+    data_category: DataCategory,
+    db: AsyncSession = Depends(get_async_db),
+) -> Dict:
+    """
+    Create a data category
+    """
+
+    if data_category.fides_key is None:
+        await create_with_key(data_category, DataCategory, db)
+
+    return await DataCategory.create(db=db, data=data_category.model_dump(mode="json"))  # type: ignore[attr-defined]
+
+
+@data_subject_router.post(
+    "/data_subject",
+    dependencies=[Security(verify_oauth_client, scopes=[DATA_SUBJECT_CREATE])],
+    response_model=DataSubject,
+    status_code=status.HTTP_201_CREATED,
+    name="Create",
+)
+async def create_data_subject(
+    data_subject: DataSubject,
+    db: AsyncSession = Depends(get_async_db),
+) -> Dict:
+    """
+    Create a data subject
+    """
+
+    if data_subject.fides_key is None:
+        await create_with_key(data_subject, DataSubject, db)
+
+    return await DataSubject.create(db=db, data=data_subject.model_dump(mode="json"))  # type: ignore[attr-defined]
+
+
 GENERIC_OVERRIDES_ROUTER = APIRouter()
 GENERIC_OVERRIDES_ROUTER.include_router(dataset_router)
+GENERIC_OVERRIDES_ROUTER.include_router(data_use_router)
+GENERIC_OVERRIDES_ROUTER.include_router(data_category_router)
+GENERIC_OVERRIDES_ROUTER.include_router(data_subject_router)

src/fides/api/api/v1/endpoints/router_factory.py

@@ -162,6 +162,7 @@ async def create(
             raise errors.ForbiddenIsDefaultTaxonomyError(
                 model_type, resource.fides_key, action="create"
             )
+
         return await create_resource(sql_model, resource.model_dump(mode="json"), db)
 
     return router

tests/ctl/core/test_api.py

@@ -3022,6 +3022,133 @@ def test_system_manager_gets_403_if_system_not_found(
         assert result.status_code == HTTP_403_FORBIDDEN
 
 
+@pytest.mark.integration
+class TestDefaultTaxonomyCrudOverrides:
+    @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS)
+    def test_api_cannot_create_if_generated_fides_key_conflicts_with_existing(
+        self,
+        test_config: FidesConfig,
+        generate_auth_header,
+        endpoint: str,
+    ) -> None:
+        """Ensure we cannot create taxonomy elements if fides key conflicts with existing key"""
+        # get a default taxonomy element as a sample resource
+        resource = getattr(DEFAULT_TAXONOMY, endpoint)[0]
+        resource = TAXONOMY_EXTENSIONS[endpoint](
+            **resource.model_dump(mode="json")
+        )  # cast resource to extended model
+        # This name will conflict with existing resource
+        resource.name = resource.fides_key
+        resource.fides_key = None
+        json_resource = resource.json(exclude_none=True)
+        token_scopes: List[str] = [f"{CLI_SCOPE_PREFIX_MAPPING[endpoint]}:{CREATE}"]
+        auth_header = generate_auth_header(scopes=token_scopes)
+        result = _api.create(
+            url=test_config.cli.server_url,
+            headers=auth_header,
+            resource_type=endpoint,
+            json_resource=json_resource,
+        )
+        assert result.status_code == HTTP_409_CONFLICT
+
+    @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS)
+    def test_api_can_create_without_explicit_fides_key(
+        self,
+        test_config: FidesConfig,
+        generate_auth_header,
+        endpoint: str,
+    ) -> None:
+        """Ensure we can create taxonomy elements without specifying a fides_key"""
+        # get a default taxonomy element as a sample resource
+        resource = getattr(DEFAULT_TAXONOMY, endpoint)[0]
+        resource = TAXONOMY_EXTENSIONS[endpoint](
+            **resource.model_dump(mode="json")
+        )  # cast resource to extended model
+        # Build unique name based on sample name
+        resource.name = resource.name + "my new resource"
+        resource.fides_key = None
+        json_resource = resource.json(exclude_none=True)
+        token_scopes: List[str] = [f"{CLI_SCOPE_PREFIX_MAPPING[endpoint]}:{CREATE}"]
+        auth_header = generate_auth_header(scopes=token_scopes)
+        result = _api.create(
+            url=test_config.cli.server_url,
+            headers=auth_header,
+            resource_type=endpoint,
+            json_resource=json_resource,
+        )
+        assert result.status_code == 201
+        assert result.json()["active"] is True
+        new_key = result.json()["fides_key"]
+        assert "my_new_resource" in new_key
+
+        result = _api.get(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type=endpoint,
+            resource_id=new_key,
+        )
+        assert result.json()["active"] is True
+
+    @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS)
+    def test_api_can_update_active_when_creating_with_same_name(
+        self,
+        test_config: FidesConfig,
+        endpoint: str,
+    ) -> None:
+        """
+        If we attempt to create a new resource with the same name as an existing inactive resource,
+        but with no explicit fides_key, we should update the existing resource to be active
+        """
+        resource = getattr(DEFAULT_TAXONOMY, endpoint)[0]
+        resource = TAXONOMY_EXTENSIONS[endpoint](
+            **resource.model_dump(mode="json")
+        )  # cast resource to extended model
+        resource.active = False
+        json_resource = resource.json(exclude_none=True)
+        # First, update the existing resource as inactive so we can use it to test
+        result = _api.update(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type=endpoint,
+            json_resource=json_resource,
+        )
+        assert result.status_code == 200
+        assert result.json()["active"] is False
+
+        # Confirm it was updated to inactive
+        result = _api.get(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type=endpoint,
+            resource_id=resource.fides_key,
+        )
+        assert result.json()["active"] is False
+
+        # Now attempt to create another resource with a name that will generate the same fides_key
+        # as the inactive resource
+
+        resource.name = resource.name  # explicitly using the same name
+        resource.fides_key = None
+        json_resource = resource.json(exclude_none=True)
+        result = _api.create(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type=endpoint,
+            json_resource=json_resource,
+        )
+        assert result.status_code == 200
+        assert result.json()["active"] is True
+
+        # Confirm the existing resource was updated to active
+        result = _api.get(
+            url=test_config.cli.server_url,
+            headers=test_config.user.auth_header,
+            resource_type=endpoint,
+            resource_id=resource.fides_key,
+        )
+        assert result.json()["active"] is True
+
+
 @pytest.mark.integration
 class TestDefaultTaxonomyCrud:
     @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS)