From 9fae53849259930a0652d0d2278cd8509a735709 Mon Sep 17 00:00:00 2001
From: Shalev Avhar <51760613+shalev007@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:35:05 +0200
Subject: [PATCH] [Core] Add support for choosing default resources integration
will create dynamically (#1129)
# Description
What - Add the option to create custom type of resources for an
integration
Why - Ocean Saas is the only type of Ocean integrations that can use
secrets
How - added a new config parameter to the integration settings
## Type of change
Please leave one option from the following and delete the rest:
- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)
All tests should be run against the port production
environment(using a testing org).
### Core testing checklist
- [x] Integration able to create all default resources from scratch
- [x] Resync finishes successfully
- [x] Resync able to create entities
- [x] Resync able to update entities
- [x] Resync able to detect and delete entities
- [ ] Scheduled resync able to abort existing resync and start a new one
- [ ] Tested with at least 2 integrations from scratch
- [ ] Tested with Kafka and Polling event listeners
- [ ] Tested deletion of entities that don't pass the selector
### Integration testing checklist
- [ ] Integration able to create all default resources from scratch
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Resync finishes successfully
- [ ] If new resource kind is added or updated in the integration, add
example raw data, mapping and expected result to the `examples` folder
in the integration directory.
- [ ] If resource kind is updated, run the integration with the example
data and check if the expected result is achieved
- [ ] If new resource kind is added or updated, validate that
live-events for that resource are working as expected
- [ ] Docs PR link [here](#)
### Preflight checklist
- [ ] Handled rate limiting
- [ ] Handled pagination
- [ ] Implemented the code in async
- [ ] Support Multi account
## Screenshots
Include screenshots from your environment showing how the resources of
the integration will look.
## API Documentation
Provide links to the API documentation used for this integration.
---------
Co-authored-by: Shalev Avhar
Co-authored-by: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com>
---
CHANGELOG.md | 7 +
port_ocean/cli/commands/defaults/clean.py | 4 +-
port_ocean/config/settings.py | 1 +
port_ocean/core/defaults/clean.py | 13 +-
port_ocean/core/defaults/common.py | 32 +++-
port_ocean/core/defaults/initialize.py | 4 +-
port_ocean/tests/core/defaults/test_common.py | 166 ++++++++++++++++++
pyproject.toml | 2 +-
8 files changed, 215 insertions(+), 14 deletions(-)
create mode 100644 port_ocean/tests/core/defaults/test_common.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 74670ca731..9bc70f5938 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
+## 0.14.0 (2024-11-12)
+
+
+### Improvements
+
+- Add support for choosing default resources that the integration will create dynamically
+
## 0.13.1 (2024-11-12)
diff --git a/port_ocean/cli/commands/defaults/clean.py b/port_ocean/cli/commands/defaults/clean.py
index 888e2a2ffd..2c6fd2eeb1 100644
--- a/port_ocean/cli/commands/defaults/clean.py
+++ b/port_ocean/cli/commands/defaults/clean.py
@@ -51,4 +51,6 @@ def clean(path: str, force: bool, wait: bool) -> None:
default_app,
)
- clean_defaults(app.integration.AppConfigHandlerClass.CONFIG_CLASS, force, wait)
+ clean_defaults(
+ app.integration.AppConfigHandlerClass.CONFIG_CLASS, app.config, force, wait
+ )
diff --git a/port_ocean/config/settings.py b/port_ocean/config/settings.py
index 5b9a8e7abb..37aa108d18 100644
--- a/port_ocean/config/settings.py
+++ b/port_ocean/config/settings.py
@@ -78,6 +78,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
default_factory=lambda: IntegrationSettings(type="", identifier="")
)
runtime: Runtime = Runtime.OnPrem
+ resources_path: str = Field(default=".port/resources")
@root_validator()
def validate_integration_config(cls, values: dict[str, Any]) -> dict[str, Any]:
diff --git a/port_ocean/core/defaults/clean.py b/port_ocean/core/defaults/clean.py
index 2670713e83..0a10d90b40 100644
--- a/port_ocean/core/defaults/clean.py
+++ b/port_ocean/core/defaults/clean.py
@@ -4,6 +4,7 @@
import httpx
from loguru import logger
+from port_ocean.config.settings import IntegrationConfiguration
from port_ocean.context.ocean import ocean
from port_ocean.core.defaults.common import (
get_port_integration_defaults,
@@ -14,12 +15,13 @@
def clean_defaults(
config_class: Type[PortAppConfig],
+ integration_config: IntegrationConfiguration,
force: bool,
wait: bool,
) -> None:
try:
asyncio.new_event_loop().run_until_complete(
- _clean_defaults(config_class, force, wait)
+ _clean_defaults(config_class, integration_config, force, wait)
)
except Exception as e:
@@ -27,13 +29,18 @@ def clean_defaults(
async def _clean_defaults(
- config_class: Type[PortAppConfig], force: bool, wait: bool
+ config_class: Type[PortAppConfig],
+ integration_config: IntegrationConfiguration,
+ force: bool,
+ wait: bool,
) -> None:
port_client = ocean.port_client
is_exists = await is_integration_exists(port_client)
if not is_exists:
return None
- defaults = get_port_integration_defaults(config_class)
+ defaults = get_port_integration_defaults(
+ config_class, integration_config.resources_path
+ )
if not defaults:
return None
diff --git a/port_ocean/core/defaults/common.py b/port_ocean/core/defaults/common.py
index dfb84bb31a..1facba88b3 100644
--- a/port_ocean/core/defaults/common.py
+++ b/port_ocean/core/defaults/common.py
@@ -3,6 +3,7 @@
from typing import Type, Any, TypedDict, Optional
import httpx
+from loguru import logger
import yaml
from pydantic import BaseModel, Field
from starlette import status
@@ -77,18 +78,33 @@ def deconstruct_blueprints_to_creation_steps(
)
+def is_valid_dir(path: Path) -> bool:
+ return path.is_dir()
+
+
def get_port_integration_defaults(
- port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".")
+ port_app_config_class: Type[PortAppConfig],
+ custom_defaults_dir: Optional[str] = None,
+ base_path: Path = Path("."),
) -> Defaults | None:
- defaults_dir = base_path / ".port/resources"
- if not defaults_dir.exists():
- return None
-
- if not defaults_dir.is_dir():
- raise UnsupportedDefaultFileType(
- f"Defaults directory is not a directory: {defaults_dir}"
+ fallback_dir = base_path / ".port/resources"
+
+ if custom_defaults_dir and is_valid_dir(base_path / custom_defaults_dir):
+ defaults_dir = base_path / custom_defaults_dir
+ elif is_valid_dir(fallback_dir):
+ logger.info(
+ f"Could not find custom defaults directory {custom_defaults_dir}, falling back to {fallback_dir}",
+ fallback_dir=fallback_dir,
+ custom_defaults_dir=custom_defaults_dir,
+ )
+ defaults_dir = fallback_dir
+ else:
+ logger.warning(
+ f"Could not find defaults directory {fallback_dir}, skipping defaults"
)
+ return None
+ logger.info(f"Loading defaults from {defaults_dir}", defaults_dir=defaults_dir)
default_jsons = {}
allowed_file_names = [
field_model.alias for _, field_model in Defaults.__fields__.items()
diff --git a/port_ocean/core/defaults/initialize.py b/port_ocean/core/defaults/initialize.py
index 66aa9efae3..c6d8d88ed1 100644
--- a/port_ocean/core/defaults/initialize.py
+++ b/port_ocean/core/defaults/initialize.py
@@ -198,7 +198,9 @@ async def _initialize_defaults(
config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration
) -> None:
port_client = ocean.port_client
- defaults = get_port_integration_defaults(config_class)
+ defaults = get_port_integration_defaults(
+ config_class, integration_config.resources_path
+ )
if not defaults:
logger.warning("No defaults found. Skipping initialization...")
return None
diff --git a/port_ocean/tests/core/defaults/test_common.py b/port_ocean/tests/core/defaults/test_common.py
new file mode 100644
index 0000000000..a62429d196
--- /dev/null
+++ b/port_ocean/tests/core/defaults/test_common.py
@@ -0,0 +1,166 @@
+import pytest
+import json
+from unittest.mock import patch
+from pathlib import Path
+from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.core.defaults.common import (
+ get_port_integration_defaults,
+ Defaults,
+)
+
+
+@pytest.fixture
+def setup_mock_directories(tmp_path: Path) -> tuple[Path, Path, Path]:
+ # Create .port/resources with sample files
+ default_dir = tmp_path / ".port/resources"
+ default_dir.mkdir(parents=True, exist_ok=True)
+
+ # Create mock JSON and YAML files with expected content
+ (default_dir / "blueprints.json").write_text(
+ json.dumps(
+ [
+ {
+ "identifier": "mock-identifier",
+ "title": "mock-title",
+ "icon": "mock-icon",
+ "schema": {
+ "type": "object",
+ "properties": {"key": {"type": "string"}},
+ },
+ }
+ ]
+ )
+ )
+ (default_dir / "port-app-config.json").write_text(
+ json.dumps(
+ {
+ "resources": [
+ {
+ "kind": "mock-kind",
+ "selector": {"query": "true"},
+ "port": {
+ "entity": {
+ "mappings": {
+ "identifier": ".id",
+ "title": ".title",
+ "blueprint": '"mock-identifier"',
+ }
+ }
+ },
+ }
+ ]
+ }
+ )
+ )
+
+ # Create .port/custom_resources with different sample files
+ custom_resources_dir = tmp_path / ".port/custom_resources"
+ custom_resources_dir.mkdir(parents=True, exist_ok=True)
+
+ # Create mock JSON and YAML files with expected content
+ (custom_resources_dir / "blueprints.json").write_text(
+ json.dumps(
+ [
+ {
+ "identifier": "mock-custom-identifier",
+ "title": "mock-custom-title",
+ "icon": "mock-custom-icon",
+ "schema": {
+ "type": "object",
+ "properties": {"key": {"type": "string"}},
+ },
+ }
+ ]
+ )
+ )
+ (custom_resources_dir / "port-app-config.json").write_text(
+ json.dumps(
+ {
+ "resources": [
+ {
+ "kind": "mock-custom-kind",
+ "selector": {"query": "true"},
+ "port": {
+ "entity": {
+ "mappings": {
+ "identifier": ".id",
+ "title": ".title",
+ "blueprint": '"mock-custom-identifier"',
+ }
+ }
+ },
+ }
+ ]
+ }
+ )
+ )
+
+ # Define the non-existing directory path
+ non_existing_dir = tmp_path / ".port/do_not_exist"
+
+ return default_dir, custom_resources_dir, non_existing_dir
+
+
+def test_custom_defaults_dir_used_if_valid(
+ setup_mock_directories: tuple[Path, Path, Path]
+) -> None:
+ # Arrange
+ _, custom_resources_dir, _ = setup_mock_directories
+
+ with (
+ patch("port_ocean.core.defaults.common.is_valid_dir") as mock_is_valid_dir,
+ patch(
+ "pathlib.Path.iterdir",
+ return_value=custom_resources_dir.iterdir(),
+ ),
+ ):
+ mock_is_valid_dir.side_effect = lambda path: path == custom_resources_dir
+
+ # Act
+ defaults = get_port_integration_defaults(
+ port_app_config_class=PortAppConfig,
+ custom_defaults_dir=".port/custom_resources",
+ base_path=custom_resources_dir.parent.parent,
+ )
+
+ # Assert
+ assert isinstance(defaults, Defaults)
+ assert defaults.blueprints[0].get("identifier") == "mock-custom-identifier"
+ assert defaults.port_app_config is not None
+ assert defaults.port_app_config.resources[0].kind == "mock-custom-kind"
+
+
+def test_fallback_to_default_dir_if_custom_dir_invalid(
+ setup_mock_directories: tuple[Path, Path, Path]
+) -> None:
+ resources_dir, _, non_existing_dir = setup_mock_directories
+
+ # Arrange
+ with (
+ patch("port_ocean.core.defaults.common.is_valid_dir") as mock_is_valid_dir,
+ patch("pathlib.Path.iterdir", return_value=resources_dir.iterdir()),
+ ):
+
+ mock_is_valid_dir.side_effect = lambda path: path == resources_dir
+
+ # Act
+ custom_defaults_dir = str(non_existing_dir.relative_to(resources_dir.parent))
+ defaults = get_port_integration_defaults(
+ port_app_config_class=PortAppConfig,
+ custom_defaults_dir=custom_defaults_dir,
+ base_path=resources_dir.parent.parent,
+ )
+
+ # Assert
+ assert isinstance(defaults, Defaults)
+ assert defaults.blueprints[0].get("identifier") == "mock-identifier"
+ assert defaults.port_app_config is not None
+ assert defaults.port_app_config.resources[0].kind == "mock-kind"
+
+
+def test_default_resources_path_does_not_exist() -> None:
+ # Act
+ defaults = get_port_integration_defaults(port_app_config_class=PortAppConfig)
+
+ # Assert
+ assert defaults is None
diff --git a/pyproject.toml b/pyproject.toml
index 8062a4f1ad..9488dd1257 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "port-ocean"
-version = "0.13.1"
+version = "0.14.0"
description = "Port Ocean is a CLI tool for managing your Port projects."
readme = "README.md"
homepage = "https://app.getport.io"