From 2421cbd61a40f394f64c18f128c9a46dc473a517 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Tue, 19 Nov 2024 07:24:00 +0100 Subject: [PATCH 01/14] add base logic for cogs (with AbstractCog & CogsRunner) --- backend/nango/cogs/__init__.py | 10 ++++ backend/nango/cogs/serializer_cog.py | 12 +++++ backend/nango/{utils.py => utils/__init__.py} | 7 +++ backend/nango/utils/abstract_cog.py | 31 ++++++++++++ backend/nango/utils/cogs_runner.py | 49 +++++++++++++++++++ 5 files changed, 109 insertions(+) create mode 100644 backend/nango/cogs/__init__.py create mode 100644 backend/nango/cogs/serializer_cog.py rename backend/nango/{utils.py => utils/__init__.py} (80%) create mode 100644 backend/nango/utils/abstract_cog.py create mode 100644 backend/nango/utils/cogs_runner.py diff --git a/backend/nango/cogs/__init__.py b/backend/nango/cogs/__init__.py new file mode 100644 index 0000000..03205e4 --- /dev/null +++ b/backend/nango/cogs/__init__.py @@ -0,0 +1,10 @@ +"""Cogs are nango's plugins that give it flexibility. + +Every Nango function is linked to a cog. + +Note: Order of the __all__ matches the order of cogs execution. +""" + +from .serializer_cog import SerializerCog + +__all__ = ["SerializerCog"] diff --git a/backend/nango/cogs/serializer_cog.py b/backend/nango/cogs/serializer_cog.py new file mode 100644 index 0000000..2635655 --- /dev/null +++ b/backend/nango/cogs/serializer_cog.py @@ -0,0 +1,12 @@ +from nango.utils import AbstractCog + + +class SerializerCog(AbstractCog): + """Generate serializer from each django's models.""" + + id = "serializer" + + def run(self) -> None: + """Generate a serializer for each django's model.""" + if not self.is_executable(): + return diff --git a/backend/nango/utils.py b/backend/nango/utils/__init__.py similarity index 80% rename from backend/nango/utils.py rename to backend/nango/utils/__init__.py index 17b674a..9a88908 100644 --- a/backend/nango/utils.py +++ b/backend/nango/utils/__init__.py @@ -1,3 +1,10 @@ +from nango.utils.abstract_cog import AbstractCog + +__all__ = [ + "AbstractCog", +] + + def setup_django() -> None: """Setup django environment.""" import os diff --git a/backend/nango/utils/abstract_cog.py b/backend/nango/utils/abstract_cog.py new file mode 100644 index 0000000..bd59963 --- /dev/null +++ b/backend/nango/utils/abstract_cog.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from django.db import models + + +class AbstractCog(ABC): + """Cog's base class, that gives the same basic functionalities for all cogs.""" + + id = None + + def __init__(self, model: models.Model, settings: dict[str, any]) -> None: + """Setup the cogs. + + Elements of the setup: + + - Settings + """ + self.model = model + self.settings = settings.get(self.id, {}) + + @abstractmethod + def run(self) -> None: + """Execute the cog logic.""" + + def is_executable(self) -> bool: + """Indicate if the cog can run or not.""" + return self.settings is not None diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py new file mode 100644 index 0000000..cf951a4 --- /dev/null +++ b/backend/nango/utils/cogs_runner.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + +from django.apps import apps + +from nango import cogs + +if TYPE_CHECKING: + from django.db import models + + +class CogsRunner: + """Load and execute cogs. + + For the order execution, follow the nango.cogs.__all__ order. + """ + + model_filter_keywords: list[str] = ("django", "django_celery_beat", "rest_framework") + + def get_cogs_classes(self) -> list[callable]: + """Return cogs's classes to execute.""" + return [getattr(import_module("nango.cogs"), cog_str) for cog_str in cogs.__all__] + + def get_settings_from_model(self, model: models.Model) -> dict[str, any]: + """Return settings define in a model.""" + + def _is_filtered_model(self, model: models.Model) -> bool: + """Indicate if a model is filter or not. + + If the model is filter, we remove it. + """ + return any(keyword in str(model) for keyword in self.model_filter_keywords) + + def run_cogs(self) -> None: + """Run all existing cogs.""" + cogs_classes = self.get_cogs_classes() + + for model in apps.get_models(): + if self._is_filtered_model(model): + return + + for cog in cogs_classes: + instanciated_cog = cog(model=model, settings=self.get_settings_from_model(model)) # noqa: F841 + + +if __name__ == "django.core.management.commands.shell": + CogsRunner().run_cogs() From 1dbe4037789bb194ad60496486b1c9dc7c3b45ae Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sat, 23 Nov 2024 15:46:52 +0100 Subject: [PATCH 02/14] add base for new CLI --- backend/nango/management/commands/nango.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 backend/nango/management/commands/nango.py diff --git a/backend/nango/management/commands/nango.py b/backend/nango/management/commands/nango.py new file mode 100644 index 0000000..493fe46 --- /dev/null +++ b/backend/nango/management/commands/nango.py @@ -0,0 +1,15 @@ +"""CLI to run the Nango's bridge.""" + +from __future__ import annotations + +from django.core.management.base import BaseCommand, CommandParser + + +class Command(BaseCommand): + """Cli to manage the bridge.""" + + def add_arguments(self, parser: CommandParser) -> None: + """.""" + + def handle(self, *args: list, **options: dict) -> None: + """Handle argument and run Nango's cogs.""" From 5dc9e4bfc1da28d1813d9f2dc77901e55733ba65 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 15:20:30 +0100 Subject: [PATCH 03/14] feat(runner): add run_cogs method to execute cogs for each (not filtered model) --- backend/nango/utils/cogs_runner.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index cf951a4..da34db6 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Callable from importlib import import_module from typing import TYPE_CHECKING @@ -15,6 +16,11 @@ class CogsRunner: """Load and execute cogs. For the order execution, follow the nango.cogs.__all__ order. + + Settings: + -------- + Nango's settings can be defined for each cog at the model level. + A model can have a staticmethod called `nango` that returns a None (for no generation) or a dict (with settings for each cogs). """ model_filter_keywords: list[str] = ("django", "django_celery_beat", "rest_framework") @@ -25,11 +31,19 @@ def get_cogs_classes(self) -> list[callable]: def get_settings_from_model(self, model: models.Model) -> dict[str, any]: """Return settings define in a model.""" + model_nango_method: Callable | None = getattr(model, "nango", None) + if not isinstance(model_nango_method, Callable): + if model_nango_method is None: + # No settings defined in the model + return {} + error_msg: str = f"{model}.nango must be a staticmethod, not a property." + raise TypeError(error_msg) + return model_nango_method() def _is_filtered_model(self, model: models.Model) -> bool: """Indicate if a model is filter or not. - If the model is filter, we remove it. + If the model is filter, we don't run it. """ return any(keyword in str(model) for keyword in self.model_filter_keywords) @@ -39,10 +53,16 @@ def run_cogs(self) -> None: for model in apps.get_models(): if self._is_filtered_model(model): - return - + continue for cog in cogs_classes: - instanciated_cog = cog(model=model, settings=self.get_settings_from_model(model)) # noqa: F841 + settings = self.get_settings_from_model(model) + if settings is None: + # This model doesn't want any cogs. + continue + + # Run the cogs + instantiated_cog = cog(model=model, settings=settings) + instantiated_cog.run() if __name__ == "django.core.management.commands.shell": From 5802e734260a57e437032af18964ee942dd808ad Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 15:21:10 +0100 Subject: [PATCH 04/14] docs(cogs): add documentation on cogs --- backend/nango/cogs/serializer_cog.py | 9 ++++++--- backend/nango/utils/abstract_cog.py | 9 ++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/nango/cogs/serializer_cog.py b/backend/nango/cogs/serializer_cog.py index 2635655..6747ec4 100644 --- a/backend/nango/cogs/serializer_cog.py +++ b/backend/nango/cogs/serializer_cog.py @@ -2,11 +2,14 @@ class SerializerCog(AbstractCog): - """Generate serializer from each django's models.""" + """Generate serializer from each django's models. + + Settings: + -------- + """ id = "serializer" def run(self) -> None: """Generate a serializer for each django's model.""" - if not self.is_executable(): - return + super().run() diff --git a/backend/nango/utils/abstract_cog.py b/backend/nango/utils/abstract_cog.py index bd59963..37e70fd 100644 --- a/backend/nango/utils/abstract_cog.py +++ b/backend/nango/utils/abstract_cog.py @@ -8,7 +8,12 @@ class AbstractCog(ABC): - """Cog's base class, that gives the same basic functionalities for all cogs.""" + """Cog's base class, that gives the same basic functionalities for all cogs. + + If settings are not defined or is `{}`, the default settings will be used. + Otherwise, if the settings are `None`, the Cog will not run. + If one of the settings is not specified, the default value will be used. + """ id = None @@ -25,6 +30,8 @@ def __init__(self, model: models.Model, settings: dict[str, any]) -> None: @abstractmethod def run(self) -> None: """Execute the cog logic.""" + if not self.is_executable(): + return def is_executable(self) -> bool: """Indicate if the cog can run or not.""" From a336e0787baed85f76e70cbaec63141a24b2d582 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 15:22:52 +0100 Subject: [PATCH 05/14] typo error --- backend/nango/utils/cogs_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index da34db6..23118dc 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -25,7 +25,7 @@ class CogsRunner: model_filter_keywords: list[str] = ("django", "django_celery_beat", "rest_framework") - def get_cogs_classes(self) -> list[callable]: + def get_cogs_classes(self) -> list[Callable]: """Return cogs's classes to execute.""" return [getattr(import_module("nango.cogs"), cog_str) for cog_str in cogs.__all__] From dcc70e12107952dca2c1608d6f0621e497d190af Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 15:28:07 +0100 Subject: [PATCH 06/14] Make reworked executable through makefile --- Makefile | 3 +++ backend/nango/management/commands/nango.py | 5 ++++- backend/nango/utils/__init__.py | 2 ++ backend/nango/utils/cogs_runner.py | 4 ---- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9cdf540..b477472 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ nango: cp -r ./backend/.nango_front/ts_types ./frontend/src/api/nango_front/ ruff format +rework: + docker exec nango_django python manage.py nango + route: docker exec nango_django python manage.py bridge -r diff --git a/backend/nango/management/commands/nango.py b/backend/nango/management/commands/nango.py index 493fe46..1a6b748 100644 --- a/backend/nango/management/commands/nango.py +++ b/backend/nango/management/commands/nango.py @@ -4,6 +4,8 @@ from django.core.management.base import BaseCommand, CommandParser +from nango.utils import CogsRunner + class Command(BaseCommand): """Cli to manage the bridge.""" @@ -11,5 +13,6 @@ class Command(BaseCommand): def add_arguments(self, parser: CommandParser) -> None: """.""" - def handle(self, *args: list, **options: dict) -> None: + def handle(self, *args: list, **options: dict) -> None: # noqa: ARG002 """Handle argument and run Nango's cogs.""" + CogsRunner().run_cogs() diff --git a/backend/nango/utils/__init__.py b/backend/nango/utils/__init__.py index 9a88908..02ad91d 100644 --- a/backend/nango/utils/__init__.py +++ b/backend/nango/utils/__init__.py @@ -1,7 +1,9 @@ from nango.utils.abstract_cog import AbstractCog +from nango.utils.cogs_runner import CogsRunner __all__ = [ "AbstractCog", + "CogsRunner", ] diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index 23118dc..8c6dd5c 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -63,7 +63,3 @@ def run_cogs(self) -> None: # Run the cogs instantiated_cog = cog(model=model, settings=settings) instantiated_cog.run() - - -if __name__ == "django.core.management.commands.shell": - CogsRunner().run_cogs() From f629ac59e05bddeb9860bb383bd4a987e9d3d6c8 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 15:39:19 +0100 Subject: [PATCH 07/14] add _generate_model_import method in abstract cog --- backend/nango/utils/abstract_cog.py | 9 +++++++++ backend/nango/utils/cogs_runner.py | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/backend/nango/utils/abstract_cog.py b/backend/nango/utils/abstract_cog.py index 37e70fd..b2ad228 100644 --- a/backend/nango/utils/abstract_cog.py +++ b/backend/nango/utils/abstract_cog.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from abc import ABC, abstractmethod from typing import TYPE_CHECKING @@ -36,3 +37,11 @@ def run(self) -> None: def is_executable(self) -> bool: """Indicate if the cog can run or not.""" return self.settings is not None + + def _generate_model_import(self) -> str: + """Return the model import line for file generation.""" + regex = r"(?<=')(.*)(?=\.)" + match = re.search(regex, str(self.model)) + import_path = match.group(1) + + return f"from {import_path} import {self.model.__name__}" diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index 8c6dd5c..eea8e82 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -32,12 +32,15 @@ def get_cogs_classes(self) -> list[Callable]: def get_settings_from_model(self, model: models.Model) -> dict[str, any]: """Return settings define in a model.""" model_nango_method: Callable | None = getattr(model, "nango", None) + if not isinstance(model_nango_method, Callable): if model_nango_method is None: # No settings defined in the model return {} + error_msg: str = f"{model}.nango must be a staticmethod, not a property." raise TypeError(error_msg) + return model_nango_method() def _is_filtered_model(self, model: models.Model) -> bool: @@ -54,6 +57,7 @@ def run_cogs(self) -> None: for model in apps.get_models(): if self._is_filtered_model(model): continue + for cog in cogs_classes: settings = self.get_settings_from_model(model) if settings is None: From 134613831f8cb13d91e0b806c0ded34e1267da74 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 20:47:46 +0100 Subject: [PATCH 08/14] ruff compliance --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdf1c20..bfe64ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.5.7 + rev: v0.8.0 hooks: # Run the formatter. - id: ruff-format From 6beeb6e317657caaa8e057d393b163d894750568 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 20:48:08 +0100 Subject: [PATCH 09/14] ruff compliance --- backend/nango/bridge/bridge.py | 4 ++-- backend/requirements/development.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/nango/bridge/bridge.py b/backend/nango/bridge/bridge.py index 575d7be..9835bbf 100644 --- a/backend/nango/bridge/bridge.py +++ b/backend/nango/bridge/bridge.py @@ -287,12 +287,12 @@ def get_model_settings(self, model: ModelBase) -> dict[str, any] | None: def setup_api_for_model(self, models_data: dict[ModelBase, dict[fields, str]]) -> None: """Setup API files for a given model.""" - for model in models_data: + for model in models_data: # noqa: PLC0206 model_name = model.__name__ # Check settings for folder construction settings = self.get_model_settings(model) - if settings is None or settings["serializers"] is None and settings["view"] is None: + if settings is None or (settings["serializers"] is None and settings["view"] is None): continue self.make_api_architecture(model_name=model_name) diff --git a/backend/requirements/development.txt b/backend/requirements/development.txt index e888718..3cf1152 100644 --- a/backend/requirements/development.txt +++ b/backend/requirements/development.txt @@ -7,7 +7,7 @@ watchfiles==0.24.0 # https://github.com/samuelcolvin/watchfiles # Code quality # ------------------------------------------------------------------------------ -ruff==0.6.9 +ruff==0.8.0 pre-commit==4.0.1 coverage==7.6.2 # https://github.com/nedbat/coveragepy From 043bb8e3752b9ebf2cca7e5275162c2f1b21b754 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 21:05:59 +0100 Subject: [PATCH 10/14] Adjust code based on feedbacks --- backend/nango/management/commands/nango.py | 6 ++-- backend/nango/utils/__init__.py | 2 +- backend/nango/utils/cogs_runner.py | 39 +++++++++++----------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/backend/nango/management/commands/nango.py b/backend/nango/management/commands/nango.py index 1a6b748..930be4b 100644 --- a/backend/nango/management/commands/nango.py +++ b/backend/nango/management/commands/nango.py @@ -1,4 +1,4 @@ -"""CLI to run the Nango's bridge.""" +"""CLI to run Nango's bridge.""" from __future__ import annotations @@ -10,8 +10,8 @@ class Command(BaseCommand): """Cli to manage the bridge.""" - def add_arguments(self, parser: CommandParser) -> None: - """.""" + def add_arguments(self, parser: CommandParser) -> None: # noqa: D102 + ... def handle(self, *args: list, **options: dict) -> None: # noqa: ARG002 """Handle argument and run Nango's cogs.""" diff --git a/backend/nango/utils/__init__.py b/backend/nango/utils/__init__.py index 02ad91d..55b72c3 100644 --- a/backend/nango/utils/__init__.py +++ b/backend/nango/utils/__init__.py @@ -8,7 +8,7 @@ def setup_django() -> None: - """Setup django environment.""" + """Setup Django's environment.""" import os import sys from pathlib import Path diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index eea8e82..458c528 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from django.apps import apps +from django.conf import settings from nango import cogs @@ -15,7 +16,7 @@ class CogsRunner: """Load and execute cogs. - For the order execution, follow the nango.cogs.__all__ order. + For execution order, follow the nango.cogs.__all__ order. Settings: -------- @@ -23,14 +24,21 @@ class CogsRunner: A model can have a staticmethod called `nango` that returns a None (for no generation) or a dict (with settings for each cogs). """ - model_filter_keywords: list[str] = ("django", "django_celery_beat", "rest_framework") - - def get_cogs_classes(self) -> list[Callable]: + def get_cog_classes(self) -> list[Callable]: """Return cogs's classes to execute.""" return [getattr(import_module("nango.cogs"), cog_str) for cog_str in cogs.__all__] + def get_local_model_classes(self) -> list[models.Model]: + """Get all models for apps defined in settings.LOCAL_APPS.""" + model_list: list[models.Model] = [] + + for app_name in settings.LOCAL_APPS: + app_config = apps.app_configs.get(app_name) + model_list.extend(app_config.get_models()) + return model_list + def get_settings_from_model(self, model: models.Model) -> dict[str, any]: - """Return settings define in a model.""" + """Return settings defined in a model.""" model_nango_method: Callable | None = getattr(model, "nango", None) if not isinstance(model_nango_method, Callable): @@ -43,27 +51,18 @@ def get_settings_from_model(self, model: models.Model) -> dict[str, any]: return model_nango_method() - def _is_filtered_model(self, model: models.Model) -> bool: - """Indicate if a model is filter or not. - - If the model is filter, we don't run it. - """ - return any(keyword in str(model) for keyword in self.model_filter_keywords) - def run_cogs(self) -> None: """Run all existing cogs.""" cogs_classes = self.get_cogs_classes() - for model in apps.get_models(): - if self._is_filtered_model(model): - continue - + for model in self.get_local_model_classes(): for cog in cogs_classes: - settings = self.get_settings_from_model(model) - if settings is None: - # This model doesn't want any cogs. + model_nango_settings = self.get_settings_from_model(model) + + # This model doesn't want any cogs. + if model_nango_settings is None: continue # Run the cogs - instantiated_cog = cog(model=model, settings=settings) + instantiated_cog = cog(model=model, settings=model_nango_settings) instantiated_cog.run() From 724ec6bc4e10d10858c0442484a5a2b3f954c8fb Mon Sep 17 00:00:00 2001 From: Xenepix Date: Sun, 24 Nov 2024 22:00:41 +0100 Subject: [PATCH 11/14] wip on SerializerCog --- backend/nango/cogs/serializer_cog.py | 57 +++++++++++++++++++++++++++- backend/nango/utils/abstract_cog.py | 2 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/backend/nango/cogs/serializer_cog.py b/backend/nango/cogs/serializer_cog.py index 6747ec4..65dd3df 100644 --- a/backend/nango/cogs/serializer_cog.py +++ b/backend/nango/cogs/serializer_cog.py @@ -2,14 +2,69 @@ class SerializerCog(AbstractCog): - """Generate serializer from each django's models. + """Generates a serializer from each django's models. Settings: -------- + - speedy: A faster serializer than rest_framework.serializers.Serializer, but with less features. """ id = "serializer" + def _get_drf_imports(self) -> list[str]: + """Return list of imports from rest_framework module.""" + if self.settings.get("speedy", False): + error_msg: str = "The SerializerCog speedy feature is not implemented yet." + raise NotImplementedError(error_msg) + + return [ + "from rest_framework import serializers" if not self.settings.get("speedy", False) else "...", + ] + + def _get_base_imports(self) -> list[str]: + """Return basics imports.""" + return [ + "from __future__ import annotations", + "from typing import ClassVar", + ] + + def get_serializer_name(self) -> dict[str]: + """Return name of generated serializers. + + Returns: + ------- + ``` + { + "cached_serializer_name": str, + "customizable_serializer_name": str, + } + ``` + """ + return { + "cached_serializer_name": f"{self.model.__name__}NangoSerializer", + "customizable_serializer_name": f"{self.model.__name__}Serializer", + } + + def generate_cached_serializer(self) -> None: + """Generate the serializer file (for the given model) that will be cached in the _nango_cache.""" + content: str = f""" + {'\n'.join(self._get_base_imports())} + {'\n'.join(self._get_drf_imports())} + {self._get_model_import()} + + class {self.get_serializer_name().get('cached_serializer_name')}(serializer.Serializer): + \"\"\"...generated...\"\"\" + + class Meta: + model = {self.model.__name__} + """ + print(content) # noqa: T201 + + def generated_customizable_serializer(self) -> None: + """Generate the serializer file (for the given model) that use the cached serializer and that can be custom by the dev.""" + def run(self) -> None: """Generate a serializer for each django's model.""" super().run() + self.generate_cached_serializer() + self.generated_customizable_serializer() diff --git a/backend/nango/utils/abstract_cog.py b/backend/nango/utils/abstract_cog.py index b2ad228..869c749 100644 --- a/backend/nango/utils/abstract_cog.py +++ b/backend/nango/utils/abstract_cog.py @@ -38,7 +38,7 @@ def is_executable(self) -> bool: """Indicate if the cog can run or not.""" return self.settings is not None - def _generate_model_import(self) -> str: + def _get_model_import(self) -> str: """Return the model import line for file generation.""" regex = r"(?<=')(.*)(?=\.)" match = re.search(regex, str(self.model)) From 2c675db178c986c2eda9128ab35f7f9d0f38d2af Mon Sep 17 00:00:00 2001 From: Xenepix Date: Mon, 25 Nov 2024 21:19:34 +0100 Subject: [PATCH 12/14] typo --- .gitignore | 1 + backend/nango/utils/cogs_runner.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d5a026e..d35293d 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ backend/.envs # backend/nango/models.py backend/api/Author backend/api/Book +backend/nango/models/test.py # ------------------------------------------------------------------------------------------------- # Frontend diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index 458c528..325c041 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -53,7 +53,7 @@ def get_settings_from_model(self, model: models.Model) -> dict[str, any]: def run_cogs(self) -> None: """Run all existing cogs.""" - cogs_classes = self.get_cogs_classes() + cogs_classes = self.get_cog_classes() for model in self.get_local_model_classes(): for cog in cogs_classes: From c5b70a6db86c3872c7a3892dd5d25a93f50b1ab9 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Mon, 25 Nov 2024 22:21:41 +0100 Subject: [PATCH 13/14] improve comments --- backend/nango/utils/cogs_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/nango/utils/cogs_runner.py b/backend/nango/utils/cogs_runner.py index 325c041..99da94e 100644 --- a/backend/nango/utils/cogs_runner.py +++ b/backend/nango/utils/cogs_runner.py @@ -59,10 +59,10 @@ def run_cogs(self) -> None: for cog in cogs_classes: model_nango_settings = self.get_settings_from_model(model) - # This model doesn't want any cogs. + # This model doesn't want to run any cogs. if model_nango_settings is None: continue - # Run the cogs + # Instantiate the cogs with settings and run it instantiated_cog = cog(model=model, settings=model_nango_settings) instantiated_cog.run() From 52170f09417721cae744b1bda22adce6f794d9a0 Mon Sep 17 00:00:00 2001 From: Xenepix Date: Mon, 25 Nov 2024 22:22:37 +0100 Subject: [PATCH 14/14] feat(serializerCog): add get_model_fields method --- backend/nango/cogs/serializer_cog.py | 51 +++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/backend/nango/cogs/serializer_cog.py b/backend/nango/cogs/serializer_cog.py index 65dd3df..3e9420f 100644 --- a/backend/nango/cogs/serializer_cog.py +++ b/backend/nango/cogs/serializer_cog.py @@ -1,3 +1,6 @@ +from django.db.models import fields as django_fields +from django.db.models.fields.reverse_related import ManyToOneRel, OneToOneRel + from nango.utils import AbstractCog @@ -6,11 +9,19 @@ class SerializerCog(AbstractCog): Settings: -------- - - speedy: A faster serializer than rest_framework.serializers.Serializer, but with less features. + - speedy (bool ; default = False): A faster serializer than rest_framework.serializers.Serializer, but with less features. + - selected_fields (list[str] ; default = []): list of selected fields' name to serialize. + - excluded_fields (list[str] ; default = []): list of fields' name to not serialize. """ id = "serializer" + # Fields to NEVER serialize. + _forbidden_fields: tuple[str] = ( + "password", + "outstandingtoken", + ) + def _get_drf_imports(self) -> list[str]: """Return list of imports from rest_framework module.""" if self.settings.get("speedy", False): @@ -28,6 +39,42 @@ def _get_base_imports(self) -> list[str]: "from typing import ClassVar", ] + def get_model_fields(self) -> dict[str, django_fields.Field]: + """Return the model fields to serialize. + + Returns: + ------- + ``` + { + "field_name": field, + ..., + } + ``` + """ + selected_fields_name: list[str] = self.settings.get("selected_fields", []) + if selected_fields_name: + return {field_name: getattr(self.model, field_name) for field_name in selected_fields_name} + + model_fields_list: dict[str, django_fields.Field] = {} + + # Select all fields except excluded_fields + excluded_fields_name: list[str] = self.settings.get("excluded_fields", []) + for field in self.model._meta.get_fields(): # noqa: SLF001 + # Get field's name + if isinstance(field, ManyToOneRel | OneToOneRel): + # External Key point at actual model + field_name = field.related_name if field.related_name is not None else f"{field.related_model.__name__.lower()}" + else: + field_name = getattr(field, "field_name", None) or getattr(field, "name", None) + + if field_name in excluded_fields_name or field_name in self._forbidden_fields: + continue + + # Basic behavior + model_fields_list[field_name] = field + + return model_fields_list + def get_serializer_name(self) -> dict[str]: """Return name of generated serializers. @@ -66,5 +113,7 @@ def generated_customizable_serializer(self) -> None: def run(self) -> None: """Generate a serializer for each django's model.""" super().run() + self._get_model_fields() + self.generate_cached_serializer() self.generated_customizable_serializer()