backup backend + crud

requirements/base.txt

@@ -1,3 +1,4 @@
+boto3<2
 deepdiff>=6.2.0,<7
 dimi >=1.3.0,< 2
 django-bootstrap5 >=24.2,<25

validity/api/helpers.py

@@ -103,7 +103,7 @@ def _validate(self, attrs):
                 setattr(instance, field, field_value)
         if not instance.subform_type:
             return
-        subform = instance.subform_cls(instance.subform_json)
+        subform = instance.get_subform()
         if not subform.is_valid():
             errors = [
                 ": ".join((field, err[0])) if field != "__all__" else err for field, err in subform.errors.items()

validity/api/serializers.py

@@ -377,6 +377,32 @@ def validate(self, data, command_types: Annotated[dict[str, list[str]], "PollerC
 NestedPollerSerializer = nested_factory(PollerSerializer, nb_version=config.netbox_version)
 
 
+class BackupPointSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:backuppoint-detail")
+    data_source = NestedDataSourceSerializer()
+    parameters = EncryptedDictField()
+
+    class Meta:
+        model = models.BackupPoint
+        fields = (
+            "id",
+            "url",
+            "display",
+            "name",
+            "data_source",
+            "backup_after_sync",
+            "method",
+            "url",
+            "ignore_rules",
+            "parameters",
+            "last_uploaded",
+            "tags",
+            "custom_fields",
+            "created",
+            "last_updated",
+        )
+
+
 class SerializedStateItemSerializer(FieldsMixin, serializers.Serializer):
     name = serializers.CharField(read_only=True)
     serializer = NestedSerializerSerializer(read_only=True)

validity/api/urls.py

@@ -15,6 +15,7 @@
 router.register("reports", views.ComplianceReportViewSet)
 router.register("pollers", views.PollerViewSet)
 router.register("commands", views.CommandViewSet)
+router.register("backup-points", views.BackupPointViewSet)
 
 
 urlpatterns = [

validity/api/views.py

@@ -101,6 +101,12 @@ class PollerViewSet(NetBoxModelViewSet):
     filterset_class = filtersets.PollerFilterSet
 
 
+class BackupPointViewSet(NetBoxModelViewSet):
+    queryset = models.BackupPoint.objects.select_related("data_source")
+    serializer_class = serializers.BackupPointSerializer
+    filterset_class = filtersets.BackupPointFilterSet
+
+
 class CommandViewSet(NetBoxModelViewSet):
     queryset = models.Command.objects.select_related("serializer").prefetch_related("tags")
     serializer_class = serializers.CommandSerializer

validity/choices.py

@@ -127,3 +127,8 @@ class JSONAPIMethodChoices(TextChoices):
     POST = "POST"
     PATCH = "PATCH"
     PUT = "PUT"
+
+
+class BackupMethodChoices(TextChoices, metaclass=ColoredChoiceMeta):
+    git = "git", "blue"
+    S3 = "S3", "Amazon S3", "yellow"

validity/data_backends.py

@@ -27,6 +27,7 @@ class PollingBackend(DataBackend):
         "datasource_id": forms.IntegerField(
             label=_("Data Source ID"),
             widget=forms.TextInput(attrs={"class": "form-control"}),
+            help_text=_("Must match the primary key of the data source"),
         )
     }
 

validity/data_backup/__init__.py

@@ -0,0 +1,2 @@
+from .backend import BackupBackend
+from .backupers import Backuper, GitBackuper, S3Backuper

validity/data_backup/backend.py

@@ -0,0 +1,26 @@
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from validity.models import BackupPoint
+    from .backupers import Backuper
+
+
+class BackupBackend:
+    def __init__(self, backupers: dict[str, "Backuper"]):
+        self.backupers = backupers
+
+    @contextmanager
+    def _datasource_in_filesytem(self, backup_point: "BackupPoint"):
+        with TemporaryDirectory() as datasource_dir:
+            for file in backup_point.data_source.datafiles.all():
+                if not backup_point.ignore_file(file.path):
+                    file.write_to_disk(datasource_dir, overwrite=True)
+            yield datasource_dir
+
+    def __call__(self, backup_point: "BackupPoint") -> None:
+        backuper = self.backupers[backup_point.method]
+        with self._datasource_in_filesytem(backup_point) as datasource_dir:
+            backuper(backup_point.url, backup_point.parameters, datasource_dir)

validity/data_backup/backupers.py

@@ -0,0 +1,74 @@
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, ClassVar
+
+from pydantic import BaseModel
+
+from validity.integrations.git import GitClient
+from validity.integrations.s3 import S3Client
+from validity.utils.filesystem import merge_directories
+from .entities import RemoteGitRepo
+from .parameters import GitParams, S3Params
+
+
+class Backuper(ABC):
+    parameters_cls: type[BaseModel]
+
+    def __call__(self, url: str, parameters: dict[str, Any], datasource_dir: Path) -> None:
+        validated_params = self.parameters_cls.model_validate(parameters)
+        self._do_backup(url, validated_params, datasource_dir)
+
+    @abstractmethod
+    def _do_backup(self, url: str, parameters: BaseModel, datasource_dir: Path) -> None: ...
+
+
+@dataclass
+class GitBackuper(Backuper):
+    message: str
+    author_username: str
+    author_email: str
+    git_client: GitClient
+
+    parameters_cls: ClassVar[type[BaseModel]] = GitParams
+
+    def _do_backup(self, url: str, parameters: GitParams, datasource_dir: Path) -> None:
+        with TemporaryDirectory() as repo_dir:
+            repo = RemoteGitRepo(
+                local_path=repo_dir,
+                remote_url=url,
+                active_branch=parameters.branch,
+                username=parameters.username,
+                password=parameters.password,
+                client=self.git_client,
+            )
+            repo.download()
+            merge_directories(datasource_dir, repo.local_path)
+            repo.save_changes(self.author_username, self.author_email, message=self.message)
+            repo.upload()
+
+
+@dataclass
+class S3Backuper(Backuper):
+    s3_client: S3Client
+
+    parameters_cls: ClassVar[type[BaseModel]] = S3Params
+
+    def _backup_archive(self, url: str, parameters: S3Params, datasource_dir: Path) -> None:
+        with TemporaryDirectory() as backup_dir:
+            archive = Path(backup_dir) / "a.zip"
+            shutil.make_archive(archive, "zip", datasource_dir)
+            self.s3_client.upload_file(archive, url, parameters.aws_access_key_id, parameters.aws_secret_access_key)
+
+    def _backup_dir(self, url: str, parameters: S3Params, datasource_dir: Path) -> None:
+        self.s3_client.upload_folder(
+            datasource_dir, url, parameters.aws_access_key_id, parameters.aws_secret_access_key
+        )
+
+    def _do_backup(self, url: str, parameters: S3Params, datasource_dir: Path):
+        if parameters.archive:
+            self._backup_archive(url, parameters, datasource_dir)
+        else:
+            self._backup_dir(url, parameters, datasource_dir)

validity/data_backup/entities.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+
+from validity.integrations.git import GitClient
+
+
+@dataclass(slots=True, kw_only=True)
+class RemoteGitRepo:
+    local_path: str
+    remote_url: str
+    active_branch: str
+    username: str = ""
+    password: str = ""
+    client: GitClient
+
+    def save_changes(self, author_username: str, author_email: str, message: str = ""):
+        self.client.stage_all(self.local_path)
+        self.client.commit(self.local_path, author_username, author_email, message)
+
+    def download(self):
+        self.client.clone(self.local_path, self.remote_url, self.active_branch, self.username, self.password, depth=1)
+
+    def upload(self):
+        self.client.push(self.local_path, self.remote_url, self.active_branch, self.username, self.password)

validity/data_backup/parameters.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class GitParams(BaseModel):
+    username: str
+    password: str
+    branch: str | None = None
+
+
+class S3Params(BaseModel):
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    archive: bool

validity/dependencies.py

@@ -8,6 +8,17 @@
 from rq.job import Job
 
 from validity import di
+from validity.compliance.serialization import (
+    SerializationBackend,
+    serialize_ros,
+    serialize_textfsm,
+    serialize_ttp,
+    serialize_xml,
+    serialize_yaml,
+)
+from validity.data_backup import BackupBackend, GitBackuper, S3Backuper
+from validity.integrations.git import DulwichGitClient
+from validity.integrations.s3 import BotoS3Client
 from validity.pollers import NetmikoPoller, RequestsPoller, ScrapliNetconfPoller
 from validity.settings import PollerInfo, ValiditySettings
 from validity.utils.misc import null_request
@@ -18,11 +29,39 @@ def django_settings():
     return settings
 
 
-@di.dependency(scope=Singleton)
-def validity_settings(django_settings: Annotated[LazySettings, django_settings]):
+@di.dependency(scope=Singleton, add_return_alias=True)
+def validity_settings(django_settings: Annotated[LazySettings, django_settings]) -> ValiditySettings:
     return ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))
 
 
+@di.dependency(scope=Singleton, add_return_alias=True)
+def backup_backend(vsettings: Annotated[ValiditySettings, ...]) -> BackupBackend:
+    return BackupBackend(
+        backupers={
+            "git": GitBackuper(
+                message="",
+                author_username=vsettings.integrations.git.author,
+                author_email=vsettings.integrations.git.email,
+                git_client=DulwichGitClient(),
+            ),
+            "S3": S3Backuper(s3_client=BotoS3Client(max_threads=vsettings.integrations.s3.threads)),
+        }
+    )
+
+
+@di.dependency(scope=Singleton, add_return_alias=True)
+def serialization_backend() -> SerializationBackend:
+    return SerializationBackend(
+        extraction_methods={
+            "YAML": serialize_yaml,
+            "ROUTEROS": serialize_ros,
+            "TTP": serialize_ttp,
+            "TEXTFSM": serialize_textfsm,
+            "XML": serialize_xml,
+        }
+    )
+
+
 @di.dependency(scope=Singleton)
 def pollers_info(custom_pollers: Annotated[list[PollerInfo], "validity_settings.custom_pollers"]) -> list[PollerInfo]:
     return [