datasources (#52)

* wip

* seems to be done

* working views

* debug_toolbar always appear

* fixes

* scripts

* most of the tests fixed

* tests done

* ci fix

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        netbox_version: [v3.4.10, v3.5.9, v3.6.2]
+        netbox_version: [v3.5.9, v3.6.5]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -65,7 +65,7 @@ jobs:
       runs-on: ubuntu-latest
       strategy:
         matrix:
-          netbox_version: [v3.4.10, v3.5.3]
+          netbox_version: [v3.5.9, v3.6.5]
       steps:
         - name: Checkout
           uses: actions/checkout@v3

development/configuration.py

@@ -1,5 +1,4 @@
 import os
-import socket
 
 from .configuration_example import *
 
@@ -63,6 +62,14 @@
 
 PLUGINS_CONFIG = {"validity": {"store_last_results": 5, "git_folder": "/opt/git_repos/", "autocopy_scripts": True}}
 
+
 # for debug toolbar
-_, _, _ips = socket.gethostbyname_ex(socket.gethostname())
-INTERNAL_IPS += tuple(ip[: ip.rfind(".")] + ".1" for ip in _ips)
+class ContainsAll:
+    def __contains__(self, v):
+        return True
+
+
+INTERNAL_IPS = ContainsAll()
+
+
+CUSTOM_VALIDATORS = {"core.datasource": ["validity.custom_validators.DataSourceValidator"]}

requirements/base.txt

@@ -1,7 +1,8 @@
 django-bootstrap-v5==1.0.*
 pydantic==1.10.*
 ttp==0.9.*
-pygit2==1.11.*
 jq==1.4.*
 deepdiff==6.2.*
 simpleeval==0.9.*
+
+dulwich # Core NetBox "optional" requirement

validity/__init__.py

@@ -1,13 +1,12 @@
 import logging
-import os
 from pathlib import Path
 
 from django.conf import settings as django_settings
 from extras.plugins import PluginConfig
 from netbox.settings import VERSION
 from pydantic import BaseModel, DirectoryPath, Field
 
-from validity.utils.misc import NetboxVersion
+from validity.utils.version import NetboxVersion
 
 
 logger = logging.getLogger(__name__)
@@ -27,14 +26,6 @@ class NetBoxValidityConfig(PluginConfig):
     # custom field
     netbox_version = NetboxVersion(VERSION)
 
-    def ready(self):
-        try:
-            os.makedirs(settings.git_folder, exist_ok=True)
-        except OSError as e:
-            if not settings.git_folder.is_dir():
-                logger.error("Cannot create git_folder (%s), %s: %s", settings.git_folder, type(e).__name__, e)
-        return super().ready()
-
 
 config = NetBoxValidityConfig
 

validity/api/helpers.py

@@ -2,7 +2,7 @@
 from typing import Sequence
 
 from netbox.api.serializers import WritableNestedSerializer
-from rest_framework.serializers import CharField, ModelSerializer
+from rest_framework.serializers import ModelSerializer
 
 
 def nested_factory(
@@ -22,8 +22,3 @@ class Meta:
         bases,
         s_attribs,
     )
-
-
-class PasswordField(CharField):
-    def to_representation(self, value):
-        return "$encrypted"

validity/api/serializers.py

@@ -1,5 +1,6 @@
 from urllib.parse import urljoin
 
+from core.api.nested_serializers import NestedDataFileSerializer, NestedDataSourceSerializer
 from dcim.api.nested_serializers import (
     NestedDeviceSerializer,
     NestedDeviceTypeSerializer,
@@ -19,7 +20,7 @@
 from tenancy.models import Tenant
 
 from validity import models
-from .helpers import PasswordField, nested_factory
+from .helpers import nested_factory
 
 
 class ComplianceSelectorSerializer(NetBoxModelSerializer):
@@ -78,36 +79,6 @@ class Meta:
 NestedComplianceSelectorSerializer = nested_factory(ComplianceSelectorSerializer, ("id", "url", "display", "name"))
 
 
-class GitRepoSerializer(NetBoxModelSerializer):
-    url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:gitrepo-detail")
-    head_hash = serializers.ReadOnlyField()
-    password = PasswordField(required=False)
-
-    class Meta:
-        model = models.GitRepo
-        fields = (
-            "id",
-            "url",
-            "display",
-            "name",
-            "git_url",
-            "web_url",
-            "device_config_path",
-            "default",
-            "username",
-            "password",
-            "branch",
-            "head_hash",
-            "tags",
-            "custom_fields",
-            "created",
-            "last_updated",
-        )
-
-
-NestedGitRepoSerializer = nested_factory(GitRepoSerializer, ("id", "url", "display", "name", "default"))
-
-
 class ComplianceTestSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:compliancetest-detail")
     selectors = SerializedPKRelatedField(
@@ -116,7 +87,8 @@ class ComplianceTestSerializer(NetBoxModelSerializer):
         required=False,
         queryset=models.ComplianceSelector.objects.all(),
     )
-    repo = NestedGitRepoSerializer(required=False)
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
     effective_expression = serializers.ReadOnlyField()
     expression = serializers.CharField(write_only=True, required=False)
 
@@ -129,10 +101,10 @@ class Meta:
             "name",
             "severity",
             "description",
-            "repo",
-            "file_path",
             "effective_expression",
             "expression",
+            "data_source",
+            "data_file",
             "selectors",
             "tags",
             "custom_fields",
@@ -220,9 +192,10 @@ class Meta:
 
 class ConfigSerializerSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:configserializer-detail")
-    repo = NestedGitRepoSerializer(required=False)
     ttp_template = serializers.CharField(write_only=True, required=False)
     effective_template = serializers.ReadOnlyField()
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
 
     class Meta:
         model = models.ConfigSerializer
@@ -232,10 +205,10 @@ class Meta:
             "display",
             "name",
             "extraction_method",
-            "repo",
-            "file_path",
             "effective_template",
             "ttp_template",
+            "data_source",
+            "data_file",
             "tags",
             "custom_fields",
             "created",
@@ -248,7 +221,8 @@ class Meta:
 
 class NameSetSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:nameset-detail")
-    repo = NestedGitRepoSerializer(required=False)
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
     definitions = serializers.CharField(write_only=True, required=False)
     effective_definitions = serializers.ReadOnlyField()
 
@@ -262,8 +236,8 @@ class Meta:
             "description",
             "_global",
             "tests",
-            "repo",
-            "file_path",
+            "data_source",
+            "data_file",
             "definitions",
             "effective_definitions",
             "tags",
@@ -288,13 +262,14 @@ def run_validation(self, data=...):
 
 class SerializedConfigSerializer(serializers.Serializer):
     serializer = NestedConfigSerializerSerializer(read_only=True, source="device.serializer")
-    repo = NestedGitRepoSerializer(read_only=True, source="device.repo")
+    data_source = NestedDataSourceSerializer(read_only=True, source="device.data_source")
+    data_file = NestedDataFileSerializer(read_only=True, source="device.data_file")
     local_copy_last_updated = serializers.DateTimeField(allow_null=True, source="last_modified")
     config_web_link = serializers.SerializerMethodField()
     serialized_config = serializers.JSONField(source="serialized")
 
     def get_config_web_link(self, obj):
-        return urljoin(obj.device.repo.web_url, obj.config_path.as_posix())
+        return urljoin(obj.device.data_source.web_url, obj.device.config_path)
 
 
 class DeviceReportSerializer(NestedDeviceSerializer):

validity/api/urls.py

@@ -10,7 +10,6 @@
 router.register("selectors", views.ComplianceSelectorViewSet)
 router.register("tests", views.ComplianceTestViewSet)
 router.register("test-results", views.ComplianceTestResultViewSet)
-router.register("git-repositories", views.GitRepoViewSet)
 router.register("serializers", views.ConfigSerializerViewSet)
 router.register("namesets", views.NameSetViewSet)
 router.register("reports", views.ComplianceReportViewSet)

validity/api/views.py

@@ -7,9 +7,9 @@
 from rest_framework.views import APIView
 
 from validity import config, filtersets, models
+from validity.choices import SeverityChoices
 from validity.config_compliance.device_config import DeviceConfig
 from validity.config_compliance.exceptions import DeviceConfigError
-from ..choices import SeverityChoices
 from . import serializers
 
 
@@ -39,7 +39,9 @@ class ComplianceSelectorViewSet(NetBoxModelViewSet):
 
 
 class ComplianceTestViewSet(NetBoxModelViewSet):
-    queryset = models.ComplianceTest.objects.select_related("repo").prefetch_related("selectors", "tags")
+    queryset = models.ComplianceTest.objects.select_related("data_source", "data_file").prefetch_related(
+        "selectors", "tags"
+    )
     serializer_class = serializers.ComplianceTestSerializer
     filterset_class = filtersets.ComplianceTestFilterSet
 
@@ -50,20 +52,14 @@ class ComplianceTestResultViewSet(ReadOnlyNetboxViewSet):
     filterset_class = filtersets.ComplianceTestResultFilterSet
 
 
-class GitRepoViewSet(NetBoxModelViewSet):
-    queryset = models.GitRepo.objects.prefetch_related("tags")
-    serializer_class = serializers.GitRepoSerializer
-    filterset_class = filtersets.GitRepoFilterSet
-
-
 class ConfigSerializerViewSet(NetBoxModelViewSet):
-    queryset = models.ConfigSerializer.objects.select_related("repo").prefetch_related("tags")
+    queryset = models.ConfigSerializer.objects.select_related("data_source", "data_file").prefetch_related("tags")
     serializer_class = serializers.ConfigSerializerSerializer
     filterset_class = filtersets.ConfigSerializerFilterSet
 
 
 class NameSetViewSet(NetBoxModelViewSet):
-    queryset = models.NameSet.objects.select_related("repo").prefetch_related("tags")
+    queryset = models.NameSet.objects.select_related("data_source", "data_file").prefetch_related("tags")
     serializer_class = serializers.NameSetSerializer
     filterset_class = filtersets.NameSetFilterSet
 
@@ -89,7 +85,7 @@ def get_queryset(self):
 
 
 class SerializedConfigView(APIView):
-    queryset = models.VDevice.objects.all()
+    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer()
 
     def get_object(self, pk):
         try:

validity/config_compliance/device_config/base.py

@@ -1,53 +1,44 @@
 from abc import abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
-from pathlib import Path
-from typing import ClassVar
+from typing import TYPE_CHECKING, ClassVar
 
-from dcim.models import Device
-from django.utils.timezone import make_aware
-
-from validity import settings
 from validity.utils.misc import reraise
 from ..exceptions import DeviceConfigError
 
 
+if TYPE_CHECKING:
+    from validity.models import VDevice
+
+
 @dataclass
 class BaseDeviceConfig:
-    device: Device
-    config_path: Path
+    device: "VDevice"
+    plain_config: str
     last_modified: datetime | None = None
     serialized: dict | list | None = None
-    _git_folder: ClassVar[Path] = settings.git_folder
-    _config_classes: ClassVar[dict[str, type]] = {}
 
-    @classmethod
-    def _full_config_path(cls, device: Device) -> Path:
-        return cls._git_folder / device.repo.name / device.repo.rendered_device_path(device)
+    _config_classes: ClassVar[dict[str, type]] = {}
 
     @classmethod
-    def from_device(cls, device: Device) -> "BaseDeviceConfig":
+    def from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
         """
         Get DeviceConfig from dcim.models.Device
-        Device MUST be annotated with ".repo" pointing to a repo with device config file
+        Device MUST be annotated with ".data_file"
         Device MUST be annotated with ".serializer" pointing to appropriate config serializer instance
         """
-        with reraise((AssertionError, FileNotFoundError), DeviceConfigError):
-            assert getattr(device, "repo", None), f"{device} has no bound repository"
+        with reraise((AssertionError, FileNotFoundError, AttributeError), DeviceConfigError):
+            assert getattr(
+                device, "data_file", None
+            ), f"{device} has no bound data file. Either no data source bound or the file does not exist"
             assert getattr(device, "serializer", None), f"{device} has no bound serializer"
             return cls._config_classes[device.serializer.extraction_method]._from_device(device)
 
     @classmethod
-    def _from_device(cls, device: Device) -> "BaseDeviceConfig":
-        with reraise(AttributeError, DeviceConfigError):
-            device_path = cls._full_config_path(device)
-            last_modified = None
-            if device_path.is_file():
-                lm_timestamp = device_path.stat().st_mtime
-                last_modified = make_aware(datetime.fromtimestamp(lm_timestamp))
-            instance = cls(device, device_path, last_modified)
-            instance.serialize()
-            return instance
+    def _from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
+        instance = cls(device, device.data_file.data_as_string, device.data_file.last_updated)
+        instance.serialize()
+        return instance
 
     @abstractmethod
     def serialize(self, override: bool = False) -> None: