From 9e6bf1718558e026356de77a8c66e6c90beac856 Mon Sep 17 00:00:00 2001 From: Anton M Date: Mon, 1 Jan 2024 17:59:18 +0400 Subject: [PATCH] device state, working views --- validity/api/serializers.py | 2 + validity/api/views.py | 6 +- validity/compliance/device_config/base.py | 4 +- validity/compliance/device_config/routeros.py | 6 +- validity/compliance/device_config/ttp.py | 4 +- validity/compliance/device_config/yaml.py | 4 +- validity/compliance/exceptions.py | 19 ++- validity/compliance/serialization/__init__.py | 10 ++ validity/compliance/serialization/backend.py | 10 ++ validity/compliance/serialization/routeros.py | 151 ++++++++++++++++++ .../compliance/serialization/serializable.py | 27 ++++ validity/compliance/serialization/ttp.py | 11 ++ validity/compliance/serialization/yaml.py | 9 ++ validity/compliance/state.py | 85 ++++++++++ validity/data_backends.py | 2 +- validity/filtersets.py | 5 +- validity/forms/__init__.py | 1 + validity/forms/filterset.py | 14 ++ validity/forms/general.py | 8 +- validity/managers.py | 50 +++--- validity/migrations/0007_polling.py | 11 +- validity/models/data.py | 10 +- validity/models/device.py | 64 ++++---- validity/models/polling.py | 7 +- validity/models/serializer.py | 5 + validity/scripts/run_tests.py | 4 +- validity/tables.py | 3 +- validity/templates/validity/command.html | 5 + .../{device_config.html => device_state.html} | 40 +++-- .../validity/inc/datasource_link.html | 2 +- .../validity/inc/path_with_link.html | 5 +- validity/utils/orm.py | 103 ++++++++---- validity/views/__init__.py | 2 +- validity/views/device.py | 30 ++-- 34 files changed, 586 insertions(+), 133 deletions(-) create mode 100644 validity/compliance/serialization/__init__.py create mode 100644 validity/compliance/serialization/backend.py create mode 100644 validity/compliance/serialization/routeros.py create mode 100644 validity/compliance/serialization/serializable.py create mode 100644 validity/compliance/serialization/ttp.py create mode 100644 validity/compliance/serialization/yaml.py create mode 100644 validity/compliance/state.py rename validity/templates/validity/{device_config.html => device_state.html} (55%) diff --git a/validity/api/serializers.py b/validity/api/serializers.py index fee438e..66777fb 100644 --- a/validity/api/serializers.py +++ b/validity/api/serializers.py @@ -290,6 +290,7 @@ class Meta(NestedDeviceSerializer.Meta): class CommandSerializer(NetBoxModelSerializer): + serializer = NestedSerializerSerializer(required=False) url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:command-detail") class Meta: @@ -301,6 +302,7 @@ class Meta: "name", "label", "retrieves_config", + "serializer", "type", "parameters", "tags", diff --git a/validity/api/views.py b/validity/api/views.py index bbcdba9..f2c3c64 100644 --- a/validity/api/views.py +++ b/validity/api/views.py @@ -8,7 +8,7 @@ from validity import config, filtersets, models from validity.choices import SeverityChoices -from validity.compliance.exceptions import DeviceConfigError +from validity.compliance.exceptions import SerializationError from . import serializers @@ -76,7 +76,7 @@ class PollerViewSet(NetBoxModelViewSet): class CommandViewSet(NetBoxModelViewSet): - queryset = models.Command.objects.prefetch_related("tags") + queryset = models.Command.objects.select_related("serializer").prefetch_related("tags") serializer_class = serializers.CommandSerializer filterset_class = filtersets.CommandFilterSet @@ -113,7 +113,7 @@ def get(self, request, pk): try: serializer = serializers.SerializedConfigSerializer(device.device_config, context={"request": request}) return Response(serializer.data) - except DeviceConfigError as e: + except SerializationError as e: return Response( data={"detail": "Unable to fetch serialized config", "error": str(e)}, status=HTTPStatus.BAD_REQUEST ) diff --git a/validity/compliance/device_config/base.py b/validity/compliance/device_config/base.py index 3fa9663..3c4d8fe 100644 --- a/validity/compliance/device_config/base.py +++ b/validity/compliance/device_config/base.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, ClassVar from validity.utils.misc import reraise -from ..exceptions import DeviceConfigError +from ..exceptions import SerializationError if TYPE_CHECKING: @@ -27,7 +27,7 @@ def from_device(cls, device: "VDevice") -> "BaseDeviceConfig": Device MUST be annotated with ".data_file" Device MUST be annotated with ".serializer" pointing to appropriate config serializer instance """ - with reraise((AssertionError, FileNotFoundError, AttributeError), DeviceConfigError): + with reraise((AssertionError, FileNotFoundError, AttributeError), SerializationError): assert getattr( device, "data_file", None ), f"{device} has no bound data file. Either there is no data source attached or the file does not exist" diff --git a/validity/compliance/device_config/routeros.py b/validity/compliance/device_config/routeros.py index 7621d7c..8d88804 100644 --- a/validity/compliance/device_config/routeros.py +++ b/validity/compliance/device_config/routeros.py @@ -5,14 +5,14 @@ from typing import ClassVar, Generator, Literal from validity.utils.misc import reraise -from ..exceptions import DeviceConfigError +from ..exceptions import SerializationError from .base import DeviceConfig logger = logging.getLogger(__name__) -class LineParsingError(DeviceConfigError): +class LineParsingError(SerializationError): pass @@ -152,5 +152,5 @@ class RouterOSDeviceConfig(DeviceConfig): def serialize(self, override: bool = False) -> None: if not self.serialized or override: - with reraise(Exception, DeviceConfigError): + with reraise(Exception, SerializationError): self.serialized = parse_config(self.plain_config) diff --git a/validity/compliance/device_config/ttp.py b/validity/compliance/device_config/ttp.py index 23f4e7f..382159a 100644 --- a/validity/compliance/device_config/ttp.py +++ b/validity/compliance/device_config/ttp.py @@ -4,7 +4,7 @@ from ttp import ttp from validity.utils.misc import reraise -from ..exceptions import DeviceConfigError +from ..exceptions import SerializationError from .base import DeviceConfig @@ -28,5 +28,5 @@ def serialize(self, override: bool = False) -> None: if not self.serialized or override: parser = ttp(data=self.plain_config, template=self._template.template) parser.parse() - with reraise(IndexError, DeviceConfigError, f"Invalid parsed config for {self.device}: {parser.result()}"): + with reraise(IndexError, SerializationError, f"Invalid parsed config for {self.device}: {parser.result()}"): self.serialized = parser.result()[0][0] diff --git a/validity/compliance/device_config/yaml.py b/validity/compliance/device_config/yaml.py index 5094e93..f2867de 100644 --- a/validity/compliance/device_config/yaml.py +++ b/validity/compliance/device_config/yaml.py @@ -3,7 +3,7 @@ import yaml from validity.utils.misc import reraise -from ..exceptions import DeviceConfigError +from ..exceptions import SerializationError from .base import DeviceConfig @@ -14,7 +14,7 @@ def serialize(self, override: bool = False) -> None: if not self.serialized or override: with reraise( yaml.YAMLError, - DeviceConfigError, + SerializationError, f"Trying to parse invalid YAML as device config for {self.device}", ): self.serialized = yaml.safe_load(self.plain_config) diff --git a/validity/compliance/exceptions.py b/validity/compliance/exceptions.py index 0ab3f65..c582211 100644 --- a/validity/compliance/exceptions.py +++ b/validity/compliance/exceptions.py @@ -6,5 +6,22 @@ def __str__(self) -> str: return f"{type(self.orig_error).__name__}: {self.orig_error}" -class DeviceConfigError(Exception): +class SerializationError(Exception): + pass + + +class NoComponentError(SerializationError): + """ + Indicates lack of the required component (e.g. serializer) to do serialization + """ + + def __init__(self, missing_component: str) -> None: + self.missing_component = missing_component + super().__init__() + + def __str__(self) -> str: + return f"There is no bound {self.missing_component}" + + +class BadDataFileContentsError(SerializationError): pass diff --git a/validity/compliance/serialization/__init__.py b/validity/compliance/serialization/__init__.py new file mode 100644 index 0000000..0a24443 --- /dev/null +++ b/validity/compliance/serialization/__init__.py @@ -0,0 +1,10 @@ +from .backend import SerializationBackend +from .routeros import serialize_ros +from .serializable import Serializable +from .ttp import serialize_ttp +from .yaml import serialize_yaml + + +serialize = SerializationBackend( + extraction_methods={"YAML": serialize_yaml, "ROUTEROS": serialize_ros, "TTP": serialize_ttp} +) diff --git a/validity/compliance/serialization/backend.py b/validity/compliance/serialization/backend.py new file mode 100644 index 0000000..8a728e8 --- /dev/null +++ b/validity/compliance/serialization/backend.py @@ -0,0 +1,10 @@ +from typing import Callable + + +class SerializationBackend: + def __init__(self, extraction_methods: dict[str, Callable[[str, str], dict]]) -> None: + self.extraction_methods = extraction_methods + + def __call__(self, extraction_method: str, plain_data: str, template: str): + extraction_function = self.extraction_methods[extraction_method] + return extraction_function(plain_data, template) diff --git a/validity/compliance/serialization/routeros.py b/validity/compliance/serialization/routeros.py new file mode 100644 index 0000000..f50c064 --- /dev/null +++ b/validity/compliance/serialization/routeros.py @@ -0,0 +1,151 @@ +import io +import logging +import re +from dataclasses import dataclass, field +from typing import Generator, Literal + +from validity.utils.misc import reraise +from ..exceptions import SerializationError + + +logger = logging.getLogger(__name__) + + +class LineParsingError(SerializationError): + pass + + +def non_quoted_characters(line: str) -> Generator[tuple[int, str], None, None]: + """ + Generator returns pairs (char_position, char) for each char in line not placed inside the quotes + Quoted substring will be returned as 1 single character with char_position equal to first character + """ + + quote_open = False + quote_start = -1 + for i, char in enumerate(line): + if char == '"' and (not i or line[i - 1] != "\\"): + quote_open = not quote_open + if quote_open: + quote_start = i + else: + yield i, line[quote_start : i + 1] + continue + if quote_open: + continue + yield i, char + + +@dataclass +class ParsedLine: + method: Literal["add", "set"] + find_by: tuple[str, str] | tuple[()] = () + properties: dict[str, str] = field(default_factory=dict) + implicit_name: bool = False + + @classmethod + def _extract_find(cls, line: str) -> tuple[tuple[str, str | int | bool], str]: + find, line = line.split("]", maxsplit=1) + find = find[1:].replace("find", "", 1).strip() + find_key, find_value = find.split("=", maxsplit=1) + find_value = cls._transform_value(find_key, find_value) + return (find_key, find_value), line + + @staticmethod + def _replace_line_breaks(line: str) -> str: + drop_match = re.compile(r"\\\n +") + new_line = [] + backslash_seq = [] + for _, char in non_quoted_characters(line): + if char == "\\" or char in {"\n", " "} and backslash_seq: + backslash_seq.append(char) + continue + if not drop_match.fullmatch("".join(backslash_seq)): + new_line.extend(backslash_seq) + backslash_seq = [] + new_line.append(char) + return "".join(new_line) + + @staticmethod + def _transform_value(key: str, value: str) -> str | int | bool: + if value and len(value) > 2 and value[0] == '"' and value[-1] == '"': + value = value[1:-1] + if key in {"name", "comment"}: + return value + if value.isdigit(): + return int(value) + booleans = {"yes": True, "no": False} + if value in booleans: + return booleans[value] + return value + + @classmethod + def from_plain_text(cls, line: str) -> "ParsedLine": + method, line = line.split(maxsplit=1) + if method not in {"add", "set"}: + raise LineParsingError("Unknown line") + find = () + if line.startswith("["): + find, line = cls._extract_find(line) + properties = {} + sub_start = 0 + implicit_name = False + line = cls._replace_line_breaks(line).strip() + for char_num, char in non_quoted_characters(line + " "): + if char == " ": + kvline = line[sub_start:char_num].strip(" \n") + if kvline and "=" not in kvline: + kvline = "name=" + kvline + implicit_name = True + with reraise(ValueError, LineParsingError, f'"{kvline}" cannot be split into key/value'): + key, value = kvline.split("=", maxsplit=1) + properties[key] = cls._transform_value(key, value) + sub_start = char_num + 1 + return cls(method=method, find_by=find, properties=properties, implicit_name=implicit_name) + + +def parse_config(plain_config: str) -> dict: + result = {} + context_path = [] + prevlines = [] + cfgfile = io.StringIO(plain_config) + for line_num, line in enumerate(cfgfile, start=1): + if line.startswith(("#", ":")) or line == "\n": + continue + if line.startswith("/"): + context_path = line[1:-1].split() + continue + if line.endswith("\\\n"): + prevlines.append(line) + continue + if prevlines: + line = "".join(prevlines) + line + prevlines = [] + try: + parsed_line = ParsedLine.from_plain_text(line) + except LineParsingError as e: + e.args = (e.args[0] + f", config line {line_num}",) + e.args[1:] + raise + current_context = result + for key in context_path: + try: + current_context = current_context[key] + except KeyError: + current_context[key] = {} + current_context = current_context[key] + if parsed_line.find_by or parsed_line.method == "add" or parsed_line.implicit_name: + if "values" not in current_context: + current_context["values"] = [] + current_context["values"].append(parsed_line.properties) + if parsed_line.find_by: + current_context["values"][-1]["find_by"] = [ + {"key": parsed_line.find_by[0], "value": parsed_line.find_by[1]} + ] + else: + current_context["properties"] = parsed_line.properties + return result + + +def serialize_ros(plain_data: str, template: str = ""): + with reraise(Exception, SerializationError): + return parse_config(plain_data) diff --git a/validity/compliance/serialization/serializable.py b/validity/compliance/serialization/serializable.py new file mode 100644 index 0000000..8964877 --- /dev/null +++ b/validity/compliance/serialization/serializable.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass +from functools import cached_property +from typing import TYPE_CHECKING, Optional + +from core.models import DataFile + +from ..exceptions import BadDataFileContentsError, NoComponentError + + +if TYPE_CHECKING: + from validity.models import Serializer + + +@dataclass(frozen=True) +class Serializable: + serializer: Optional["Serializer"] + data_file: DataFile | None + + @cached_property + def serialized(self): + if self.data_file is None: + raise NoComponentError("Data File") + if self.serializer is None: + raise NoComponentError("Serializer") + if (file_data := self.data_file.data_as_string) is not None: + return self.serializer.serialize(file_data) + raise BadDataFileContentsError(f"Cannot decode data file {self.data_file.path}") diff --git a/validity/compliance/serialization/ttp.py b/validity/compliance/serialization/ttp.py new file mode 100644 index 0000000..02492ab --- /dev/null +++ b/validity/compliance/serialization/ttp.py @@ -0,0 +1,11 @@ +from ttp import ttp + +from validity.utils.misc import reraise +from ..exceptions import SerializationError + + +def serialize_ttp(plain_data: str, template: str): + parser = ttp(data=plain_data, template=template) + parser.parse() + with reraise(IndexError, SerializationError, f"Invalid parsed config: {parser.result()}"): + return parser.result()[0][0] diff --git a/validity/compliance/serialization/yaml.py b/validity/compliance/serialization/yaml.py new file mode 100644 index 0000000..ee46227 --- /dev/null +++ b/validity/compliance/serialization/yaml.py @@ -0,0 +1,9 @@ +import yaml + +from validity.utils.misc import reraise +from ..exceptions import SerializationError + + +def serialize_yaml(plain_data: str, template: str = "") -> dict: + with reraise(yaml.YAMLError, SerializationError, "Got invalid JSON/YAML"): + return yaml.safe_load(plain_data) diff --git a/validity/compliance/state.py b/validity/compliance/state.py new file mode 100644 index 0000000..f2d861e --- /dev/null +++ b/validity/compliance/state.py @@ -0,0 +1,85 @@ +from contextlib import suppress +from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterable, Optional + +from django.utils.translation import gettext_lazy as _ + +from validity.compliance.serialization import Serializable +from .exceptions import SerializationError + + +if TYPE_CHECKING: + from validity.models import Command + + +@dataclass(frozen=True) +class StateItem(Serializable): + command: Optional["Command"] + + @classmethod + def from_command(cls, command: "Command"): + return cls(data_file=command.data_file, serializer=command.serializer, command=command) + + @property + def contains_config(self) -> bool: + return self.command is None or self.command.retrieves_config + + @property + def name(self) -> str: + return "config" if self.contains_config else self.command.label + + @property + def verbose_name(self) -> str: + return _("Config") if self.contains_config else self.command.name + + @property + def error(self) -> SerializationError | None: + try: + self.serialized + return + except SerializationError as exc: + return exc + + +class State(dict): + def __init__(self, items, config_command_label: str | None = None): + super().__init__(items) + self.config_command_label = config_command_label + + @classmethod + def from_commands(cls, commands: Iterable["Command"]): + items = [] + config_label = None + for command in commands: + if command.retrieves_config: + config_label = command.label + items.append(StateItem.from_command(command)) + return cls(((item.name, item) for item in items), config_label) + + def with_config(self, serializable: Serializable): + state_item = StateItem(serializer=serializable.serializer, data_file=serializable.data_file, command=None) + with suppress(SerializationError): + state_item.serialized + super().__setitem__("config", state_item) + self.config_command_label = None + return self + + def _blocked_op(self, *_): + raise AttributeError("State is read only") + + __setitem__ = __delitem__ = __ior__ = pop = popitem = update = setdefault = clear = _blocked_op + + def __getattr__(self, key): + return self[key] + + def __getitem__(self, key): + state_item = super().__getitem__(key) + return state_item.serialized + + def get(self, key, default=None, ignore_errors=False): + with suppress(Exception if ignore_errors else KeyError): + return self[key] + return default + + def get_full_item(self, key, default=None): + return super().get(key, default) diff --git a/validity/data_backends.py b/validity/data_backends.py index f7ddf8b..60b0d5e 100644 --- a/validity/data_backends.py +++ b/validity/data_backends.py @@ -35,7 +35,7 @@ class PollingBackend(DataBackend): ) } - devices_qs = VDevice.objects.prefetch_poller().annotate_datasource_id().order_by("poller_id") + devices_qs = VDevice.objects.prefetch_poller(with_commands=True).annotate_datasource_id().order_by("poller_id") metainfo_file = Path("polling_info.yaml") def bound_devices_qs(self, device_filter: Q): diff --git a/validity/filtersets.py b/validity/filtersets.py index 4e44ad2..11653ac 100644 --- a/validity/filtersets.py +++ b/validity/filtersets.py @@ -121,7 +121,10 @@ class Meta: class CommandFilterSet(SearchMixin, NetBoxModelFilterSet): + serializer_id = ModelMultipleChoiceFilter(field_name="serializer", queryset=models.Serializer.objects.all()) + poller_id = ModelMultipleChoiceFilter(field_name="pollers", queryset=models.Poller.objects.all()) + class Meta: model = models.Command - fields = ("id", "name", "label", "type", "retrieves_config") + fields = ("id", "name", "label", "type", "retrieves_config", "serializer_id", "poller_id") search_fields = ("name", "label") diff --git a/validity/forms/__init__.py b/validity/forms/__init__.py index b0abd5d..01d243b 100644 --- a/validity/forms/__init__.py +++ b/validity/forms/__init__.py @@ -8,6 +8,7 @@ PollerFilterForm, ReportGroupByForm, SerializerFilterForm, + StateSelectForm, TestResultFilterForm, ) from .general import CommandForm, ComplianceSelectorForm, ComplianceTestForm, NameSetForm, PollerForm, SerializerForm diff --git a/validity/forms/filterset.py b/validity/forms/filterset.py index a029651..37f0ef7 100644 --- a/validity/forms/filterset.py +++ b/validity/forms/filterset.py @@ -99,6 +99,16 @@ class ReportGroupByForm(Form): ) +class StateSelectForm(Form): + state_item = PlaceholderChoiceField( + label=_("State Item"), placeholder=_("Select State Item"), required=False, choices=[] + ) + + def __init__(self, *args, state, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.fields["state_item"].choices += [(item.name, item.verbose_name) for item in state.values()] + + class NameSetFilterForm(NetBoxModelFilterSetForm): model = models.NameSet name = CharField(required=False) @@ -158,3 +168,7 @@ class CommandFilterForm(NetBoxModelFilterSetForm): retrieves_config = NullBooleanField( label=_("Global"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES) ) + serializer_id = DynamicModelMultipleChoiceField( + label=_("Serializer"), queryset=models.Serializer.objects.all(), required=False + ) + poller_id = DynamicModelMultipleChoiceField(label=_("Poller"), queryset=models.Poller.objects.all(), required=False) diff --git a/validity/forms/general.py b/validity/forms/general.py index 667f822..33406b9 100644 --- a/validity/forms/general.py +++ b/validity/forms/general.py @@ -6,7 +6,7 @@ from netbox.forms import NetBoxModelForm from tenancy.models import Tenant from utilities.forms import get_field_value -from utilities.forms.fields import DynamicModelMultipleChoiceField +from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField from utilities.forms.widgets import HTMXSelect from validity import models @@ -132,11 +132,13 @@ def clean(self): class CommandForm(SubformMixin, NetBoxModelForm): + serializer = DynamicModelChoiceField(queryset=models.Serializer.objects.all(), required=False) + main_fieldsets = [ - (_("Command"), ("name", "label", "type", "retrieves_config", "tags")), + (_("Command"), ("name", "label", "type", "retrieves_config", "serializer", "tags")), ] class Meta: model = models.Command - fields = ("name", "label", "type", "retrieves_config", "tags") + fields = ("name", "label", "type", "retrieves_config", "serializer", "tags") widgets = {"type": HTMXSelect()} diff --git a/validity/managers.py b/validity/managers.py index 85779b0..36be3f9 100644 --- a/validity/managers.py +++ b/validity/managers.py @@ -21,7 +21,7 @@ from validity import settings from validity.choices import DeviceGroupByChoices, SeverityChoices -from validity.utils.orm import CustomPrefetchMixin +from validity.utils.orm import CustomPrefetchMixin, SetAttributesMixin class ComplianceTestQS(RestrictedQuerySet): @@ -130,26 +130,9 @@ def delete_old(self, _settings=settings): ) -class VDeviceQS(CustomPrefetchMixin, RestrictedQuerySet): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.selector = None - - def _clone(self, *args, **kwargs): - c = super()._clone(*args, **kwargs) - c.selector = self.selector - return c - +class VDeviceQS(SetAttributesMixin, CustomPrefetchMixin, RestrictedQuerySet): def set_selector(self, selector): - self.selector = selector - return self - - def _fetch_all(self): - super()._fetch_all() - if self.selector: - for item in self._result_cache: - if isinstance(item, self.model): - item.selector = self.selector + self.set_attribute("selector", selector) def annotate_datasource_id(self): from validity.models import VDataSource @@ -210,10 +193,13 @@ def prefetch_serializer(self): "serializer", Serializer.objects.select_related("data_file") ) - def prefetch_poller(self): + def prefetch_poller(self, with_commands: bool = False): from validity.models import Poller - return self.annotate_poller_id().custom_prefetch("poller", Poller.objects.prefetch_commands()) + poller_qs = Poller.objects.all() + if with_commands: + poller_qs = poller_qs.prefetch_commands() + return self.annotate_poller_id().custom_prefetch("poller", poller_qs) def _count_per_something(self, field: str, annotate_method: str) -> dict[int | None, int]: qs = getattr(self, annotate_method)().values(field).annotate(cnt=Count("id", distinct=True)) @@ -258,3 +244,23 @@ class PollerQS(RestrictedQuerySet): def prefetch_commands(self): Command = self.model._meta.get_field("commands").remote_field.model return self.prefetch_related(Prefetch("commands", Command.objects.order_by("-retrieves_config"))) + + +class CommandQS(SetAttributesMixin, CustomPrefetchMixin, RestrictedQuerySet): + def set_file_paths(self, device, data_source): + """ + Sets up 'path' attribute to each command + """ + self.set_attribute("device", device) + self.set_attribute("data_source", data_source) + return self + + def bind_attributes(self, instance): + initial_attrs = self._aux_attributes.copy() + device = self._aux_attributes.pop("device", None) + data_source = self._aux_attributes.pop("data_source", None) + if device and data_source: + path = data_source.get_command_path(device, instance) + instance.path = path + super().bind_attributes(instance) + self._aux_attributes = initial_attrs diff --git a/validity/migrations/0007_polling.py b/validity/migrations/0007_polling.py index 8b74778..5b384dc 100644 --- a/validity/migrations/0007_polling.py +++ b/validity/migrations/0007_polling.py @@ -6,7 +6,7 @@ import validity.models.base import validity.utils.dbfields from django.utils.translation import gettext_lazy as _ -from django.core.validators import RegexValidator +import django.core.validators import django.db.models.deletion @@ -89,10 +89,13 @@ class Migration(migrations.Migration): max_length=100, unique=True, validators=[ - RegexValidator( + django.core.validators.RegexValidator( + message="Only lowercase ASCII letters, numbers and underscores are allowed", regex="^[a-z][a-z0-9_]*$", - message=_("Only lowercase ASCII letters, numbers and underscores are allowed"), - ) + ), + django.core.validators.RegexValidator( + inverse_match=True, message="This label name is reserved", regex="^config$" + ), ], ), ), diff --git a/validity/models/data.py b/validity/models/data.py index 9b9cc33..ba2fdbf 100644 --- a/validity/models/data.py +++ b/validity/models/data.py @@ -42,11 +42,17 @@ def web_url(self) -> str: @property def config_path_template(self) -> str: - return self.cf.get("device_config_path", "") + return self.cf.get("device_config_path") or "" @property def command_path_template(self) -> str: - return self.cf.get("device_command_path", "") + return self.cf.get("device_command_path") or "" + + def get_config_path(self, device) -> str: + return Environment().from_string(self.config_path_template).render(device=device) + + def get_command_path(self, device, command) -> str: + return Environment().from_string(self.command_path_template).render(device=device, command=command) @contextmanager def _sync_status(self): diff --git a/validity/models/device.py b/validity/models/device.py index 7c11b58..86065f2 100644 --- a/validity/models/device.py +++ b/validity/models/device.py @@ -1,43 +1,58 @@ from functools import cached_property -from typing import Any, Optional +from typing import TYPE_CHECKING, Optional from dcim.models import Device -from validity.compliance.device_config import DeviceConfig -from validity.j2_env import Environment +from validity.compliance.serialization import Serializable +from validity.compliance.state import State from validity.managers import VDeviceQS -from .data import VDataFile, VDataSource +from .data import VDataSource + + +if TYPE_CHECKING: + from .selector import ComplianceSelector class VDevice(Device): objects = VDeviceQS.as_manager() data_source: VDataSource + selector: Optional["ComplianceSelector"] class Meta: proxy = True - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.selector = None + def _config_item(self) -> Serializable: + """ + Serializable from "device_config_path" file + """ + try: + config_path = self.data_source.get_config_path(self) + data_file = self.data_source.datafiles.filter(path=config_path).first() + return Serializable(self.serializer, data_file=data_file) + except AttributeError as exc: + if exc.obj is not None: + raise + return Serializable(self.serializer, data_file=None) @property - def config_path(self) -> str: - assert hasattr(self, "data_source"), "You must prefetch data_source first" - template = Environment().from_string(self.data_source.config_path_template) - return template.render(device=self) - - @cached_property - def data_file(self) -> VDataFile | None: - path = self.config_path - return self.data_source.datafiles.filter(path=path).first() - - @cached_property - def device_config(self) -> DeviceConfig: - return DeviceConfig.from_device(self) + def config(self) -> dict | list | None: + return self.state.config @cached_property - def config(self) -> dict | list | None: - return self.device_config.serialized + def state(self): + try: + commands = ( + self.poller.commands.all() + .set_file_paths(self, self.data_source) + .finally_prefetch( + "data_file", self.data_source.datafiles.all(), pk_field="path", remote_pk_field="path" + ) + ) + except AttributeError: + # if device has no poller or data_source + commands = [] + raise + return State.from_commands(commands).with_config(self._config_item()) @cached_property def dynamic_pair(self) -> Optional["VDevice"]: @@ -50,8 +65,3 @@ def dynamic_pair(self) -> Optional["VDevice"]: if filter_ is None: return return type(self).objects.filter(filter_).first() - - @property - def commands(self): - assert hasattr(self, "poller"), "You must prefetch poller first" - return self.poller.commands.all() diff --git a/validity/models/polling.py b/validity/models/polling.py index 8f69447..5b5024c 100644 --- a/validity/models/polling.py +++ b/validity/models/polling.py @@ -9,7 +9,7 @@ from django.utils.translation import gettext_lazy as _ from validity.choices import CommandTypeChoices, ConnectionTypeChoices -from validity.managers import PollerQS +from validity.managers import CommandQS, PollerQS from validity.pollers import get_poller from validity.subforms import CLICommandForm from validity.utils.dbfields import EncryptedDictField @@ -28,7 +28,8 @@ class Command(SubformMixin, BaseModel): RegexValidator( regex="^[a-z][a-z0-9_]*$", message=_("Only lowercase ASCII letters, numbers and underscores are allowed"), - ) + ), + RegexValidator(regex="^config$", message=_("This label name is reserved"), inverse_match=True), ], ) retrieves_config = models.BooleanField( @@ -47,6 +48,8 @@ class Command(SubformMixin, BaseModel): type = models.CharField(_("Type"), max_length=50, choices=CommandTypeChoices.choices) parameters = models.JSONField(_("Parameters")) + objects = CommandQS.as_manager() + subform_type_field = "type" subform_json_field = "parameters" subforms = {"CLI": CLICommandForm} diff --git a/validity/models/serializer.py b/validity/models/serializer.py index fe96d06..2c7b5d8 100644 --- a/validity/models/serializer.py +++ b/validity/models/serializer.py @@ -4,6 +4,7 @@ from django.utils.translation import gettext_lazy as _ from validity.choices import ExtractionMethodChoices +from validity.compliance.serialization import serialize from validity.netbox_changes import DEVICE_ROLE_RELATION from .base import BaseModel, DataSourceMixin @@ -17,6 +18,7 @@ class Serializer(DataSourceMixin, BaseModel): clone_fields = ("template", "extraction_method", "data_source", "data_file") text_db_field_name = "template" + _serialize = serialize class Meta: ordering = ("name",) @@ -54,3 +56,6 @@ def bound_devices(self) -> models.QuerySet[Device]: @property def effective_template(self) -> str: return self.effective_text_field() + + def serialize(self, data: str) -> dict: + return self._serialize(self.extraction_method, data, self.effective_template) diff --git a/validity/scripts/run_tests.py b/validity/scripts/run_tests.py index 68fffc1..6357513 100644 --- a/validity/scripts/run_tests.py +++ b/validity/scripts/run_tests.py @@ -17,7 +17,7 @@ import validity import validity.compliance.eval.default_nameset as default_nameset from validity.compliance.eval import ExplanationalEval -from validity.compliance.exceptions import DeviceConfigError, EvalError +from validity.compliance.exceptions import EvalError, SerializationError from validity.models import ( ComplianceReport, ComplianceSelector, @@ -132,7 +132,7 @@ def run_tests_for_selector( for device in qs: try: yield from self.run_tests_for_device(selector.tests.all(), device, report) - except DeviceConfigError as e: + except SerializationError as e: self.log_failure(f"`{e}`, ignoring all tests for *{device}*") continue diff --git a/validity/tables.py b/validity/tables.py index 8c32f50..ab6ad64 100644 --- a/validity/tables.py +++ b/validity/tables.py @@ -115,10 +115,11 @@ class Meta(NetBoxTable.Meta): class CommandTable(NetBoxTable): name = Column(linkify=True) type = ChoiceFieldColumn() + serializer = Column(linkify=True) class Meta(NetBoxTable.Meta): model = models.Command - fields = ("name", "type", "retrieves_config", "bound_pollers", "label") + fields = ("name", "type", "retrieves_config", "serializer", "label") class ExplanationColumn(Column): diff --git a/validity/templates/validity/command.html b/validity/templates/validity/command.html index 7ad33d7..678ab99 100644 --- a/validity/templates/validity/command.html +++ b/validity/templates/validity/command.html @@ -24,6 +24,11 @@
Command
Type {{ object | colored_choice:"type" }} + + + Serializer + {{ object.serializer | linkify | placeholder }} + Pollers {{ object.pollers.all | linkify_list }} diff --git a/validity/templates/validity/device_config.html b/validity/templates/validity/device_state.html similarity index 55% rename from validity/templates/validity/device_config.html rename to validity/templates/validity/device_state.html index db00cf5..a25123e 100644 --- a/validity/templates/validity/device_config.html +++ b/validity/templates/validity/device_state.html @@ -1,9 +1,10 @@ {% extends 'generic/object.html' %} {% load helpers %} {% load validity %} +{% load bootstrap5 %} {% block head %} {% endblock %} -{% block title %}{{ object }}: Serialized Configuration{% endblock %} +{% block title %}{{ object }}: Serialized State{% endblock %} {% block subtitle %}
{% endblock %} @@ -19,35 +20,54 @@
Metainfo
{{ object.data_source | linkify | placeholder }} - Device Config File - - {% include "validity/inc/path_with_link.html" with file_path=object.config_path web_url=object.data_source.web_url only %} - + Poller + {{ object.poller | linkify | placeholder }} Serializer - {{ object.serializer | linkify | placeholder }} + {{ state_item.serializer | linkify | placeholder }} + + + Command + {{ state_item.command | linkify | placeholder }} + + + Data File + + {% include "validity/inc/path_with_link.html" with data_file=state_item.data_file web_url=object.data_source.web_url only %} + Local copy last modified - {{ config.last_modified | date:"Y-m-d G:i:s" | placeholder}} + {{ state_item.data_file.last_updated | date:"Y-m-d G:i:s" | placeholder}} +
+
+
State
+
+
+ {% bootstrap_form state_form layout="inline" %} +
{% buttons submit="Show" %}{% endbuttons %}
+
+
+
+
-
+
{% if not error %}
-
Serialized Configuration
+
{{ state_item.verbose_name }}
{% include 'extras/inc/configcontext_format.html' %}
- {% include 'extras/inc/configcontext_data.html' with data=config.serialized format=format %} + {% include 'extras/inc/configcontext_data.html' with data=state_item.serialized format=format %}
{% else %} diff --git a/validity/templates/validity/inc/datasource_link.html b/validity/templates/validity/inc/datasource_link.html index da5a173..8aa041c 100644 --- a/validity/templates/validity/inc/datasource_link.html +++ b/validity/templates/validity/inc/datasource_link.html @@ -6,6 +6,6 @@ File Path - {% include "validity/inc/path_with_link.html" with file_path=object.data_file.path web_url=object.data_source.web_url only %} + {% include "validity/inc/path_with_link.html" with data_file=object.data_file web_url=object.data_source.web_url only %} diff --git a/validity/templates/validity/inc/path_with_link.html b/validity/templates/validity/inc/path_with_link.html index 383d939..8d5a137 100644 --- a/validity/templates/validity/inc/path_with_link.html +++ b/validity/templates/validity/inc/path_with_link.html @@ -1,11 +1,12 @@ {% load validity %} +{% load helpers %}
- {{ file_path | placeholder }} + {{ data_file | linkify | placeholder }}
{% if file_path and web_url %}
- ⤴ + ⤴
{% endif %}
\ No newline at end of file diff --git a/validity/utils/orm.py b/validity/utils/orm.py index 26bfb42..20561e6 100644 --- a/validity/utils/orm.py +++ b/validity/utils/orm.py @@ -1,27 +1,22 @@ +from __future__ import annotations + from dataclasses import dataclass from itertools import chain -from typing import Any, Generic, Iterable, Iterator, TypeVar - -from django.db.models import F, Func, QuerySet, TextField +from typing import Generic, Iterable, Iterator, TypeVar +from django.db.models import Model, QuerySet -class RegexpReplace(Func): - function = "REGEXP_REPLACE" - def __init__(self, source: F, pattern: str, replacement_string: str, flags: str = "", **extra: Any) -> None: - extra.setdefault("output_field", TextField()) - expressions = [source, pattern, replacement_string] - if flags: - expressions.append(flags) - super().__init__(*expressions, **extra) +M = TypeVar("M", bound=Model) +N = TypeVar("N", bound=Model) -class QuerySetMap: +class QuerySetMap(Generic[M]): """ Lazy pk:model dict which hits the DB when first queried """ - def __init__(self, qs: QuerySet, attribute: str = "pk"): + def __init__(self, qs: QuerySet[M], attribute: str = "pk"): self._qs = qs self._attribute = attribute self._evaluated = False @@ -29,7 +24,8 @@ def __init__(self, qs: QuerySet, attribute: str = "pk"): def _evaluate(self): if not self._evaluated: - for model in self._qs.iterator(chunk_size=2000): + qs = self._qs if self._qs._result_cache is not None else self._qs.iterator(chunk_size=2000) + for model in qs: self._map[getattr(model, self._attribute)] = model self._evaluated = True @@ -45,14 +41,15 @@ def get(self, key, default=None): self._evaluate() return self._map.get(key, default) + def keys(self): + self._evaluate() + return self._map.keys() + @property def model(self): return self._qs.model -M = TypeVar("M") - - class M2MIterator(Generic[M]): """ This class mimics lazy handling of the QuerySet @@ -71,16 +68,16 @@ def all(self) -> Iterator[M]: @dataclass class CustomPrefetch: field: str + pk_field: str + remote_pk_field: str qs: QuerySet many: bool - pk_field = property(lambda self: self.field + "_id") - def get_qs_map(self, main_queryset: QuerySet) -> QuerySetMap: pk_values = main_queryset.values_list(self.pk_field, flat=True) if self.many: pk_values = chain.from_iterable(pk_values) - return QuerySetMap(self.qs.filter(pk__in=pk_values)) + return QuerySetMap(self.qs.filter(**{f"{self.remote_pk_field}__in": pk_values}), attribute=self.remote_pk_field) class CustomPrefetchMixin(QuerySet): @@ -93,12 +90,30 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.custom_prefetches = [] - def custom_prefetch(self, field: str, prefetch_qs: QuerySet, many: bool = False): - self.custom_prefetches.append(CustomPrefetch(field, prefetch_qs, many)) + def custom_prefetch( + self, field: str, prefetch_qs: QuerySet, many: bool = False, pk_field: str = "", remote_pk_field: str = "pk" + ): + pk_field = pk_field or field + "_id" + self.custom_prefetches.append(CustomPrefetch(field, pk_field, remote_pk_field, prefetch_qs, many)) return self custom_prefetch.queryset_only = True + def custom_postfetch(self, field: str, postfetch_qs: QuerySet, pk_field: str = "", remote_pk_field: str = "pk"): + """ + Causes evaluation of the queryset when called. DO NOT USE IN View.queryset + Allows to use prefetch with runtime model attributes (like dynamically set by .set_attribute) + """ + pk_field = pk_field or field + "_id" + pk_values = (getattr(obj, pk_field) for obj in self) + qs_map = QuerySetMap(postfetch_qs.filter(**{f"{remote_pk_field}__in": pk_values}), attribute=remote_pk_field) + for obj in self: + pk_value = getattr(obj, pk_field) + setattr(obj, field, qs_map.get(pk_value)) + return self + + custom_postfetch.queryset_only = True + def _clone(self, *args, **kwargs): c = super()._clone(*args, **kwargs) c.custom_prefetches = self.custom_prefetches.copy() @@ -106,15 +121,43 @@ def _clone(self, *args, **kwargs): def _fetch_all(self): super()._fetch_all() - qs_dicts = {custom_pf.field: custom_pf.get_qs_map(self) for custom_pf in self.custom_prefetches} - for item in self._result_cache: - if not isinstance(item, self.model): + qs_maps = {custom_pf.field: custom_pf.get_qs_map(self) for custom_pf in self.custom_prefetches} + for model_instance in self._result_cache: + if not isinstance(model_instance, self.model): continue for custom_prefetch in self.custom_prefetches: - prefetch_pk_values = getattr(item, custom_prefetch.pk_field) - qs_dict = qs_dicts[custom_prefetch.field] + prefetch_pk_values = getattr(model_instance, custom_prefetch.pk_field) + qs_map = qs_maps[custom_prefetch.field] if custom_prefetch.many: - prefetch_values = M2MIterator(qs_dict[pk] for pk in prefetch_pk_values) + prefetch_values = M2MIterator(qs_map[pk] for pk in prefetch_pk_values) else: - prefetch_values = qs_dict.get(prefetch_pk_values) - setattr(item, custom_prefetch.field, prefetch_values) + prefetch_values = qs_map.get(prefetch_pk_values) + setattr(model_instance, custom_prefetch.field, prefetch_values) + + +class SetAttributesMixin(QuerySet): + """ + Allows to define aux qs-level attributes which will be assigned to model instances + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._aux_attributes = {} + + def _clone(self, *args, **kwargs): + c = super()._clone(*args, **kwargs) + c._aux_attributes = self._aux_attributes + return c + + def bind_attributes(self, instance): + for attr, attr_value in self._aux_attributes: + setattr(instance, attr, attr_value) + + def _fetch_all(self): + super()._fetch_all() + for item in self._result_cache: + if isinstance(item, self.model): + self.bind_attributes(item) + + def set_attribute(self, name, value): + self._aux_attributes[name] = value diff --git a/validity/views/__init__.py b/validity/views/__init__.py index 4b35547..6421dc3 100644 --- a/validity/views/__init__.py +++ b/validity/views/__init__.py @@ -1,5 +1,5 @@ from .command import CommandBulkDeleteView, CommandDeleteView, CommandEditView, CommandListView, CommandView -from .device import DeviceSerializedConfigView, TestResultView +from .device import DeviceSerializedStateView, TestResultView from .nameset import NameSetBulkDeleteView, NameSetDeleteView, NameSetEditView, NameSetListView, NameSetView from .poller import PollerBulkDeleteView, PollerDeleteView, PollerEditView, PollerListView, PollerView from .report import ComplianceReportListView, ComplianceReportView diff --git a/validity/views/device.py b/validity/views/device.py index 685888b..3df10bb 100644 --- a/validity/views/device.py +++ b/validity/views/device.py @@ -4,7 +4,7 @@ from netbox.views import generic from utilities.views import ViewTab, register_model_view -from validity.compliance.exceptions import DeviceConfigError +from validity.forms import StateSelectForm from validity.models import VDevice from .base import TestResultBaseView @@ -19,16 +19,24 @@ class TestResultView(TestResultBaseView): exclude_form_fields = ("platform_id", "tenant_id", "device_role_id", "manufacturer_id", "report_id", "selector_id") -@register_model_view(Device, "serialized_config") -class DeviceSerializedConfigView(generic.ObjectView): - template_name = "validity/device_config.html" - tab = ViewTab("Serialized Config", permission="dcim.view_device") +@register_model_view(Device, "serialized_state") +class DeviceSerializedStateView(generic.ObjectView): + template_name = "validity/device_state.html" + tab = ViewTab("Serialized State", permission="dcim.view_device") queryset = VDevice.objects.prefetch_datasource().prefetch_serializer().prefetch_poller() + form_cls = StateSelectForm + default_state_item = "config" def get_extra_context(self, request, instance): - try: - instance._meta = Device()._meta - return {"config": instance.device_config, "format": request.GET.get("format", "yaml")} - except DeviceConfigError as e: - error = f"Cannot render serialized config, {e}" - return {"error": error} + state_item_name = request.GET.get("state_item", self.default_state_item) + state_item = instance.state.get_full_item(state_item_name) + state_form = self.form_cls( + state=instance.state, initial={"state_item": state_item.name} if state_item else None + ) + instance._meta = Device()._meta + context = {"state_item": state_item, "state_form": state_form, "format": request.GET.get("format", "yaml")} + if state_item is None: + context["error"] = f'"{state_item_name}" is not a member of {instance} state.' + elif (error := state_item.error) is not None: + context["error"] = f"Cannot render state item. {error}" + return context