From 46d31aa55e39263559d96d7ecefa5e6f5200e612 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 5 Sep 2023 15:11:08 +0100 Subject: [PATCH 01/15] Bump version after release 2.0.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f77a6..a69e585 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1 +2.0.1.dev0 From f2a930ec19d645fb01450da732001b97d5b8bd0c Mon Sep 17 00:00:00 2001 From: Alain ZAMBONI Date: Thu, 4 Apr 2024 17:12:57 +0200 Subject: [PATCH 02/15] Fix the backup of startup configuration format --- ztpserver/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ztpserver/controller.py b/ztpserver/controller.py index 29ab912..b3de175 100644 --- a/ztpserver/controller.py +++ b/ztpserver/controller.py @@ -218,7 +218,7 @@ def put_config(self, request, **kwargs): fobj = None filename = self.expand(node_id, STARTUP_CONFIG_FN) - body = str(request.body) + body = request.body.decode("utf-8") content_type = str(request.content_type) try: fobj = self.repository.get_file(filename) From e383ba8b854dedaece04aa9516b95577f0b31eb2 Mon Sep 17 00:00:00 2001 From: Alain ZAMBONI Date: Thu, 4 Apr 2024 17:14:46 +0200 Subject: [PATCH 03/15] Added the model filter in neighbordb --- docs/config.rst | 13 ++++++++++--- docs/examples.rst | 21 +++++++++++++++++++++ ztpserver/topology.py | 8 ++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 2ced5cd..b947ed6 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -476,7 +476,8 @@ unique_id of the node) and global patterns. Rules: - if multiple node-specific entries reference the same unique_id, only the first will be in effect - all others will be ignored - - if both the **node** and **interfaces** attributes are specified and a node's unique_id is a match, but the topology information is not, then the overall match will fail and the global patterns will not be considered + - if the **node** and **interfaces** or **model** attributes are specified and a node's unique_id is a match, but the topology information or model information are not, then the overall match will fail and the global patterns will not be considered + - **model** can't be the only matching attributes, **node** or **interfaces** are mandatory to have a valid match - if there is no matching node-specific pattern for a node's unique_id, then the server will attempt to match the node against the global patterns (in the order they are specified in ``neighbordb``) - if a node-specific pattern matches, the server will automatically generate an open pattern in the node's folder. This pattern will match any device with at least one LLDP-capable neighbor. Example: ``any: any:any`` @@ -488,8 +489,9 @@ Rules: ... patterns: - name: - definition: + definition: node: + model: config-handler: variables: : @@ -504,7 +506,7 @@ Rules: Mandatory attributes: **name**, **definition**, and either **node**, **interfaces** or both. - Optional attributes: **variables**, **config-handler**. + Optional attributes: **variables**, **config-handler**, **model**. variables ''''''''' @@ -528,6 +530,11 @@ node: unique_id Serial number or MAC address, depending on the global 'identifier' attribute in **ztpserver.conf**. +model: model_regexp +''''''''''''''''''' + +Defines a regex pattern to match the node model against. + interfaces: port\_name '''''''''''''''''''''' diff --git a/docs/examples.rst b/docs/examples.rst index 7059af3..75374ba 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -367,6 +367,27 @@ Example #5 In this case, the pattern matches if `any` local interface is connected to a device with `spine` in the hostname and to the 4th or 5th slot in the chassis. +Example #6 +'''''''''' + +.. code-block:: yaml + + --- + - name: old switch + definition: old-switch + model: "DCS-7010T-48" + interfaces: + - Ethernet49: $uplink:any + - name: new switch + definition: new-switch + model: "DCS-7010TX-48-F" + interfaces: + - Ethernet49: $uplink:any + +In this case, the two patterns match the same uplink switch on the same +local interface, but with a different model. This will allow to use a +different definition to upload a version of EOS compatible with the device. + More examples ````````````` diff --git a/ztpserver/topology.py b/ztpserver/topology.py index 217162c..aff3093 100644 --- a/ztpserver/topology.py +++ b/ztpserver/topology.py @@ -496,6 +496,7 @@ def __init__( node=None, variables=None, node_id=None, + model=None, ): self.name = name self.definition = definition @@ -505,6 +506,8 @@ def __init__( self.node_id = node_id self.variables = variables or {} + self.model=model + self.interfaces = [] if interfaces: self.add_interfaces(interfaces) @@ -551,6 +554,7 @@ def serialize(self): "definition": self.definition, "variables": self.variables, "node": self.node, + "model": self.model, "config-handler": self.config_handler, } @@ -640,6 +644,10 @@ def match_node(self, node): # while selecting the set of nodes which are eligible for a # match. + # Match the model first + if self.model and not re.match(self.model, node.model): + return False + patterns = [] for entry in self.interfaces: for pattern in entry["patterns"]: From fed6e527ac9632c89c03cf38ece993d308876c02 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 30 Apr 2024 15:05:38 +0100 Subject: [PATCH 04/15] Fix controller.py:put_config body decoding --- test/server/server_test_lib.py | 4 ++++ test/server/test_controller.py | 10 ++++++++-- ztpserver/controller.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/test/server/server_test_lib.py b/test/server/server_test_lib.py index 7a97574..9d0017e 100644 --- a/test/server/server_test_lib.py +++ b/test/server/server_test_lib.py @@ -61,6 +61,10 @@ def random_string(): ) +def random_bytes() -> bytes: + return random_string().encode("utf-8") + + def random_json(keys=None): data = {} if keys: diff --git a/test/server/test_controller.py b/test/server/test_controller.py index 9300405..832b872 100644 --- a/test/server/test_controller.py +++ b/test/server/test_controller.py @@ -42,6 +42,7 @@ create_node, enable_logging, mock_match, + random_bytes, random_string, remove_all, write_file, @@ -1006,18 +1007,23 @@ def test_do_resources_success(self): foo = resp["definition"]["actions"][0]["attributes"]["foo"] self.assertEqual(foo, var_foo) + @patch("ztpserver.controller.create_repository") @patch("os.path.isfile") - def test_put_config_success(self, m_is_file): + def test_put_config_success(self, m_is_file, m_create_repository): m_is_file.return_value = False + file_mock = MagicMock() + m_create_repository.return_value.get_file.return_value = file_mock resource = random_string() - body = random_string() + body = random_bytes() request = Mock(content_type=constants.CONTENT_TYPE_OTHER, body=body) controller = ztpserver.controller.NodesController() + resp = controller.put_config(request, resource=resource) self.assertEqual(resp, {}) + file_mock.write.assert_called_with(body.decode("utf-8"), constants.CONTENT_TYPE_OTHER) class NodesControllerPostFsmIntegrationTests(unittest.TestCase): diff --git a/ztpserver/controller.py b/ztpserver/controller.py index 29ab912..b3de175 100644 --- a/ztpserver/controller.py +++ b/ztpserver/controller.py @@ -218,7 +218,7 @@ def put_config(self, request, **kwargs): fobj = None filename = self.expand(node_id, STARTUP_CONFIG_FN) - body = str(request.body) + body = request.body.decode("utf-8") content_type = str(request.content_type) try: fobj = self.repository.get_file(filename) From 25e6f3b010db9f777136fdd93773aac2cd24017a Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 30 Apr 2024 17:04:44 +0100 Subject: [PATCH 05/15] Fix linting errors Fix outdated .pylintrc --- .github/workflows/pylint.yml | 2 +- .pylintrc | 2 -- ztpserver/controller.py | 19 ++++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 95ce21f..829eb36 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,7 +18,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint - pip install -r requirements.txt + pip install . pip install -r requirements-node.txt - name: Analysing the code with pylint run: | diff --git a/.pylintrc b/.pylintrc index bd2dee2..2107c24 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,3 @@ -[MASTER] -init-hook="from pylint.config import find_pylintrc;import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))" [BASIC] method-rgx=[a-z_][a-z0-9_]{2,50}$ diff --git a/ztpserver/controller.py b/ztpserver/controller.py index b3de175..da75f2b 100644 --- a/ztpserver/controller.py +++ b/ztpserver/controller.py @@ -225,12 +225,12 @@ def put_config(self, request, **kwargs): except FileObjectNotFound: log.debug("%s: file not found: %s (adding it)", node_id, filename) fobj = self.repository.add_file(filename) - finally: - if fobj: - fobj.write(body, content_type) - else: - log.error("%s: unable to write %s", node_id, filename) - return self.http_bad_request() + + try: + fobj.write(body, content_type) + except OSError: + log.error("%s: unable to write %s", node_id, filename) + return self.http_bad_request() # Execute event-handler script = self.repository.expand(self.expand(node_id, CONFIG_HANDLER_FN)) @@ -520,10 +520,11 @@ def dump_node(self, response, *args, **kwargs): fobj = self.repository.get_file(filename) except FileObjectNotFound: fobj = self.repository.add_file(filename) - finally: - if fobj and contents: + + if contents: + try: fobj.write(contents, CONTENT_TYPE_JSON) - else: + except OSError: log.error("%s: unable to write %s", node_id, filename) return self.http_bad_request() From cb726b6c8122df7bb216dbe9bae3ac6dbaab757c Mon Sep 17 00:00:00 2001 From: David Lobato Date: Wed, 1 May 2024 13:07:43 +0100 Subject: [PATCH 06/15] Add tests Improve pattern validation to account for model attribute --- test/neighbordb/model_pattern_test.yml | 30 +++++++ test/neighbordb/model_topology_test.yml | 108 ++++++++++++++++++++++++ ztpserver/validators.py | 20 ++--- 3 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 test/neighbordb/model_pattern_test.yml create mode 100644 test/neighbordb/model_topology_test.yml diff --git a/test/neighbordb/model_pattern_test.yml b/test/neighbordb/model_pattern_test.yml new file mode 100644 index 0000000..1ee1425 --- /dev/null +++ b/test/neighbordb/model_pattern_test.yml @@ -0,0 +1,30 @@ +tag: model_pattern_test +tests: + - pattern +valid_patterns: + nodes: + - node pattern + globals: + - pattern with model + - pattern with model and interfaces + +neighbordb: + patterns: + - name: node pattern + definition: test + node: 2b3c + + - name: invalid pattern with node and model + definition: test + model: modelA + node: 2b3caa + + - name: pattern with model + definition: test + model: modelA + + - name: pattern with model and interfaces + definition: test + model: modelA + interfaces: + - Ethernet1: localhost:Ethernet1 diff --git a/test/neighbordb/model_topology_test.yml b/test/neighbordb/model_topology_test.yml new file mode 100644 index 0000000..daad52f --- /dev/null +++ b/test/neighbordb/model_topology_test.yml @@ -0,0 +1,108 @@ +tag: model_pattern_test +tests: + - pattern + - topology +valid_patterns: + nodes: + - node pattern + globals: + - pattern with modelA + - pattern with modelB and neighbour localhost:Ethernet1 + - pattern with modelB and neighbour localhost:Ethernet2 + - pattern with modelB and no neighbours + - pattern with modelC and neighbour localhost:Ethernet1 + +nodes: + pass: + - name: node_2b3c + match: node pattern + - name: node_3b3c + match: pattern with modelA + - name: node_4b3c + match: pattern with modelB and neighbour localhost:Ethernet1 + - name: node_5b3c + match: pattern with modelB and neighbour localhost:Ethernet2 + - name: node_6b3c + match: pattern with modelB and no neighbours + fail: + - name: node_7b3c + +node_2b3c: + serialnumber: 2b3c + model: modelA + neighbors: + Ethernet1: + - device: localhost + port: Ethernet1 + +node_3b3c: + serialnumber: 3b3c + model: modelA + neighbors: + Ethernet1: + - device: localhost + port: Ethernet1 + +node_4b3c: + serialnumber: 4b3c + model: modelB + neighbors: + Ethernet1: + - device: localhost + port: Ethernet1 + +node_5b3c: + serialnumber: 5b3c + model: modelB + neighbors: + Ethernet1: + - device: localhost + port: Ethernet2 + +node_6b3c: + serialnumber: 6b3c + model: modelB + neighbors: + Ethernet1: + - device: localhost + port: Ethernet3 + +node_7b3c: + serialnumber: 7b3c + model: modelC + neighbors: + Ethernet1: + - device: localhost + port: Ethernet3 + +neighbordb: + patterns: + - name: node pattern + definition: test + node: 2b3c + + - name: pattern with modelA + definition: test + model: modelA + + - name: pattern with modelB and neighbour localhost:Ethernet1 + definition: test + model: modelB + interfaces: + - Ethernet1: localhost:Ethernet1 + + - name: pattern with modelB and neighbour localhost:Ethernet2 + definition: test + model: modelB + interfaces: + - Ethernet1: localhost:Ethernet2 + + - name: pattern with modelB and no neighbours + definition: test + model: modelB + + - name: pattern with modelC and neighbour localhost:Ethernet1 + definition: test + model: modelC + interfaces: + - Ethernet1: localhost:Ethernet1 diff --git a/ztpserver/validators.py b/ztpserver/validators.py index f88c47e..338acea 100644 --- a/ztpserver/validators.py +++ b/ztpserver/validators.py @@ -40,7 +40,7 @@ from ztpserver.utils import expand_range, parse_interface REQUIRED_PATTERN_ATTRIBUTES = ["name", "definition"] -OPTIONAL_PATTERN_ATTRIBUTES = ["node", "variables", "interfaces"] +OPTIONAL_PATTERN_ATTRIBUTES = ["node", "variables", "interfaces", "model"] INTERFACE_PATTERN_KEYWORDS = ["any", "none"] ANTINODE_PATTERN = rf"[^{string.hexdigits}]" KW_ANY_RE = re.compile(r" *any *") @@ -170,17 +170,15 @@ def validate_attributes(self): if attr not in self.data: raise ValidationError(f"missing attribute: {attr}") - if "node" not in self.data and "interfaces" not in self.data: - raise ValidationError("missing attribute: 'node' OR 'interfaces'") + if "node" not in self.data and "model" not in self.data and "interfaces" not in self.data: + raise ValidationError("missing attribute: 'node' OR 'model' OR 'interfaces'") - for attr in OPTIONAL_PATTERN_ATTRIBUTES: - if attr not in self.data: - log.warning( - "%s: PatternValidator warning: '%s' is missing optional attribute (%s)", - self.node_id, - self.data["name"], - attr, - ) + if "node" in self.data and "model" in self.data: + raise ValidationError("'node' AND 'model' are mutually exclusive") + + for attr in self.data: + if attr not in REQUIRED_PATTERN_ATTRIBUTES + OPTIONAL_PATTERN_ATTRIBUTES: + raise ValidationError(f"{attr} not allowed") def validate_name(self): if not self.data or "name" not in self.data: From d6cea344b81b0af08e7f3f8916bdf0e3f3dd9ff2 Mon Sep 17 00:00:00 2001 From: Jason Robertson <133690336+siguroot@users.noreply.github.com> Date: Thu, 2 May 2024 02:31:54 -0700 Subject: [PATCH 07/15] install_image change image to reference filename provided by url attribute (#406) * changed image to reference filename provided via url attribute * added image_name_from_url attribute --------- Co-authored-by: David Lobato --- actions/install_image | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/actions/install_image b/actions/install_image index 62f88c4..6593288 100644 --- a/actions/install_image +++ b/actions/install_image @@ -67,6 +67,8 @@ def main(attributes): version: EOS version of new image file downgrade: Boolean - Should EOS images be downgraded to match? (Default: True) + image_name_from_url: Boolean - Should EOS images be named from URL? + (Default: False) Special_attributes: NODE: API object - see documentation for details @@ -90,6 +92,7 @@ def main(attributes): node = attributes.get("NODE") url = attributes.get("url") + image_name_from_url = attributes.get("image_name_from_url", False) if not url: raise RuntimeError("Missing attribute('url')") @@ -113,7 +116,11 @@ def main(attributes): return # In all other cases, copy the image - image = "EOS-{}.swi".format(version) + if image_name_from_url: + image = url.split("/")[-1] + else: + image = "EOS-{}.swi".format(version) + try: node.retrieve_url(url, "{}/{}".format(node.flash(), image)) except Exception as exc: From 6450f4d4a669aefd02188b1c4b390842b3b9da08 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Thu, 2 May 2024 11:40:25 +0100 Subject: [PATCH 08/15] Trigger actions on pull_requests Add pre-commit and publish actions Dynamic version handling --- .github/workflows/pre-commit.yml | 20 ++++ .github/workflows/publish-to-pypi.yml | 127 ++++++++++++++++++++++++ .github/workflows/pylint.yml | 11 +- .github/workflows/unittest-node-py2.yml | 9 +- .github/workflows/unittest-node.yml | 11 +- .github/workflows/unittest-server.yml | 11 +- .pre-commit-config.yaml | 6 +- Makefile | 4 +- VERSION | 2 +- setup.py | 2 + 10 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .github/workflows/publish-to-pypi.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..200cade --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: pre-commit + +on: + push: + branches: + - main + - develop + pull_request: + +jobs: + main: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.1 + - uses: pre-commit-ci/lite-action@v1.0.2 + if: always() diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..98b3f68 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,127 @@ +name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Install dunamai + run: >- + python3 -m + pip install + dunamai + --user + - name: Build a binary wheel and a source tarball + run: DYNAMIC_VERSION=$(dunamai from git --no-metadata) python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ztpserver + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python 🐍 distribution 📦 with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + + publish-to-testpypi: + name: Publish Python 🐍 distribution 📦 to TestPyPI + if: ${{ github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/') }} + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/ztpserver + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 829eb36..5e0cee2 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,11 @@ name: Pylint -on: [push] +on: + push: + branches: + - main + - develop + pull_request: jobs: build: @@ -9,9 +14,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/unittest-node-py2.yml b/.github/workflows/unittest-node-py2.yml index 3986d51..c40b575 100644 --- a/.github/workflows/unittest-node-py2.yml +++ b/.github/workflows/unittest-node-py2.yml @@ -1,6 +1,11 @@ name: Unittest Node py2 -on: [push] +on: + push: + branches: + - main + - develop + pull_request: jobs: build: @@ -12,7 +17,7 @@ jobs: env: USER: root steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/unittest-node.yml b/.github/workflows/unittest-node.yml index 3ddfcb9..dc26718 100644 --- a/.github/workflows/unittest-node.yml +++ b/.github/workflows/unittest-node.yml @@ -1,6 +1,11 @@ name: Unittest Node -on: [push] +on: + push: + branches: + - main + - develop + pull_request: jobs: build: @@ -9,9 +14,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/unittest-server.yml b/.github/workflows/unittest-server.yml index 4eac3ce..e9a23fd 100644 --- a/.github/workflows/unittest-server.yml +++ b/.github/workflows/unittest-server.yml @@ -1,6 +1,11 @@ name: Unittest server -on: [push] +on: + push: + branches: + - main + - develop + pull_request: jobs: build: @@ -9,9 +14,9 @@ jobs: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7457520..8b41947 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,17 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.4.2 hooks: - id: black args: ["-l", "100"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) diff --git a/Makefile b/Makefile index c033c09..996f2ae 100644 --- a/Makefile +++ b/Makefile @@ -96,10 +96,10 @@ install: $(PYTHON) setup.py install sdist: clean ztpserver.spec - $(PYTHON) setup.py sdist + DYNAMIC_VERSION=$$(dunamai from git --no-metadata) $(PYTHON) setup.py sdist sdist-dev: clean ztpserver.spec - DEV_VERSION_HASH=$$(git rev-parse --short HEAD) $(PYTHON) setup.py sdist + DYNAMIC_VERSION=$$(dunamai from git) $(PYTHON) setup.py sdist docker_dev: sdist @docker build -t ${IMG} . diff --git a/VERSION b/VERSION index a69e585..6c51535 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1.dev0 +0.0.0 #uses dynamic version with: dunamai from git --no-metadata diff --git a/setup.py b/setup.py index 8dfaa8c..66e07ee 100755 --- a/setup.py +++ b/setup.py @@ -99,6 +99,8 @@ def get_long_description(): x.strip() for x in install_requirements if x.strip() and "dev only" not in x ] version = open("VERSION").read().split()[0].strip() +if os.environ.get("DYNAMIC_VERSION"): + version = os.environ.get("DYNAMIC_VERSION") if os.environ.get("DEV_VERSION_HASH"): version = f"{version}+{os.environ.get('DEV_VERSION_HASH')}" From 6335639d010386884378d4bb7513944da5f00037 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 10:42:00 +0000 Subject: [PATCH 09/15] [pre-commit.ci lite] apply automatic fixes --- CHANGELOG.md | 1 - CONTRIBUTING.md | 6 +- INSTALL.md | 2 +- conf/bootstrap.conf | 1 - conf/neighbordb | 1 - conf/ztpserver.conf | 4 +- conf/ztpserver.wsgi | 2 +- docs/CONTRIBUTING.md | 1 - docs/README.md | 1 - docs/ReleaseNotes1.1.rst | 7 +- docs/ReleaseNotes1.2.rst | 3 +- docs/ReleaseNotes1.3.1.rst | 1 - docs/ReleaseNotes1.3.2.rst | 1 - docs/ReleaseNotes1.3.rst | 7 +- docs/ReleaseNotes1.5.0.rst | 2 - docs/actions.rst | 1 - docs/api.rst | 14 +- docs/client.rst | 1 - docs/conf.py | 179 ++++++++++++----------- docs/cookbook/actions.rst | 2 +- docs/cookbook/resourcePools.rst | 2 +- docs/cookbook/ztpsVMonEOS.rst | 2 +- docs/cookbook/ztpsVMonEOS/l2wom.rst | 2 +- docs/examples.rst | 31 ++-- docs/internals.rst | 1 - docs/overview.rst | 16 +- docs/setup_rtd_files.sh | 1 - docs/tips.rst | 1 - docs/troubleshooting.rst | 7 +- plugins/allocate | 2 +- plugins/test | 2 +- test/neighbordb/README.md | 6 +- test/neighbordb/chassis_test.yml | 1 - test/neighbordb/large_pattern_test.yml | 2 - test/neighbordb/node_interfaces_test.yml | 1 - test/neighbordb/node_test.yml | 1 - test/neighbordb/small_pattern_test.yml | 2 - utils/create_db.py | 6 +- 38 files changed, 149 insertions(+), 174 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92db598..24ae13f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,4 +5,3 @@ ## 1.1.0 See http://ztpserver.readthedocs.org/en/v1.1.0/ReleaseNotes1.1.html - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6cff453..f66c1ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,9 +4,9 @@ Arista EOS+ ZTPServer provides a community based implementation of a bootstrap s Contributing Code ================= -Arista EOS+ ZTPServer provides all of its source available to anyone on Github at github.com/arista-eosplus/ztpserver. The project is freely available to anyone to fork and use in their own implementations. The Arista EOS+ community gladly accepts pull requests that add new features, enhance existing features or fix bugs. +Arista EOS+ ZTPServer provides all of its source available to anyone on Github at github.com/arista-eosplus/ztpserver. The project is freely available to anyone to fork and use in their own implementations. The Arista EOS+ community gladly accepts pull requests that add new features, enhance existing features or fix bugs. -All contributed code should be done using pull requests. Once a pull request is initiated, a member of the Arista EOS+ community will review the code and either accept it as is or provide feedback on things to change. +All contributed code should be done using pull requests. Once a pull request is initiated, a member of the Arista EOS+ community will review the code and either accept it as is or provide feedback on things to change. As a procedural note, all pull requests that add new features and/or enhance the operation of ZTPServer are expected to have corresponding test cases with them. Pull requests will be not accepted without them. @@ -16,7 +16,7 @@ Another available way to contribute to this project is to provide requests for f Bugs ==== -If you happen to find a bug with ZTPServer, please open an issue and flag it as a bug. In the issue description please provide details about what you were trying to do, any errors and/or tracebacks and any other information necessary to replicate the bug. +If you happen to find a bug with ZTPServer, please open an issue and flag it as a bug. In the issue description please provide details about what you were trying to do, any errors and/or tracebacks and any other information necessary to replicate the bug. Contact ======= diff --git a/INSTALL.md b/INSTALL.md index 3520b4b..8fa8185 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -15,4 +15,4 @@ $ sudo easy_install PyYaml $ sudo make install $ ztps -`````` \ No newline at end of file +`````` diff --git a/conf/bootstrap.conf b/conf/bootstrap.conf index 3f3b47a..3bbb805 100644 --- a/conf/bootstrap.conf +++ b/conf/bootstrap.conf @@ -15,4 +15,3 @@ # ... # # See documentation for the detailed list of possible values. - diff --git a/conf/neighbordb b/conf/neighbordb index f874ec9..5337fb5 100644 --- a/conf/neighbordb +++ b/conf/neighbordb @@ -22,4 +22,3 @@ # ... # # See documentation for the detailed list of possible values. - diff --git a/conf/ztpserver.conf b/conf/ztpserver.conf index 3df9ef1..13fbec9 100644 --- a/conf/ztpserver.conf +++ b/conf/ztpserver.conf @@ -23,10 +23,10 @@ disable_topology_validation = False [server] -# Note: this section only applies to using the standalone server. If +# Note: this section only applies to using the standalone server. If # running under a WSGI server, these values are ignored -# Interface to which the server will bind to (0:0:0:0 will bind to +# Interface to which the server will bind to (0:0:0:0 will bind to # all available IPv4 addresses on the local machine) interface = 0.0.0.0 diff --git a/conf/ztpserver.wsgi b/conf/ztpserver.wsgi index de23d18..c206f8c 100644 --- a/conf/ztpserver.wsgi +++ b/conf/ztpserver.wsgi @@ -35,7 +35,7 @@ import sys from ztpserver.app import start_wsgiapp -sys.stdout.write('Starting ZTPServer...') +sys.stdout.write("Starting ZTPServer...") application = start_wsgiapp() # To enable debug output, use: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 95c93c5..1319a9b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -56,4 +56,3 @@ Building / Publishing Docs locally * `make` \(default make target is now `make html`\) * Open file://_build/html/index.html in your browser to view. * Publish by copying `docs/_build/html/*` to the `gh-pages` branch - diff --git a/docs/README.md b/docs/README.md index 5307b83..da9bb53 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,4 +19,3 @@ Documenting REST APIs --------------------- REST APIs are documented via the [httpdomain](https://pythonhosted.org/sphinxcontrib-httpdomain/) plugin for sphinx. - diff --git a/docs/ReleaseNotes1.1.rst b/docs/ReleaseNotes1.1.rst index 253036e..a072a47 100644 --- a/docs/ReleaseNotes1.1.rst +++ b/docs/ReleaseNotes1.1.rst @@ -11,7 +11,7 @@ Enhancements * V1.1.0 docs (`181 `_) Documentation has been completely restructured and is now hosted at http://ztpserver.readthedocs.org/. * refresh_ztps - util script to refresh ZTP Server installation (`177 `_) - /utils/refresh_ztps can be used in order to automatically refresh the installation of ZTP Server to the latest code on GitHub. This can be useful in order to pull bug fixes or run the latest version of various development branches. + /utils/refresh_ztps can be used in order to automatically refresh the installation of ZTP Server to the latest code on GitHub. This can be useful in order to pull bug fixes or run the latest version of various development branches. * Et49 does not match Ethernet49 in neighbordb/pattern files (`172 `_) The local interface in an interface pattern does not have to use the long interface name. For example, all of the following will be treated similarly: Et1, e1, et1, eth1, Eth1, ethernet1, Ethernet1. @@ -100,8 +100,8 @@ Enhancements “domain”*: , “password”*: , “nickname”: , // REMOVED - “rooms”*: [ , … ] - “msg_type”: [ “info” | “debug” ] // Optional, default “debug” + “rooms”*: [ , … ] + “msg_type”: [ “info” | “debug” ] // Optional, default “debug” } } @@ -224,4 +224,3 @@ Resolved issues .. comment * fix issue with Pattern creation from neighbordb (`44 `_) .. comment - diff --git a/docs/ReleaseNotes1.2.rst b/docs/ReleaseNotes1.2.rst index f9b76d7..d051a38 100644 --- a/docs/ReleaseNotes1.2.rst +++ b/docs/ReleaseNotes1.2.rst @@ -72,7 +72,7 @@ Enhancements s7056.lab.local - - [03/Nov/2014 21:05:33] "POST /nodes HTTP/1.1" 400 0 * Deal more gracefully with DNS/connectivity errors while trying to access remote syslog servers (`215 `_) - Logging errors (e.g. bogus destination) will not be automatically logged by the bootstrap script. In order to debug logging issues, simply uncomment the following lines in the bootstrap script: + Logging errors (e.g. bogus destination) will not be automatically logged by the bootstrap script. In order to debug logging issues, simply uncomment the following lines in the bootstrap script: :: #---------------------------------SYSLOG---------------------- @@ -179,4 +179,3 @@ Fixed * ZTPS server fails to write .node because lack of permissions (`126 `_) .. comment - diff --git a/docs/ReleaseNotes1.3.1.rst b/docs/ReleaseNotes1.3.1.rst index 7128db9..6680200 100644 --- a/docs/ReleaseNotes1.3.1.rst +++ b/docs/ReleaseNotes1.3.1.rst @@ -9,4 +9,3 @@ Bug fixes ^^^^^^^^^ * fixes *pip* install/uninstall issues - diff --git a/docs/ReleaseNotes1.3.2.rst b/docs/ReleaseNotes1.3.2.rst index ea86209..ca4f031 100644 --- a/docs/ReleaseNotes1.3.2.rst +++ b/docs/ReleaseNotes1.3.2.rst @@ -31,4 +31,3 @@ Bug fixes .. comment - ZTP Server benchmarking results .. comment - diff --git a/docs/ReleaseNotes1.3.rst b/docs/ReleaseNotes1.3.rst index 5bb6b83..13b5b24 100644 --- a/docs/ReleaseNotes1.3.rst +++ b/docs/ReleaseNotes1.3.rst @@ -30,7 +30,7 @@ Enhancements Validating definitions... Validating /usr/share/ztpserver/definitions/leaf.definition... Ok! Validating /usr/share/ztpserver/definitions/leaf-no_vars.definition... Ok! - + Validating resources... Validating /usr/share/ztpserver/resources/leaf_man_ip... Ok! Validating /usr/share/ztpserver/resources/leaf_spine_ip... @@ -40,11 +40,11 @@ Enhancements 10.0.0.53/24: null dfdsf dsfsd 10.0.0.54/24: JPE14140273 - + Error: while scanning a simple key in "", line 3, column 1: - dfdsf dsfsd + dfdsf dsfsd could not found expected ':' in "", line 5, column 1: 10.0.0.54/24: JPE14140273 @@ -97,4 +97,3 @@ Bug fixes logging: xmpp: ... - diff --git a/docs/ReleaseNotes1.5.0.rst b/docs/ReleaseNotes1.5.0.rst index 3f21701..4d7ebf5 100644 --- a/docs/ReleaseNotes1.5.0.rst +++ b/docs/ReleaseNotes1.5.0.rst @@ -25,5 +25,3 @@ Fixed Known Caveats ^^^^^^^^^^^^^ - - diff --git a/docs/actions.rst b/docs/actions.rst index 180a8a3..3877bae 100644 --- a/docs/actions.rst +++ b/docs/actions.rst @@ -69,4 +69,3 @@ Actions # #.. automodule:: template # :members: - diff --git a/docs/api.rst b/docs/api.rst index 553da35..7874c2e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -130,16 +130,16 @@ provisioned. Content-Type: application/json { - “model”*: , - “serialnumber”*: , + “model”*: , + “serialnumber”*: , “systemmac”*: , - “version”*: , + “version”*: , “neighbors”*: { : [ { - 'device': , + 'device': , 'remote_interface': } ] - }, + }, } **Note**: \* Items are mandatory (even if value is empty list/dict) @@ -148,7 +148,7 @@ provisioned. Status: 201 Created OR 409 Conflict will both return: - .. sourcecode:: http + .. sourcecode:: http Content-Type: text/html Location: @@ -233,7 +233,7 @@ This is used to retrieve the startup-config that was backed-up from a node to th Status: 201 Created OR 409 Conflict will both return: - .. sourcecode:: http + .. sourcecode:: http Content-Type: text/plain diff --git a/docs/client.rst b/docs/client.rst index cea40fe..bfd68c7 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -8,4 +8,3 @@ Bootstrap Client .. autoclass:: Node :members: - diff --git a/docs/conf.py b/docs/conf.py index 4e222c2..f4aaa4d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,38 +13,41 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os import re +import sys + # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -#sys.path.insert(0, os.path.join('ztpserver', 'lib')) -#sys.path.insert(0, os.path.join('ztpserver')) -sys.path.insert(0, os.path.join('..', 'actions')) -sys.path.insert(0, os.path.join('..', 'client')) -sys.path.insert(0, os.path.join('actions')) -sys.path.insert(0, os.path.join('client')) -sys.path.insert(0, os.path.abspath('..')) +# sys.path.insert(0, os.path.join('ztpserver', 'lib')) +# sys.path.insert(0, os.path.join('ztpserver')) +sys.path.insert(0, os.path.join("..", "actions")) +sys.path.insert(0, os.path.join("..", "client")) +sys.path.insert(0, os.path.join("actions")) +sys.path.insert(0, os.path.join("client")) +sys.path.insert(0, os.path.abspath("..")) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +# extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] extensions = [ "sphinx.ext.autodoc", "sphinx.ext.viewcode", @@ -53,211 +56,205 @@ ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'ZTPServer' -copyright = u'2015, Arista Networks' # pylint: disable=W0622 +project = "ZTPServer" +copyright = "2015, Arista Networks" # pylint: disable=W0622 # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -#version = '1.2.0' +# version = '1.2.0' # The full version, including alpha/beta/rc tags. -#release = '1.2.0' +# release = '1.2.0' -release = open('../VERSION').read().split()[0].strip() +release = open("../VERSION").read().split()[0].strip() # Assume PEP 440 version strings -p = re.compile('(\d+!)?((\d+)(.\d+)*(.\d+)*)(.?[a|b|rc]\d*)?(.post\d*)?(.dev\d*)?', re.IGNORECASE) -vers = p.search(release) +p = re.compile("(\d+!)?((\d+)(.\d+)*(.\d+)*)(.?[a|b|rc]\d*)?(.post\d*)?(.dev\d*)?", re.IGNORECASE) +vers = p.search(release) version = vers.group(2) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -today_fmt = '%B %d, %Y' +today_fmt = "%B %d, %Y" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'old', 'exts', 'cookbook/_template.rst'] +exclude_patterns = ["_build", "old", "exts", "cookbook/_template.rst"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False show_authors = True # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -#html_theme = 'default' +# html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None -#html_logo = '_static/AristaLogo.png' -html_logo = '_static/arista_logo_11-trans-w.png' +# html_logo = None +# html_logo = '_static/AristaLogo.png' +html_logo = "_static/arista_logo_11-trans-w.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None -#html_favicon = 'favicon.ico' -html_favicon = '_static/favicon.ico' +# html_favicon = None +# html_favicon = 'favicon.ico' +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' -html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%b %d, %Y" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'ZTPServerdoc' +htmlhelp_basename = "ZTPServerdoc" # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'ZTPServer.tex', u'ZTPServer Documentation', - u'Arista Networks', 'manual'), + ("index", "ZTPServer.tex", "ZTPServer Documentation", "Arista Networks", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None -latex_logo = '_static/arista_logo_jpg-11.jpg' +# latex_logo = None +latex_logo = "_static/arista_logo_jpg-11.jpg" # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'ztpserver', u'ZTPServer Documentation', - [u'Arista Networks'], 1) -] +man_pages = [("index", "ztpserver", "ZTPServer Documentation", ["Arista Networks"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -266,16 +263,22 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'ZTPServer', u'ZTPServer Documentation', - u'Arista Networks', 'ZTPServer', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "ZTPServer", + "ZTPServer Documentation", + "Arista Networks", + "ZTPServer", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' diff --git a/docs/cookbook/actions.rst b/docs/cookbook/actions.rst index 72d6c49..04efd0d 100644 --- a/docs/cookbook/actions.rst +++ b/docs/cookbook/actions.rst @@ -154,7 +154,7 @@ The ZTPServer will then write the SYSTEM_ID as the value, overwriting ``null``. If you wanted to use the assigned value elsewhere in the definition, simply call ``allocate(mgmt_subnet)`` and the plugin will not assign a new value, rather it will return the key already assigned. Note that this is an implementation-detail -specific to this particular plugin - other plugins might vary (please read the +specific to this particular plugin - other plugins might vary (please read the associated documentation for each). The result would look like: diff --git a/docs/cookbook/resourcePools.rst b/docs/cookbook/resourcePools.rst index 4491214..2c62c45 100644 --- a/docs/cookbook/resourcePools.rst +++ b/docs/cookbook/resourcePools.rst @@ -86,6 +86,6 @@ Explanation Clearing all resource pools can be done via the command line on the ZTPServer. The command will analyze ``data_root/resources`` and any file that exists in -that directory that resembles a ZTPServer resource pool will be cleared. +that directory that resembles a ZTPServer resource pool will be cleared. .. End of Clearing a Resource Pool diff --git a/docs/cookbook/ztpsVMonEOS.rst b/docs/cookbook/ztpsVMonEOS.rst index b1ceb98..ded0330 100644 --- a/docs/cookbook/ztpsVMonEOS.rst +++ b/docs/cookbook/ztpsVMonEOS.rst @@ -56,4 +56,4 @@ Explanation The USB key method leverages the Arista Password Recovery mechanism. When the ``fullrecover`` and ``boot-config`` file is present on the USB key, the system will check the timestamp on the ``boot-config`` file.If the timestamp is different, all files on the USB key will be copied to the flash on the switch, and the switch will be rebooted and come up with the ``startup-config`` and the ``EOS.swi`` included on the USB key. -.. End of Deployment Steps \ No newline at end of file +.. End of Deployment Steps diff --git a/docs/cookbook/ztpsVMonEOS/l2wom.rst b/docs/cookbook/ztpsVMonEOS/l2wom.rst index 6ca953f..234759a 100644 --- a/docs/cookbook/ztpsVMonEOS/l2wom.rst +++ b/docs/cookbook/ztpsVMonEOS/l2wom.rst @@ -237,4 +237,4 @@ The reason we could not just bridge Vlan1 with the Linux bridge (and therefore j -.. End of ztps.xml \ No newline at end of file +.. End of ztps.xml diff --git a/docs/examples.rst b/docs/examples.rst index 7059af3..8c88883 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -13,41 +13,41 @@ Global configuration file [default] # Location of all ztps boostrap process data files data_root = /usr/share/ztpserver - + # UID used in the /nodes structure (serialnumber or systemmac) identifier = serialnumber - + # Server URL to-be-advertised to clients (via POST replies) during the bootstrap process server_url = http://172.16.130.10:8080 - + # Enable local logging logging = True - + # Enable console logging console_logging = True - + # Console logging format console_logging_format = %(asctime)s:%(levelname)s:[%(module)s:%(lineno)d] %(message)s # Globally disable topology validation in the bootstrap process disable_topology_validation = False - + [server] # Note: this section only applies to using the standalone server. If # running under a WSGI server, these values are ignored - + # Interface to which the server will bind to (0:0:0:0 will bind to # all available IPv4 addresses on the local machine) interface = 172.16.130.10 - + # TCP listening port port = 8080 - + [bootstrap] # Bootstrap filename (file located in /bootstrap) filename = bootstrap - + [neighbordb] # Neighbordb filename (file located in ) filename = neighbordb @@ -181,7 +181,7 @@ Sample templates .. _resources_example: -Sample resources +Sample resources ```````````````` :: @@ -219,11 +219,11 @@ Example #1 node: ABC12345678 interfaces: - Ethernet49: pod1-spine1:Ethernet1/1 - - Ethernet50: + - Ethernet50: device: pod1-spine2 port: Ethernet1/1 -In example #1, the topology map would only apply to a node with system ID +In example #1, the topology map would only apply to a node with system ID equal to **ABC12345678**. The following interface map rules apply: - Interface Ethernet49 must be connected to node pod1-spine1 on port @@ -242,7 +242,7 @@ Example #2 node: 001c73aabbcc interfaces: - any: regex('pod\d+-spine\d+'):Ethernet1/$ - - any: + - any: device: regex('pod\d+-spine1') port: Ethernet2/3 @@ -273,7 +273,7 @@ Example #3 - Ethernet2: $pod1-spine2:any - any: excludes('spine1'):Ethernet49 - any: excludes('spine2'):Ethernet49 - - Ethernet49: + - Ethernet49: device: $not_spine port: Ethernet49 - Ethernet50: @@ -372,4 +372,3 @@ More examples ````````````` Additional ZTPServer file examples are available on GitHub at the `ZTPServer Demo `_. - diff --git a/docs/internals.rst b/docs/internals.rst index 18055a5..b7fc381 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -7,4 +7,3 @@ Internals implementation api modules - diff --git a/docs/overview.rst b/docs/overview.rst index 3733562..d921f40 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -3,7 +3,7 @@ Overview ZTPServer provides a robust server which enables comprehensive bootstrap solutions for Arista network elements. ZTPserver takes advantage of the the ZeroTouch Provisioning (ZTP) feature in Arista's EOS (Extensible Operating System) which enables a node to connect to a provisioning server whenever a valid configuration file is missing from the internal flash storage. -ZTPServer provides a number of features that extend beyond simply loading a configuration file and a boot image on a node, including: +ZTPServer provides a number of features that extend beyond simply loading a configuration file and a boot image on a node, including: * sending an advanced bootstrap client to the node * mapping each node to an individual definition which describes the bootstrap steps specific to that node @@ -12,7 +12,7 @@ ZTPServer provides a number of features that extend beyond simply loading a conf * validation topology using a simple syntax for expressing LLDP neighbor adjacencies * enabling Zero Touch Replacement, as well as configuration backup and management -ZTPServer is written in Python and leverages standard protocols like DHCP (DHCP options for boot functions), HTTP(S) (for bi-directional transport), XMPP and syslog (for logging). Most of the configuration files are YAML-based. +ZTPServer is written in Python and leverages standard protocols like DHCP (DHCP options for boot functions), HTTP(S) (for bi-directional transport), XMPP and syslog (for logging). Most of the configuration files are YAML-based. **Highlights:** @@ -51,7 +51,7 @@ See the `ZTP Tech Bulletin Date: Thu, 2 May 2024 12:07:26 +0000 Subject: [PATCH 10/15] [pre-commit.ci lite] apply automatic fixes --- ztpserver/topology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ztpserver/topology.py b/ztpserver/topology.py index aff3093..a022166 100644 --- a/ztpserver/topology.py +++ b/ztpserver/topology.py @@ -506,7 +506,7 @@ def __init__( self.node_id = node_id self.variables = variables or {} - self.model=model + self.model = model self.interfaces = [] if interfaces: From 19120aaf08ca9e4844676e41a7166b2eb437a54b Mon Sep 17 00:00:00 2001 From: David Lobato Date: Thu, 2 May 2024 13:23:41 +0100 Subject: [PATCH 11/15] Fix validation error with optional attribute --- ztpserver/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ztpserver/validators.py b/ztpserver/validators.py index 338acea..ca9eafc 100644 --- a/ztpserver/validators.py +++ b/ztpserver/validators.py @@ -40,7 +40,7 @@ from ztpserver.utils import expand_range, parse_interface REQUIRED_PATTERN_ATTRIBUTES = ["name", "definition"] -OPTIONAL_PATTERN_ATTRIBUTES = ["node", "variables", "interfaces", "model"] +OPTIONAL_PATTERN_ATTRIBUTES = ["node", "variables", "interfaces", "model", "config_handler"] INTERFACE_PATTERN_KEYWORDS = ["any", "none"] ANTINODE_PATTERN = rf"[^{string.hexdigits}]" KW_ANY_RE = re.compile(r" *any *") From 07f44892ddc2addcdde09f4df5f0ce294b1af157 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Thu, 2 May 2024 12:54:06 +0100 Subject: [PATCH 12/15] Add test_image_name_from_url --- actions/install_image | 2 +- test/actions/test_install_image.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/actions/install_image b/actions/install_image index 6593288..4030a08 100644 --- a/actions/install_image +++ b/actions/install_image @@ -117,7 +117,7 @@ def main(attributes): # In all other cases, copy the image if image_name_from_url: - image = url.split("/")[-1] + image = url.rsplit("/", 1)[-1] else: image = "EOS-{}.swi".format(version) diff --git a/test/actions/test_install_image.py b/test/actions/test_install_image.py index 3277587..abbdd90 100644 --- a/test/actions/test_install_image.py +++ b/test/actions/test_install_image.py @@ -150,6 +150,42 @@ def test_success(self): remove_file(image_file) bootstrap.end_test() + def test_image_name_from_url(self): + bootstrap = Bootstrap(ztps_default_config=True) + version = random_string() + image = random_string() + url = "http://{}/{}".format(bootstrap.server, image) + bootstrap.ztps.set_definition_response( + actions=[ + { + "action": "test_action", + "attributes": {"url": url, "version": version, "image_name_from_url": True}, + }, + {"action": "startup_config_action"}, + ] + ) + + action = get_action("install_image") + bootstrap.ztps.set_action_response("test_action", action) + bootstrap.ztps.set_action_response("startup_config_action", startup_config_action()) + bootstrap.ztps.set_file_response(image, print_action()) + bootstrap.start_test() + + image_file = "{}/{}".format(bootstrap.flash, url.rsplit("/", 1)[-1]) + try: + self.assertTrue(os.path.isfile(image_file)) + self.assertTrue(bootstrap.success()) + self.assertEqual( + eapi_log()[-1], "install source flash:{}".format(url.rsplit("/", 1)[-1]) + ) + except AssertionError as assertion: + print("Output: {}".format(bootstrap.output)) + print("Error: {}".format(bootstrap.error)) + raise_exception(assertion) + finally: + remove_file(image_file) + bootstrap.end_test() + if __name__ == "__main__": unittest.main() From 8813c43b03347b222de8c821c17b4ac4dc5717aa Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 25 Jun 2024 09:46:18 +0100 Subject: [PATCH 13/15] Update docs --- docs/config.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index b947ed6..4cb7578 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -476,9 +476,10 @@ unique_id of the node) and global patterns. Rules: - if multiple node-specific entries reference the same unique_id, only the first will be in effect - all others will be ignored - - if the **node** and **interfaces** or **model** attributes are specified and a node's unique_id is a match, but the topology information or model information are not, then the overall match will fail and the global patterns will not be considered - - **model** can't be the only matching attributes, **node** or **interfaces** are mandatory to have a valid match + - if both the **node** and **interfaces** attributes are specified and a node's unique_id is a match, but the topology information is not, then the overall match will fail and the global patterns will not be considered + - if both the **model** and **interfaces** attributes are specified and a node's model is a match, but the topology information is not, then the overall match will fail and the global patterns will not be considered - if there is no matching node-specific pattern for a node's unique_id, then the server will attempt to match the node against the global patterns (in the order they are specified in ``neighbordb``) + - **node** and **model** are mutually exclusive and can't be specified in the same pattern - if a node-specific pattern matches, the server will automatically generate an open pattern in the node's folder. This pattern will match any device with at least one LLDP-capable neighbor. Example: ``any: any:any`` .. code-block:: yaml From f61c98224ad02b21cd3c857fc4d684d48a72b103 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 25 Jun 2024 09:59:07 +0100 Subject: [PATCH 14/15] Add rtd config --- .readthedocs.yml | 31 +++++++++++++++++++++++++++++++ docs/conf.py | 15 +++------------ 2 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..2421d02 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,31 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + jobs: + pre_build: + - cd docs && make actionlinks + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/conf.py b/docs/conf.py index f4aaa4d..39def73 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,14 +17,7 @@ import re import sys -# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org -on_rtd = os.environ.get("READTHEDOCS", None) == "True" - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme = "sphinx_rtd_theme" # sys.path.insert(0, os.path.join('ztpserver', 'lib')) # sys.path.insert(0, os.path.join('ztpserver')) @@ -34,7 +27,6 @@ sys.path.insert(0, os.path.join("client")) sys.path.insert(0, os.path.abspath("..")) - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -69,7 +61,7 @@ # General information about the project. project = "ZTPServer" -copyright = "2015, Arista Networks" # pylint: disable=W0622 +copyright = "2024, Arista Networks" # pylint: disable=W0622 # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -82,7 +74,7 @@ release = open("../VERSION").read().split()[0].strip() # Assume PEP 440 version strings -p = re.compile("(\d+!)?((\d+)(.\d+)*(.\d+)*)(.?[a|b|rc]\d*)?(.post\d*)?(.dev\d*)?", re.IGNORECASE) +p = re.compile(r"(\d+!)?((\d+)(.\d+)*(.\d+)*)(.?(a|b|rc)\d*)?(.post\d*)?(.dev\d*)?", re.IGNORECASE) vers = p.search(release) version = vers.group(2) @@ -207,7 +199,6 @@ # Output file base name for HTML help builder. htmlhelp_basename = "ZTPServerdoc" - # -- Options for LaTeX output -------------------------------------------------- latex_elements = { From 67f99d2fcc971a6d538871a17e564955c018ceb4 Mon Sep 17 00:00:00 2001 From: David Lobato Date: Tue, 2 Jul 2024 09:54:09 +0100 Subject: [PATCH 15/15] Release 2.0.2 --- docs/ReleaseNotes2.0.2.rst | 20 ++++++++++++++++++++ docs/support.rst | 1 + 2 files changed, 21 insertions(+) create mode 100644 docs/ReleaseNotes2.0.2.rst diff --git a/docs/ReleaseNotes2.0.2.rst b/docs/ReleaseNotes2.0.2.rst new file mode 100644 index 0000000..06261fd --- /dev/null +++ b/docs/ReleaseNotes2.0.2.rst @@ -0,0 +1,20 @@ +Release 2.0.2 +------------- + +New Modules +^^^^^^^^^^^ + +Enhancements +^^^^^^^^^^^^ + +* install_image change image to reference filename provided by url attribute (`406 `_)[`siguroot `_] +* Added model filter in neighbordb (`403 `_)[`az-blip `_] + +Fixed +^^^^^ + +* Fix controller.py:put_config body decoding (`407 `_)[`dlobato `_] + + +Known Caveats +^^^^^^^^^^^^^ diff --git a/docs/support.rst b/docs/support.rst index 4fa0afc..970e098 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -42,6 +42,7 @@ The authoritative state for any known issue can be found in `GitHub issues