From c5e71963aeb4aacdf4d82b9f076b993a7f7251be Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Tue, 2 Jan 2024 03:24:50 +0530 Subject: [PATCH 01/17] fixed collectstatic command --- netbox_cloud_pilot/iaas.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/netbox_cloud_pilot/iaas.py b/netbox_cloud_pilot/iaas.py index 0c2eae5..57df594 100644 --- a/netbox_cloud_pilot/iaas.py +++ b/netbox_cloud_pilot/iaas.py @@ -459,12 +459,11 @@ def install_plugin( plugins[plugin.get("app_label")] = plugin_settings or {} self.dump_plugins(plugins) - # TODO: Uncomment this after a fix is suggested by Virtuozzo, slack thread: https://omsmsp.slack.com/archives/C05QT7WD71U/p1704140085788849 # Run collectstatic command - # self.execute_cmd( - # node_id=master_node_id, - # command=f"{activate_env} && /opt/netbox/netbox/manage.py collectstatic --no-input", - # ) + self.execute_cmd( + node_id=master_node_id, + command=f"{activate_env} && /opt/netbox/netbox/manage.py collectstatic --no-input --clear 1>/dev/null", + ) return self.restart_nodes( node_groups=[ From 694123461cff025a8722c5d67daec1cae14501b0 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Tue, 2 Jan 2024 17:08:31 +0530 Subject: [PATCH 02/17] install git for private plugins --- netbox_cloud_pilot/iaas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/netbox_cloud_pilot/iaas.py b/netbox_cloud_pilot/iaas.py index 57df594..fc8a412 100644 --- a/netbox_cloud_pilot/iaas.py +++ b/netbox_cloud_pilot/iaas.py @@ -441,6 +441,9 @@ def install_plugin( # Install the plugin version if plugin.get("private"): + # Ensure `git` is installed + self.execute_cmd(master_node_id, "apt-get install -y git") + github_url = plugin.get("github_url") github_url = github_url.replace( "https://github.com", f"git+https://{github_token}@github.com" From d6acd1ca40ace6c5207365b5150e6c53cccfebe0 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Tue, 2 Jan 2024 17:08:37 +0530 Subject: [PATCH 03/17] lint fixes --- README.md | 12 ++++++------ setup.py | 6 ++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7a1c47b..795b827 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # NetBox Cloud Pilot -[NetBox Cloud Pilot](https://github.com/Onemind-Services-LLC/netbox-cloud-pilot) is a specialized plugin tailored for -NetBox, offering enhanced functionality for users on the -[CloudMyDC public Cloud platform (VAP)](https://app.xapp.cloudmydc.com/). This plugin simplifies the management of +[NetBox Cloud Pilot](https://github.com/Onemind-Services-LLC/netbox-cloud-pilot) is a specialized plugin tailored for +NetBox, offering enhanced functionality for users on the +[CloudMyDC public Cloud platform (VAP)](https://app.xapp.cloudmydc.com/). This plugin simplifies the management of various NetBox components and infrastructure, streamlining processes and offering a more integrated experience. -NetBox Cloud Pilot, developed and maintained by [Onemind Services LLC](https://onemindservices.com/), introduces an -exclusive plugin store for NetBox. It caters to users with a diverse range of both free and paid plugins, enhancing the +NetBox Cloud Pilot, developed and maintained by [Onemind Services LLC](https://onemindservices.com/), introduces an +exclusive plugin store for NetBox. It caters to users with a diverse range of both free and paid plugins, enhancing the NetBox experience. ## Features @@ -35,5 +35,5 @@ For support, questions, or feedback, please file an issue on our [GitHub issue t ## Contributing -Contributions to NetBox Cloud Pilot are welcome! If you'd like to contribute, please fork the repository and submit a +Contributions to NetBox Cloud Pilot are welcome! If you'd like to contribute, please fork the repository and submit a pull request. For more detailed information, refer to our contributing guidelines in the repository. diff --git a/setup.py b/setup.py index 935c089..165d126 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ from setuptools import find_packages, setup -description = "Enhances NetBox on CloudMyDC's VAP with advanced management and control features." +description = ( + "Enhances NetBox on CloudMyDC's VAP with advanced management and control features." +) -with open("README.md", "r") as fh: +with open("README.md") as fh: long_description = fh.read() setup( From 3e440909aaf42e1d0495b586b442a97225731533 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Tue, 2 Jan 2024 18:23:13 +0530 Subject: [PATCH 04/17] fixed BASE_PATH initial value --- netbox_cloud_pilot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_cloud_pilot/constants.py b/netbox_cloud_pilot/constants.py index 2a6b2eb..386a745 100644 --- a/netbox_cloud_pilot/constants.py +++ b/netbox_cloud_pilot/constants.py @@ -174,7 +174,7 @@ help_text="Base path for URL patterns", placeholder="/", required=False, - initial="/", + initial=None, ), Param( key="EMAIL_FROM", From c863074f88003922ca039d3447040d8955667c18 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Tue, 2 Jan 2024 20:48:01 +0530 Subject: [PATCH 05/17] fixed required_settings check --- netbox_cloud_pilot/forms.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index 0d14773..378ff98 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -322,6 +322,7 @@ def __init__(self, *args, **kwargs): def clean(self): plugins = get_plugins_list() plugin = plugins.get(self.cleaned_data.get("name")) + selected_version = self.cleaned_data.get("version") # If the plugin is private, ensure that license is provided nc = NetBoxConfiguration.objects.first() @@ -344,7 +345,14 @@ def clean(self): ) # Get the required_settings from the plugin - required_settings = plugin.get("required_settings", []) + required_settings = next( + ( + release.get("netbox", {}).get("required_settings", []) + for release in plugin.get("releases", []) + if release.get("tag") == selected_version + ), + [], + ) configuration = self.cleaned_data.get("configuration") # Check if the required_settings are in the configuration From ff9a183b9b9f51530bed63bf78ef476095f6f6d2 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Wed, 3 Jan 2024 00:16:12 +0530 Subject: [PATCH 06/17] minor fix --- .../templates/netbox_cloud_pilot/plugins_store.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html index 3531033..d122bb9 100644 --- a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html +++ b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html @@ -17,7 +17,7 @@
- {% if plugin.private %}Paid{% else %}Free{% endif %} + {% if plugin.private %}Subscription{% else %}Community{% endif %} · {{ plugin.stars }} From 1a714457e13741db0516635d107f2f23b1bd64ed Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Wed, 3 Jan 2024 01:51:55 +0530 Subject: [PATCH 07/17] hide github link for private plugins --- .../templates/netbox_cloud_pilot/plugins_store.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html index d122bb9..b75256b 100644 --- a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html +++ b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html @@ -29,7 +29,7 @@

{{ plugin_name }}

{{ plugin.description }}

- {% if plugin.github_url %} + {% if plugin.github_url and not plugin.private %} View on GitHub From 43c1dee80da415c6799f1589a3415a5656294cef Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Wed, 3 Jan 2024 02:27:24 +0530 Subject: [PATCH 08/17] improved plugin store view --- .../netbox_cloud_pilot/inc/plugin_store.html | 47 ++++++++++++++ .../netbox_cloud_pilot/plugins_store.html | 63 ++++++------------- netbox_cloud_pilot/views.py | 28 +++++++++ 3 files changed, 93 insertions(+), 45 deletions(-) create mode 100644 netbox_cloud_pilot/templates/netbox_cloud_pilot/inc/plugin_store.html diff --git a/netbox_cloud_pilot/templates/netbox_cloud_pilot/inc/plugin_store.html b/netbox_cloud_pilot/templates/netbox_cloud_pilot/inc/plugin_store.html new file mode 100644 index 0000000..6b23a13 --- /dev/null +++ b/netbox_cloud_pilot/templates/netbox_cloud_pilot/inc/plugin_store.html @@ -0,0 +1,47 @@ +
+
+
+ + {% if plugin.private %}Subscription{% else %}Community{% endif %} + + · + {{ plugin.stars }} + + {% if plugin.installed %} + v{{ plugin.current_version }} + {% endif %} +
+
+

{{ plugin_name }}

+

{{ plugin.description }}

+ {% if plugin.github_url and not plugin.private %} + + View on GitHub + + {% endif %} + {% if plugin.pypi_url %} + + View on PyPI + + {% endif %} +
+
+ {% if plugin.installed %} + + Configure + + + + Uninstall + + {% else %} + + Install + + {% endif %} +
+
+
\ No newline at end of file diff --git a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html index b75256b..3a9a37c 100644 --- a/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html +++ b/netbox_cloud_pilot/templates/netbox_cloud_pilot/plugins_store.html @@ -12,51 +12,24 @@
{% block content %}
- {% for plugin_name, plugin in plugins.items %} -
-
-
- - {% if plugin.private %}Subscription{% else %}Community{% endif %} - - · - {{ plugin.stars }} - - {% if plugin.installed %} - v{{ plugin.current_version }} - {% endif %} -
-
-

{{ plugin_name }}

-

{{ plugin.description }}

- {% if plugin.github_url and not plugin.private %} - - View on GitHub - - {% endif %} - {% if plugin.pypi_url %} - - View on PyPI - - {% endif %} -
-
- {% if plugin.installed %} - - Configure - - - - Uninstall - - {% else %} - - Install - - {% endif %} -
-
-
+

Installed Plugins

+
+ {% for plugin_name, plugin in plugins.installed.items %} + {% include 'netbox_cloud_pilot/inc/plugin_store.html' with plugin_name=plugin_name plugin=plugin %} + {% endfor %} +
+
+

Subscription Store

+
+ {% for plugin_name, plugin in plugins.not_installed.subscription.items %} + {% include 'netbox_cloud_pilot/inc/plugin_store.html' with plugin_name=plugin_name plugin=plugin %} + {% endfor %} +
+
+

Community Store

+
+ {% for plugin_name, plugin in plugins.not_installed.community.items %} + {% include 'netbox_cloud_pilot/inc/plugin_store.html' with plugin_name=plugin_name plugin=plugin %} {% endfor %}
{% endblock %} diff --git a/netbox_cloud_pilot/views.py b/netbox_cloud_pilot/views.py index 2ad4900..572688c 100644 --- a/netbox_cloud_pilot/views.py +++ b/netbox_cloud_pilot/views.py @@ -286,6 +286,34 @@ def get(self, request): } ) + # Divide the plugins into two lists: installed and not installed + plugins = { + "installed": { + plugin_name: plugin + for plugin_name, plugin in plugins.items() + if plugin.get("installed") + }, + "not_installed": { + plugin_name: plugin + for plugin_name, plugin in plugins.items() + if not plugin.get("installed") + }, + } + + # Divide not installed plugins into two lists: subscription and community + plugins["not_installed"] = { + "subscription": { + plugin_name: plugin + for plugin_name, plugin in plugins["not_installed"].items() + if plugin.get("private") + }, + "community": { + plugin_name: plugin + for plugin_name, plugin in plugins["not_installed"].items() + if not plugin.get("private") + }, + } + return render( request, "netbox_cloud_pilot/plugins_store.html", From 92d7452f5acec39abe1db537ba6f25a6a720e70d Mon Sep 17 00:00:00 2001 From: yash-pal1 Date: Wed, 3 Jan 2024 14:13:06 +0530 Subject: [PATCH 09/17] pyproject.toml file added --- pyproject.toml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1192d59 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +# See PEP 518 for the spec of this file +# https://www.python.org/dev/peps/pep-0518/ + +[tool.black] +line-length = 120 +target_version = ['py38', 'py39', 'py310'] +skip-string-normalization = true + +[tool.isort] +profile = "black" + +[tool.pylint] +max-line-length = 120 + +[tool.pyright] +include = ["netbox_cloud_pilot"] +exclude = [ + "**/node_modules", + "**/__pycache__", +] +reportMissingImports = true +reportMissingTypeStubs = false + From 4c008a42ade4a15b4095cc3c34d3526806a800be Mon Sep 17 00:00:00 2001 From: yash-pal1 Date: Wed, 3 Jan 2024 14:13:28 +0530 Subject: [PATCH 10/17] lint fix --- configuration/configuration.py | 13 +--- netbox_cloud_pilot/constants.py | 56 +++++------------ netbox_cloud_pilot/forms.py | 57 ++++-------------- netbox_cloud_pilot/iaas.py | 99 ++++++++----------------------- netbox_cloud_pilot/models.py | 47 ++++----------- netbox_cloud_pilot/nb_settings.py | 4 +- netbox_cloud_pilot/utils.py | 4 +- netbox_cloud_pilot/views.py | 28 +++------ pyproject.toml | 1 - setup.py | 4 +- 10 files changed, 74 insertions(+), 239 deletions(-) diff --git a/configuration/configuration.py b/configuration/configuration.py index c28e838..f5ae2cd 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -49,10 +49,7 @@ def _read_secret(secret_name, default=None): # Database connection SSLMODE "CONN_MAX_AGE": int(environ.get("DB_CONN_MAX_AGE", "300")), # Max database connection age - "DISABLE_SERVER_SIDE_CURSORS": environ.get( - "DB_DISABLE_SERVER_SIDE_CURSORS", "False" - ).lower() - == "true", + "DISABLE_SERVER_SIDE_CURSORS": environ.get("DB_DISABLE_SERVER_SIDE_CURSORS", "False").lower() == "true", # Disable the use of server-side cursors transaction pooling } @@ -66,10 +63,7 @@ def _read_secret(secret_name, default=None): "PASSWORD": _read_secret("redis_password", environ.get("REDIS_PASSWORD", "")), "DATABASE": int(environ.get("REDIS_DATABASE", 0)), "SSL": environ.get("REDIS_SSL", "False").lower() == "true", - "INSECURE_SKIP_TLS_VERIFY": environ.get( - "REDIS_INSECURE_SKIP_TLS_VERIFY", "False" - ).lower() - == "true", + "INSECURE_SKIP_TLS_VERIFY": environ.get("REDIS_INSECURE_SKIP_TLS_VERIFY", "False").lower() == "true", }, "caching": { "HOST": environ.get("REDIS_CACHE_HOST", environ.get("REDIS_HOST", "localhost")), @@ -79,8 +73,7 @@ def _read_secret(secret_name, default=None): environ.get("REDIS_CACHE_PASSWORD", environ.get("REDIS_PASSWORD", "")), ), "DATABASE": int(environ.get("REDIS_CACHE_DATABASE", 1)), - "SSL": environ.get("REDIS_CACHE_SSL", environ.get("REDIS_SSL", "False")).lower() - == "true", + "SSL": environ.get("REDIS_CACHE_SSL", environ.get("REDIS_SSL", "False")).lower() == "true", "INSECURE_SKIP_TLS_VERIFY": environ.get( "REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY", environ.get("REDIS_INSECURE_SKIP_TLS_VERIFY", "False"), diff --git a/netbox_cloud_pilot/constants.py b/netbox_cloud_pilot/constants.py index 386a745..3cf6e8a 100644 --- a/netbox_cloud_pilot/constants.py +++ b/netbox_cloud_pilot/constants.py @@ -5,9 +5,7 @@ from .nb_settings import * JELASTIC_API = "https://app.xapp.cloudmydc.com" -NETBOX_JPS_REPO = ( - "https://raw.githubusercontent.com/Onemind-Services-LLC/netbox-jps/master" -) +NETBOX_JPS_REPO = "https://raw.githubusercontent.com/Onemind-Services-LLC/netbox-jps/master" NODE_GROUP_CP = "cp" NODE_GROUP_SQLDB = "sqldb" @@ -296,9 +294,7 @@ placeholder="{'filter_name': 'path.to.filter'}", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, initial={}, ), Param( @@ -336,9 +332,7 @@ placeholder="{'location': '/var/netbox/media'}", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, initial={}, ), ], @@ -372,9 +366,7 @@ placeholder="[]", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, initial=[ [ { @@ -500,9 +492,7 @@ help_text="List of custom data validators", placeholder="[]", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="FIELD_CHOICES", @@ -510,9 +500,7 @@ help_text="List of custom field choices", placeholder="[]", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), ], ), @@ -525,9 +513,7 @@ help_text="Default dashboard for users", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, initial=DEFAULT_DASHBOARD, ), Param( @@ -536,9 +522,7 @@ help_text="Default user preferences", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, initial={}, ), Param( @@ -654,9 +638,7 @@ help_text="List of NetBox administrators", placeholder="[('NetBox Admin', 'admin@example.com')]", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="BANNER_BOTTOM", @@ -664,9 +646,7 @@ help_text="Bottom banner text", placeholder="Banner text", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="BANNER_LOGIN", @@ -674,9 +654,7 @@ help_text="Login banner text", placeholder="Banner text", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="BANNER_TOP", @@ -684,9 +662,7 @@ help_text="Top banner text", placeholder="Banner text", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="BANNER_MAINTENANCE", @@ -694,9 +670,7 @@ help_text="Maintenance banner text", placeholder="Banner text", required=False, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="CENSUS_REPORTING_ENABLED", @@ -822,9 +796,7 @@ placeholder="{'queue_name': 'queue_name'}", required=False, field=forms.JSONField, - field_kwargs={ - "widget": forms.Textarea(attrs={"class": "vLargeTextField"}) - }, + field_kwargs={"widget": forms.Textarea(attrs={"class": "vLargeTextField"})}, ), Param( key="RELEASE_CHECK_URL", diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index 378ff98..d3d6176 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -98,8 +98,7 @@ def __init__(self, *args, **kwargs): f"{env_info['env']['displayName']} ({env_info['env']['envName']})", ) for env_info in env_infos - if env_info.get("env", {}).get("properties", {}).get("projectScope", "") - == "backup" + if env_info.get("env", {}).get("properties", {}).get("projectScope", "") == "backup" ] @@ -107,11 +106,7 @@ class NetBoxSettingsForm(BootstrapMixin, forms.Form): fieldsets = create_fieldset() class Meta: - fields = [ - param.key.lower() - for section in NETBOX_SETTINGS.sections - for param in section.params - ] + fields = [param.key.lower() for section in NETBOX_SETTINGS.sections for param in section.params] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -200,11 +195,7 @@ def __init__(self, *args, **kwargs): # Fetch the region list and build the choices nc = NetBoxConfiguration.objects.first() - regions = ( - nc.iaas(nc.env_name, auto_init=False) - .client.environment.Control.GetRegions() - .get("array", []) - ) + regions = nc.iaas(nc.env_name, auto_init=False).client.environment.Control.GetRegions().get("array", []) self.fields["region"].choices = [ (hard_node_group["uniqueName"], region["displayName"]) for region in regions @@ -216,17 +207,11 @@ def clean(self): if cleaned_data["deployment"] == "cluster": if cleaned_data["node_count"] == "1": - raise ValidationError( - { - "node_count": "Node count must be greater than 1 for a cluster deployment." - } - ) + raise ValidationError({"node_count": "Node count must be greater than 1 for a cluster deployment."}) if cleaned_data["deployment"] == "standalone": if cleaned_data["node_count"] != "1": - raise ValidationError( - {"node_count": "Node count must be 1 for a standalone deployment."} - ) + raise ValidationError({"node_count": "Node count must be 1 for a standalone deployment."}) return cleaned_data @@ -254,9 +239,7 @@ def __init__(self, *args, **kwargs): if self.instance.pk: # Fetch the database password from the addon settings - app = self.instance.netbox_env.get_env().get_installed_addon( - "db-backup", node_group=NODE_GROUP_SQLDB - ) + app = self.instance.netbox_env.get_env().get_installed_addon("db-backup", node_group=NODE_GROUP_SQLDB) data = app.get("settings", {}).get("main", {}).get("data", {}) self.fields["db_password"].initial = data.get("dbpass") @@ -267,9 +250,7 @@ def clean(self): self.instance._db_password = db_password if not self.instance.pk and not db_password: - raise ValidationError( - {"db_password": "This field is required when adding a new backup."} - ) + raise ValidationError({"db_password": "This field is required when adding a new backup."}) class NetBoxPluginInstallForm(BootstrapMixin, forms.Form): @@ -308,9 +289,7 @@ def __init__(self, *args, **kwargs): # Get the plugins.yaml plugins = get_plugins_list() if plugin := plugins.get(initial.get("name")): - self.fields["version"].choices = [ - (release, release) for release in filter_releases(plugin) - ] + self.fields["version"].choices = [(release, release) for release in filter_releases(plugin)] if initial.get("type") == "update": from importlib.metadata import metadata @@ -328,9 +307,7 @@ def clean(self): nc = NetBoxConfiguration.objects.first() if plugin.get("private"): if not nc.license: - raise ValidationError( - {"name": "This plugin requires a NetBox Enterprise license."} - ) + raise ValidationError({"name": "This plugin requires a NetBox Enterprise license."}) # Check if the plugin is accessible using the license response = requests.get( @@ -338,11 +315,7 @@ def clean(self): headers={"Authorization": f"Bearer {nc.license}"}, ) if not response.ok: - raise ValidationError( - { - "name": "This plugin is not accessible using the provided license." - } - ) + raise ValidationError({"name": "This plugin is not accessible using the provided license."}) # Get the required_settings from the plugin required_settings = next( @@ -357,11 +330,7 @@ def clean(self): # Check if the required_settings are in the configuration if not all(key in configuration.keys() for key in required_settings): - raise ValidationError( - { - "configuration": f"Missing required settings: {', '.join(required_settings)}" - } - ) + raise ValidationError({"configuration": f"Missing required settings: {', '.join(required_settings)}"}) class NetBoxUpgradeForm(BootstrapMixin, forms.Form): @@ -380,6 +349,4 @@ def __init__(self, *args, **kwargs): env = instance.get_env() versions = env.get_patch_upgrades() - self.fields["version"].choices = [ - (str(version), str(version)) for version in versions - ] + self.fields["version"].choices = [(str(version), str(version)) for version in versions] diff --git a/netbox_cloud_pilot/iaas.py b/netbox_cloud_pilot/iaas.py index fc8a412..dde9f71 100644 --- a/netbox_cloud_pilot/iaas.py +++ b/netbox_cloud_pilot/iaas.py @@ -116,9 +116,7 @@ def get_nodes(self, node_group: str = None, is_master: bool = True) -> dict | li """ Get the environment nodes. """ - logger.debug( - f"Getting nodes for node group {node_group}, is_master={is_master}" - ) + logger.debug(f"Getting nodes for node group {node_group}, is_master={is_master}") nodes = self._get_env_info().get("nodes", []) if node_group: nodes = [node for node in nodes if node["nodeGroup"] == node_group] @@ -137,9 +135,7 @@ def get_node_groups(self): # For each node group, get the related nodes for node_group in node_groups: - node_group["node"] = self.get_nodes( - node_group=node_group["name"], is_master=True - ) + node_group["node"] = self.get_nodes(node_group=node_group["name"], is_master=True) return node_groups @@ -162,9 +158,7 @@ def is_ssl_enabled(self): """ Check if SSL is enabled. """ - logger.debug( - f"Checking if built-in SSL is enabled for environment {self.env_name}" - ) + logger.debug(f"Checking if built-in SSL is enabled for environment {self.env_name}") return self.get_env().get("sslstate", False) def get_node_log(self, node_id, path="/var/log/run.log"): @@ -172,9 +166,9 @@ def get_node_log(self, node_id, path="/var/log/run.log"): Get the logs of a node. """ logger.debug(f"Getting log for node {node_id} for environment {self.env_name}") - return self.client.environment.Control.ReadLog( - env_name=self.env_name, node_id=node_id, path=path - ).get("body", "") + return self.client.environment.Control.ReadLog(env_name=self.env_name, node_id=node_id, path=path).get( + "body", "" + ) def get_url(self): """ @@ -201,17 +195,11 @@ def get_installed_addon(self, app_id, node_group, search=None): """ Get the installed addon for a node group. """ - logger.debug( - f"Checking if addon ({app_id}) is installed for node group {node_group}" - ) + logger.debug(f"Checking if addon ({app_id}) is installed for node group {node_group}") addons = self.get_addons(node_group=node_group, search=search) return next( - ( - addon - for addon in addons - if addon["app_id"] == app_id and addon.get("isInstalled", False) - ), + (addon for addon in addons if addon["app_id"] == app_id and addon.get("isInstalled", False)), None, ) @@ -257,16 +245,12 @@ def run_script(self, name, code, description=None, params=None): for script in scripts: if script.get("name") == name: logger.debug(f"Deleting existing script {name}") - self.client.development.Scripting.DeleteScript( - app_id=app_id, name=name - ) + self.client.development.Scripting.DeleteScript(app_id=app_id, name=name) continue except JelasticApiError as e: logger.error(e) - self.client.development.Scripting.CreateScript( - app_id=app_id, name=name, type="js", code=code - ) + self.client.development.Scripting.CreateScript(app_id=app_id, name=name, type="js", code=code) return self.client.utils.Scheduler.CreateEnvTask( env_name=self.env_name, @@ -276,9 +260,7 @@ def run_script(self, name, code, description=None, params=None): params=params, ) - def restart_nodes( - self, node_groups: list[str], lazy: bool = False, delay: int = 10000 - ): + def restart_nodes(self, node_groups: list[str], lazy: bool = False, delay: int = 10000): """ Restart nodes for a list of node groups. """ @@ -367,9 +349,7 @@ def uninstall_addon(self, app_id, node_group, search=None): Uninstall an addon. """ # Check if the addon is already installed - if addon := self.get_installed_addon( - app_id=app_id, node_group=node_group, search=search - ): + if addon := self.get_installed_addon(app_id=app_id, node_group=node_group, search=search): logger.info(f"Uninstalling addon {app_id}") return self.client.marketplace.Installation.Uninstall( @@ -401,9 +381,7 @@ def get_nb_node_groups(self): # For each node group, get the related nodes for node_group in node_groups: - if "netbox" in node_group.get("node", {}).get("customitem", {}).get( - "dockerName" - ): + if "netbox" in node_group.get("node", {}).get("customitem", {}).get("dockerName"): results.append(node_group) return results @@ -413,9 +391,7 @@ def load_plugins(self): Loads the plugins from the plugins.yaml file. """ master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") - plugins_yaml = self.execute_cmd( - master_node_id, "cat /etc/netbox/config/plugins.yaml" - )[0].get("out", "") + plugins_yaml = self.execute_cmd(master_node_id, "cat /etc/netbox/config/plugins.yaml")[0].get("out", "") return yaml.safe_load(plugins_yaml) or {} def dump_plugins(self, plugins): @@ -433,9 +409,7 @@ def dump_plugins(self, plugins): is_append_mode=False, ) - def install_plugin( - self, plugin: dict, version, plugin_settings=None, github_token=None - ): + def install_plugin(self, plugin: dict, version, plugin_settings=None, github_token=None): master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") activate_env = "source /opt/netbox/venv/bin/activate" @@ -445,13 +419,9 @@ def install_plugin( self.execute_cmd(master_node_id, "apt-get install -y git") github_url = plugin.get("github_url") - github_url = github_url.replace( - "https://github.com", f"git+https://{github_token}@github.com" - ) + github_url = github_url.replace("https://github.com", f"git+https://{github_token}@github.com") - self.execute_cmd( - master_node_id, f"{activate_env} && pip install {github_url}@{version}" - ) + self.execute_cmd(master_node_id, f"{activate_env} && pip install {github_url}@{version}") else: self.execute_cmd( master_node_id, @@ -469,9 +439,7 @@ def install_plugin( ) return self.restart_nodes( - node_groups=[ - node_group["name"] for node_group in self.get_nb_node_groups() - ], + node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], lazy=True, ) @@ -484,9 +452,7 @@ def uninstall_plugin(self, plugin: dict): self.dump_plugins(plugins) return self.restart_nodes( - node_groups=[ - node_group["name"] for node_group in self.get_nb_node_groups() - ], + node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], lazy=True, ) @@ -495,10 +461,7 @@ def get_env_var(self, variable, default=None): Get the environment variable for NetBox. """ container_vars = self._get_env_var(NODE_GROUP_CP) - return ( - getattr(get_config(), variable, container_vars.get(variable, None)) - or default - ) + return getattr(get_config(), variable, container_vars.get(variable, None)) or default def _get_docker_tags(self): """ @@ -507,9 +470,7 @@ def _get_docker_tags(self): master_node = self.get_master_node(NODE_GROUP_CP) docker = master_node.get("customitem", {}) - response = requests.get( - f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000' - ) + response = requests.get(f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000') response.raise_for_status() response = response.json() @@ -544,10 +505,7 @@ def get_patch_upgrades(self): # Filter out only the patch upgrades patch_upgrades = [] for upgrade in all_upgrades: - if ( - upgrade.major == current_version.major - and upgrade.minor == current_version.minor - ): + if upgrade.major == current_version.major and upgrade.minor == current_version.minor: patch_upgrades.append(upgrade) return patch_upgrades @@ -562,10 +520,7 @@ def get_minor_upgrades(self): # Filter out only the minor upgrades minor_upgrades = [] for upgrade in all_upgrades: - if ( - upgrade.major == current_version.major - and upgrade.minor > current_version.minor - ): + if upgrade.major == current_version.major and upgrade.minor > current_version.minor: minor_upgrades.append(upgrade) return minor_upgrades @@ -593,9 +548,7 @@ def is_upgrade_available(self): def is_db_backup_running(self, app_unique_name): # Get current running actions - current_actions = self.client.environment.Tracking.GetCurrentActions().get( - "array", [] - ) + current_actions = self.client.environment.Tracking.GetCurrentActions().get("array", []) for action in current_actions: action_parameters = action.get("parameters", {}) @@ -623,9 +576,7 @@ def upgrade(self, version): """ version = f"v{version}" - if addon := self.get_installed_addon( - app_id="db-backup", node_group=NODE_GROUP_SQLDB - ): + if addon := self.get_installed_addon(app_id="db-backup", node_group=NODE_GROUP_SQLDB): self.db_backup(app_unique_name=addon.get("uniqueName")) # TODO: Run plugin compatibility checks diff --git a/netbox_cloud_pilot/models.py b/netbox_cloud_pilot/models.py index 2918e0f..631a13a 100644 --- a/netbox_cloud_pilot/models.py +++ b/netbox_cloud_pilot/models.py @@ -35,9 +35,7 @@ class NetBoxConfiguration(JobsMixin, PrimaryModel): NetBoxConfig is a model that represents the configuration of NetBox. """ - key = models.CharField( - max_length=255, unique=True, validators=[MinLengthValidator(40)] - ) + key = models.CharField(max_length=255, unique=True, validators=[MinLengthValidator(40)]) env_name = models.CharField( max_length=255, @@ -112,12 +110,7 @@ def get_env(self): return self.iaas(self.env_name) def get_docker_tag(self): - return ( - self.get_env() - .get_master_node(NODE_GROUP_CP) - .get("customitem", {}) - .get("dockerTag", "") - ) + return self.get_env().get_master_node(NODE_GROUP_CP).get("customitem", {}).get("dockerTag", "") def get_env_storage(self): return self.iaas(self.env_name_storage) @@ -151,9 +144,7 @@ def netbox_settings(self): initial = env.get_env_var(param.key, param.initial) # Alter list of strings to comma-separated string - if isinstance(initial, (tuple, list)) and all( - [isinstance(x, str) for x in initial] - ): + if isinstance(initial, (tuple, list)) and all([isinstance(x, str) for x in initial]): initial = ", ".join(initial) param.initial = initial @@ -168,11 +159,7 @@ def apply_settings(self, data: dict): """ logger.debug(f"Applying NetBox settings: {data}") - all_keys = [ - param.key - for section in NETBOX_SETTINGS.sections - for param in section.params - ] + all_keys = [param.key for section in NETBOX_SETTINGS.sections for param in section.params] env = self.get_env() # Get all NetBox node groups @@ -329,20 +316,14 @@ def clean(self): # Ensure netbox_env has a storage env connected before proceeding if not self.netbox_env.env_name_storage: raise ValidationError( - { - "netbox_env": "Add a backup storage environment to the NetBoxConfiguration instance." - } + {"netbox_env": "Add a backup storage environment to the NetBoxConfiguration instance."} ) if self.keep_backups > 30: - raise ValidationError( - {"keep_backups": "The maximum number of backups to keep is 30."} - ) + raise ValidationError({"keep_backups": "The maximum number of backups to keep is 30."}) if self.keep_backups < 1: - raise ValidationError( - {"keep_backups": "The minimum number of backups to keep is 1."} - ) + raise ValidationError({"keep_backups": "The minimum number of backups to keep is 1."}) try: croniter(self.crontab) @@ -367,11 +348,7 @@ def list_backups(self): node_id=master_node.get("id"), command="/root/getBackupsAllEnvs.sh" )[0] - if ( - backups := json.loads(result.get("out", "")) - .get("backups", {}) - .get(self.netbox_env.env_name, []) - ): + if backups := json.loads(result.get("out", "")).get("backups", {}).get(self.netbox_env.env_name, []): # Cache backups in case next time it returns empty cache.set(f"netbox_db_backups_{self.pk}", backups, timeout=60 * 60) else: @@ -420,9 +397,7 @@ def backup(self, request=None): Create a new database backup. """ # Get installed addon - addon = self.netbox_env.get_env().get_installed_addon( - app_id="db-backup", node_group=NODE_GROUP_SQLDB - ) + addon = self.netbox_env.get_env().get_installed_addon(app_id="db-backup", node_group=NODE_GROUP_SQLDB) return self.netbox_env.enqueue( self.netbox_env.get_env().db_backup, @@ -435,9 +410,7 @@ def restore(self, request, backup_name): Restore a database backup. """ # Get installed addon - addon = self.netbox_env.get_env().get_installed_addon( - app_id="db-backup", node_group=NODE_GROUP_SQLDB - ) + addon = self.netbox_env.get_env().get_installed_addon(app_id="db-backup", node_group=NODE_GROUP_SQLDB) return self.netbox_env.enqueue( self.netbox_env.get_env().execute_action, diff --git a/netbox_cloud_pilot/nb_settings.py b/netbox_cloud_pilot/nb_settings.py index 0edfcf1..388ee73 100644 --- a/netbox_cloud_pilot/nb_settings.py +++ b/netbox_cloud_pilot/nb_settings.py @@ -63,9 +63,7 @@ def __init__( if widget and hasattr(widget, "attrs"): # Update the class attribute, retaining existing classes existing_classes = widget.attrs.get("class", "") - widget.attrs["class"] = " ".join( - filter(None, [existing_classes, "form-control"]) - ) + widget.attrs["class"] = " ".join(filter(None, [existing_classes, "form-control"])) else: # If widget is not provided or doesn't have attrs, use the default field_kwargs.update(default_kwargs) diff --git a/netbox_cloud_pilot/utils.py b/netbox_cloud_pilot/utils.py index ec5aab9..56db9d7 100644 --- a/netbox_cloud_pilot/utils.py +++ b/netbox_cloud_pilot/utils.py @@ -54,6 +54,4 @@ def filter_releases(plugin): def job_msg(job): - return mark_safe( - f"Job {job} has been created successfully." - ) + return mark_safe(f"Job {job} has been created successfully.") diff --git a/netbox_cloud_pilot/views.py b/netbox_cloud_pilot/views.py index 572688c..73082b7 100644 --- a/netbox_cloud_pilot/views.py +++ b/netbox_cloud_pilot/views.py @@ -132,14 +132,10 @@ def post(self, request, pk): node_groups=[node_group], ) messages.success(request, utils.job_msg(job)) - return redirect( - "plugins:netbox_cloud_pilot:netboxconfiguration", pk=instance.pk - ) + return redirect("plugins:netbox_cloud_pilot:netboxconfiguration", pk=instance.pk) -@register_model_view( - models.NetBoxConfiguration, "backup_storage", path="backup-storage" -) +@register_model_view(models.NetBoxConfiguration, "backup_storage", path="backup-storage") class NetBoxStorageView(PermissionRequiredMixin, GetReturnURLMixin, View): def get_permission_required(self): return ["netbox_cloud_pilot.view_netboxconfiguration"] @@ -272,9 +268,7 @@ class NetBoxPluginListView(View): def get(self, request): if nc := models.NetBoxConfiguration.objects.first(): installed_plugins = settings.PLUGINS - installed_plugins = [ - metadata(plugin).get("Name") for plugin in installed_plugins - ] + installed_plugins = [metadata(plugin).get("Name") for plugin in installed_plugins] plugins = utils.get_plugins_list() for plugin_name, _ in plugins.items(): @@ -289,14 +283,10 @@ def get(self, request): # Divide the plugins into two lists: installed and not installed plugins = { "installed": { - plugin_name: plugin - for plugin_name, plugin in plugins.items() - if plugin.get("installed") + plugin_name: plugin for plugin_name, plugin in plugins.items() if plugin.get("installed") }, "not_installed": { - plugin_name: plugin - for plugin_name, plugin in plugins.items() - if not plugin.get("installed") + plugin_name: plugin for plugin_name, plugin in plugins.items() if not plugin.get("installed") }, } @@ -367,9 +357,7 @@ def post(self, request, pk, *args, **kwargs): ) -@register_model_view( - models.NetBoxConfiguration, "plugin_install", path="plugin-install" -) +@register_model_view(models.NetBoxConfiguration, "plugin_install", path="plugin-install") class NetBoxPluginInstallView(generic.ObjectEditView): queryset = models.NetBoxConfiguration.objects.all() form = forms.NetBoxPluginInstallForm @@ -430,9 +418,7 @@ def post(self, request, *args, **kwargs): ) -@register_model_view( - models.NetBoxConfiguration, "plugin_uninstall", path="plugin-uninstall" -) +@register_model_view(models.NetBoxConfiguration, "plugin_uninstall", path="plugin-uninstall") class NetBoxPluginUninstallView(generic.ObjectDeleteView): queryset = models.NetBoxConfiguration.objects.all() template_name = "netbox_cloud_pilot/plugin_uninstall.html" diff --git a/pyproject.toml b/pyproject.toml index 1192d59..653edae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,4 +20,3 @@ exclude = [ ] reportMissingImports = true reportMissingTypeStubs = false - diff --git a/setup.py b/setup.py index 165d126..6e9ceb7 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ from setuptools import find_packages, setup -description = ( - "Enhances NetBox on CloudMyDC's VAP with advanced management and control features." -) +description = "Enhances NetBox on CloudMyDC's VAP with advanced management and control features." with open("README.md") as fh: long_description = fh.read() From f640828e6adae5cf2182bea2c18df81be4357d16 Mon Sep 17 00:00:00 2001 From: yash-pal1 Date: Thu, 4 Jan 2024 16:54:22 +0530 Subject: [PATCH 11/17] Fix plugin config and setup.py file --- netbox_cloud_pilot/__init__.py | 4 ++-- setup.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/netbox_cloud_pilot/__init__.py b/netbox_cloud_pilot/__init__.py index d6d2669..2f203dc 100644 --- a/netbox_cloud_pilot/__init__.py +++ b/netbox_cloud_pilot/__init__.py @@ -7,8 +7,8 @@ class NetBoxCloudPilot(PluginConfig): name = metadata.get("Name").replace("-", "_") - verbose_name = metadata.get("Summary") - description = metadata.get("Long-Description") + verbose_name = metadata.get("Name") + description = metadata.get("Summary") version = metadata.get("Version") author = metadata.get("Author") author_email = metadata.get("Author-email") diff --git a/setup.py b/setup.py index 6e9ceb7..4bcdb4a 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,16 @@ +import os + from setuptools import find_packages, setup -description = "Enhances NetBox on CloudMyDC's VAP with advanced management and control features." +readme = os.path.join(os.path.dirname(__file__), 'README.md') -with open("README.md") as fh: +with open(readme) as fh: long_description = fh.read() setup( name="netbox-cloud-pilot", version="0.0.1", - description=description, + description="Enhances NetBox on CloudMyDC's VAP with advanced management and control features.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/Onemind-Services-LLC/netbox-cloud-pilot/", From 489131a8a3fc3f85209c40df089b4215616525f1 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 17:41:11 +0530 Subject: [PATCH 12/17] added plugin upgrade checks --- netbox_cloud_pilot/forms.py | 17 ++++- netbox_cloud_pilot/iaas.py | 138 +++++++++++++++++++----------------- netbox_cloud_pilot/utils.py | 5 +- netbox_cloud_pilot/views.py | 2 +- 4 files changed, 92 insertions(+), 70 deletions(-) diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index d3d6176..a7d5fae 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -348,5 +348,18 @@ def __init__(self, *args, **kwargs): instance = NetBoxConfiguration.objects.first() env = instance.get_env() - versions = env.get_patch_upgrades() - self.fields["version"].choices = [(str(version), str(version)) for version in versions] + versions = env.get_upgrades() + self.fields["version"].choices = [ + (str(version), str(version)) for version in versions + ] + + def clean(self): + super().clean() + + instance = NetBoxConfiguration.objects.first() + env = instance.get_env() + + # Run upgrade checks + upgrade_check, error = env.upgrade_checks(self.cleaned_data.get("version")) + if not upgrade_check: + raise ValidationError({"version": error}) diff --git a/netbox_cloud_pilot/iaas.py b/netbox_cloud_pilot/iaas.py index dde9f71..dcdcfbe 100644 --- a/netbox_cloud_pilot/iaas.py +++ b/netbox_cloud_pilot/iaas.py @@ -1,6 +1,7 @@ import logging import time from functools import lru_cache +from typing import Tuple import requests import yaml @@ -12,6 +13,7 @@ from core.choices import JobStatusChoices from core.models import Job from netbox.config import get_config +from . import utils from .constants import JELASTIC_API, NODE_GROUP_CP, NODE_GROUP_SQLDB logger = logging.getLogger("netbox_cloud_pilot") @@ -195,11 +197,17 @@ def get_installed_addon(self, app_id, node_group, search=None): """ Get the installed addon for a node group. """ - logger.debug(f"Checking if addon ({app_id}) is installed for node group {node_group}") + logger.debug( + f"Checking if addon ({app_id}) is installed for node group {node_group}" + ) addons = self.get_addons(node_group=node_group, search=search) return next( - (addon for addon in addons if addon["app_id"] == app_id and addon.get("isInstalled", False)), + ( + addon + for addon in addons + if addon["app_id"] == app_id and addon.get("isInstalled", False) + ), None, ) @@ -245,12 +253,16 @@ def run_script(self, name, code, description=None, params=None): for script in scripts: if script.get("name") == name: logger.debug(f"Deleting existing script {name}") - self.client.development.Scripting.DeleteScript(app_id=app_id, name=name) + self.client.development.Scripting.DeleteScript( + app_id=app_id, name=name + ) continue except JelasticApiError as e: logger.error(e) - self.client.development.Scripting.CreateScript(app_id=app_id, name=name, type="js", code=code) + self.client.development.Scripting.CreateScript( + app_id=app_id, name=name, type="js", code=code + ) return self.client.utils.Scheduler.CreateEnvTask( env_name=self.env_name, @@ -260,7 +272,9 @@ def run_script(self, name, code, description=None, params=None): params=params, ) - def restart_nodes(self, node_groups: list[str], lazy: bool = False, delay: int = 10000): + def restart_nodes( + self, node_groups: list[str], lazy: bool = False, delay: int = 10000 + ): """ Restart nodes for a list of node groups. """ @@ -349,7 +363,9 @@ def uninstall_addon(self, app_id, node_group, search=None): Uninstall an addon. """ # Check if the addon is already installed - if addon := self.get_installed_addon(app_id=app_id, node_group=node_group, search=search): + if addon := self.get_installed_addon( + app_id=app_id, node_group=node_group, search=search + ): logger.info(f"Uninstalling addon {app_id}") return self.client.marketplace.Installation.Uninstall( @@ -381,7 +397,9 @@ def get_nb_node_groups(self): # For each node group, get the related nodes for node_group in node_groups: - if "netbox" in node_group.get("node", {}).get("customitem", {}).get("dockerName"): + if "netbox" in node_group.get("node", {}).get("customitem", {}).get( + "dockerName" + ): results.append(node_group) return results @@ -391,7 +409,9 @@ def load_plugins(self): Loads the plugins from the plugins.yaml file. """ master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") - plugins_yaml = self.execute_cmd(master_node_id, "cat /etc/netbox/config/plugins.yaml")[0].get("out", "") + plugins_yaml = self.execute_cmd( + master_node_id, "cat /etc/netbox/config/plugins.yaml" + )[0].get("out", "") return yaml.safe_load(plugins_yaml) or {} def dump_plugins(self, plugins): @@ -409,7 +429,9 @@ def dump_plugins(self, plugins): is_append_mode=False, ) - def install_plugin(self, plugin: dict, version, plugin_settings=None, github_token=None): + def install_plugin( + self, plugin: dict, version, plugin_settings=None, github_token=None + ): master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") activate_env = "source /opt/netbox/venv/bin/activate" @@ -419,9 +441,13 @@ def install_plugin(self, plugin: dict, version, plugin_settings=None, github_tok self.execute_cmd(master_node_id, "apt-get install -y git") github_url = plugin.get("github_url") - github_url = github_url.replace("https://github.com", f"git+https://{github_token}@github.com") + github_url = github_url.replace( + "https://github.com", f"git+https://{github_token}@github.com" + ) - self.execute_cmd(master_node_id, f"{activate_env} && pip install {github_url}@{version}") + self.execute_cmd( + master_node_id, f"{activate_env} && pip install {github_url}@{version}" + ) else: self.execute_cmd( master_node_id, @@ -439,7 +465,9 @@ def install_plugin(self, plugin: dict, version, plugin_settings=None, github_tok ) return self.restart_nodes( - node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], + node_groups=[ + node_group["name"] for node_group in self.get_nb_node_groups() + ], lazy=True, ) @@ -452,7 +480,9 @@ def uninstall_plugin(self, plugin: dict): self.dump_plugins(plugins) return self.restart_nodes( - node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], + node_groups=[ + node_group["name"] for node_group in self.get_nb_node_groups() + ], lazy=True, ) @@ -461,7 +491,10 @@ def get_env_var(self, variable, default=None): Get the environment variable for NetBox. """ container_vars = self._get_env_var(NODE_GROUP_CP) - return getattr(get_config(), variable, container_vars.get(variable, None)) or default + return ( + getattr(get_config(), variable, container_vars.get(variable, None)) + or default + ) def _get_docker_tags(self): """ @@ -470,7 +503,9 @@ def _get_docker_tags(self): master_node = self.get_master_node(NODE_GROUP_CP) docker = master_node.get("customitem", {}) - response = requests.get(f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000') + response = requests.get( + f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000' + ) response.raise_for_status() response = response.json() @@ -495,60 +530,17 @@ def _get_upgrades(self): return [tag for tag in docker_tags if tag > current_version] - def get_patch_upgrades(self): - """ - Get the available patch upgrades for NetBox. - """ - all_upgrades = self._get_upgrades() - current_version = Version.parse(settings.VERSION) - - # Filter out only the patch upgrades - patch_upgrades = [] - for upgrade in all_upgrades: - if upgrade.major == current_version.major and upgrade.minor == current_version.minor: - patch_upgrades.append(upgrade) - - return patch_upgrades - - def get_minor_upgrades(self): - """ - Get the available minor upgrades for NetBox. - """ - all_upgrades = self._get_upgrades() - current_version = Version.parse(settings.VERSION) - - # Filter out only the minor upgrades - minor_upgrades = [] - for upgrade in all_upgrades: - if upgrade.major == current_version.major and upgrade.minor > current_version.minor: - minor_upgrades.append(upgrade) - - return minor_upgrades - - def get_major_upgrades(self): - """ - Get the available major upgrades for NetBox. - """ - all_upgrades = self._get_upgrades() - current_version = Version.parse(settings.VERSION) - - # Filter out only the major upgrades - major_upgrades = [] - for upgrade in all_upgrades: - if upgrade.major > current_version.major: - major_upgrades.append(upgrade) - - return major_upgrades - def is_upgrade_available(self): """ Check if an upgrade is available for NetBox. """ - return bool(self._get_upgrades()) + return bool(self.get_upgrades()) def is_db_backup_running(self, app_unique_name): # Get current running actions - current_actions = self.client.environment.Tracking.GetCurrentActions().get("array", []) + current_actions = self.client.environment.Tracking.GetCurrentActions().get( + "array", [] + ) for action in current_actions: action_parameters = action.get("parameters", {}) @@ -570,17 +562,33 @@ def db_backup(self, app_unique_name): return self.execute_action(app_unique_name=app_unique_name, action="backup") + def upgrade_checks(self, version) -> Tuple[bool, str]: + """ + Run upgrade checks for NetBox. + """ + # Fetch the plugins from the store + plugins = utils.get_plugins_list() + for plugin_name, plugin in plugins.items(): + if plugin_name in settings.PLUGINS: + if not utils.filter_releases(plugin, version): + return ( + False, + f"Plugin {plugin_name} does not have a release for version {version}", + ) + + return True, "" + def upgrade(self, version): """ Upgrade NetBox. """ version = f"v{version}" - if addon := self.get_installed_addon(app_id="db-backup", node_group=NODE_GROUP_SQLDB): + if addon := self.get_installed_addon( + app_id="db-backup", node_group=NODE_GROUP_SQLDB + ): self.db_backup(app_unique_name=addon.get("uniqueName")) - # TODO: Run plugin compatibility checks - # Fetch all node groups for node_group in self.get_nb_node_groups(): node_group_name = node_group["name"] diff --git a/netbox_cloud_pilot/utils.py b/netbox_cloud_pilot/utils.py index 56db9d7..7131b25 100644 --- a/netbox_cloud_pilot/utils.py +++ b/netbox_cloud_pilot/utils.py @@ -33,18 +33,19 @@ def is_compatible(netbox_version, min_version, max_version): return True -def filter_releases(plugin): +def filter_releases(plugin, version: str = None): """ Filter the releases based on the NetBox version. """ compatible_releases = [] + version = version or settings.VERSION for release in plugin.get("releases", []): netbox = release.get("netbox") min_version = netbox.get("min") max_version = netbox.get("max") - if is_compatible(settings.VERSION, min_version, max_version): + if is_compatible(version, min_version, max_version): compatible_releases.append(release["tag"]) try: diff --git a/netbox_cloud_pilot/views.py b/netbox_cloud_pilot/views.py index 73082b7..dcb6f58 100644 --- a/netbox_cloud_pilot/views.py +++ b/netbox_cloud_pilot/views.py @@ -352,7 +352,7 @@ def post(self, request, pk, *args, **kwargs): "generic/object_edit.html", { "object": obj, - "form": self.form(instance=obj), + "form": form, }, ) From d639713c3b9e07ca167a6981c89a6a29921a81c5 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 17:45:28 +0530 Subject: [PATCH 13/17] lint fixes --- netbox_cloud_pilot/forms.py | 4 +-- netbox_cloud_pilot/iaas.py | 71 +++++++++---------------------------- 2 files changed, 18 insertions(+), 57 deletions(-) diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index a7d5fae..ab7a37b 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -349,9 +349,7 @@ def __init__(self, *args, **kwargs): env = instance.get_env() versions = env.get_upgrades() - self.fields["version"].choices = [ - (str(version), str(version)) for version in versions - ] + self.fields["version"].choices = [(str(version), str(version)) for version in versions] def clean(self): super().clean() diff --git a/netbox_cloud_pilot/iaas.py b/netbox_cloud_pilot/iaas.py index dcdcfbe..8e6d5b2 100644 --- a/netbox_cloud_pilot/iaas.py +++ b/netbox_cloud_pilot/iaas.py @@ -197,17 +197,11 @@ def get_installed_addon(self, app_id, node_group, search=None): """ Get the installed addon for a node group. """ - logger.debug( - f"Checking if addon ({app_id}) is installed for node group {node_group}" - ) + logger.debug(f"Checking if addon ({app_id}) is installed for node group {node_group}") addons = self.get_addons(node_group=node_group, search=search) return next( - ( - addon - for addon in addons - if addon["app_id"] == app_id and addon.get("isInstalled", False) - ), + (addon for addon in addons if addon["app_id"] == app_id and addon.get("isInstalled", False)), None, ) @@ -253,16 +247,12 @@ def run_script(self, name, code, description=None, params=None): for script in scripts: if script.get("name") == name: logger.debug(f"Deleting existing script {name}") - self.client.development.Scripting.DeleteScript( - app_id=app_id, name=name - ) + self.client.development.Scripting.DeleteScript(app_id=app_id, name=name) continue except JelasticApiError as e: logger.error(e) - self.client.development.Scripting.CreateScript( - app_id=app_id, name=name, type="js", code=code - ) + self.client.development.Scripting.CreateScript(app_id=app_id, name=name, type="js", code=code) return self.client.utils.Scheduler.CreateEnvTask( env_name=self.env_name, @@ -272,9 +262,7 @@ def run_script(self, name, code, description=None, params=None): params=params, ) - def restart_nodes( - self, node_groups: list[str], lazy: bool = False, delay: int = 10000 - ): + def restart_nodes(self, node_groups: list[str], lazy: bool = False, delay: int = 10000): """ Restart nodes for a list of node groups. """ @@ -363,9 +351,7 @@ def uninstall_addon(self, app_id, node_group, search=None): Uninstall an addon. """ # Check if the addon is already installed - if addon := self.get_installed_addon( - app_id=app_id, node_group=node_group, search=search - ): + if addon := self.get_installed_addon(app_id=app_id, node_group=node_group, search=search): logger.info(f"Uninstalling addon {app_id}") return self.client.marketplace.Installation.Uninstall( @@ -397,9 +383,7 @@ def get_nb_node_groups(self): # For each node group, get the related nodes for node_group in node_groups: - if "netbox" in node_group.get("node", {}).get("customitem", {}).get( - "dockerName" - ): + if "netbox" in node_group.get("node", {}).get("customitem", {}).get("dockerName"): results.append(node_group) return results @@ -409,9 +393,7 @@ def load_plugins(self): Loads the plugins from the plugins.yaml file. """ master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") - plugins_yaml = self.execute_cmd( - master_node_id, "cat /etc/netbox/config/plugins.yaml" - )[0].get("out", "") + plugins_yaml = self.execute_cmd(master_node_id, "cat /etc/netbox/config/plugins.yaml")[0].get("out", "") return yaml.safe_load(plugins_yaml) or {} def dump_plugins(self, plugins): @@ -429,9 +411,7 @@ def dump_plugins(self, plugins): is_append_mode=False, ) - def install_plugin( - self, plugin: dict, version, plugin_settings=None, github_token=None - ): + def install_plugin(self, plugin: dict, version, plugin_settings=None, github_token=None): master_node_id = self.get_master_node(NODE_GROUP_CP).get("id") activate_env = "source /opt/netbox/venv/bin/activate" @@ -441,13 +421,9 @@ def install_plugin( self.execute_cmd(master_node_id, "apt-get install -y git") github_url = plugin.get("github_url") - github_url = github_url.replace( - "https://github.com", f"git+https://{github_token}@github.com" - ) + github_url = github_url.replace("https://github.com", f"git+https://{github_token}@github.com") - self.execute_cmd( - master_node_id, f"{activate_env} && pip install {github_url}@{version}" - ) + self.execute_cmd(master_node_id, f"{activate_env} && pip install {github_url}@{version}") else: self.execute_cmd( master_node_id, @@ -465,9 +441,7 @@ def install_plugin( ) return self.restart_nodes( - node_groups=[ - node_group["name"] for node_group in self.get_nb_node_groups() - ], + node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], lazy=True, ) @@ -480,9 +454,7 @@ def uninstall_plugin(self, plugin: dict): self.dump_plugins(plugins) return self.restart_nodes( - node_groups=[ - node_group["name"] for node_group in self.get_nb_node_groups() - ], + node_groups=[node_group["name"] for node_group in self.get_nb_node_groups()], lazy=True, ) @@ -491,10 +463,7 @@ def get_env_var(self, variable, default=None): Get the environment variable for NetBox. """ container_vars = self._get_env_var(NODE_GROUP_CP) - return ( - getattr(get_config(), variable, container_vars.get(variable, None)) - or default - ) + return getattr(get_config(), variable, container_vars.get(variable, None)) or default def _get_docker_tags(self): """ @@ -503,9 +472,7 @@ def _get_docker_tags(self): master_node = self.get_master_node(NODE_GROUP_CP) docker = master_node.get("customitem", {}) - response = requests.get( - f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000' - ) + response = requests.get(f'https://hub.docker.com/v2/repositories/{docker["dockerName"]}/tags?page_size=1000') response.raise_for_status() response = response.json() @@ -538,9 +505,7 @@ def is_upgrade_available(self): def is_db_backup_running(self, app_unique_name): # Get current running actions - current_actions = self.client.environment.Tracking.GetCurrentActions().get( - "array", [] - ) + current_actions = self.client.environment.Tracking.GetCurrentActions().get("array", []) for action in current_actions: action_parameters = action.get("parameters", {}) @@ -584,9 +549,7 @@ def upgrade(self, version): """ version = f"v{version}" - if addon := self.get_installed_addon( - app_id="db-backup", node_group=NODE_GROUP_SQLDB - ): + if addon := self.get_installed_addon(app_id="db-backup", node_group=NODE_GROUP_SQLDB): self.db_backup(app_unique_name=addon.get("uniqueName")) # Fetch all node groups From 31b7c21ddd467cc8ac5ec834567bcfaaa898a0ad Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 22:00:22 +0530 Subject: [PATCH 14/17] fixed uninstall plugin view --- netbox_cloud_pilot/forms.py | 11 ++++++++++- netbox_cloud_pilot/views.py | 9 +++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index ab7a37b..280a8bc 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -4,7 +4,7 @@ from django.forms import ValidationError from netbox.forms import NetBoxModelForm -from utilities.forms import BootstrapMixin +from utilities.forms import BootstrapMixin, ConfirmationForm as _ConfirmationForm from utilities.forms.fields import CommentField from .constants import NETBOX_SETTINGS, NODE_GROUP_SQLDB from .models import * @@ -361,3 +361,12 @@ def clean(self): upgrade_check, error = env.upgrade_checks(self.cleaned_data.get("version")) if not upgrade_check: raise ValidationError({"version": error}) + + +class ConfirmationForm(_ConfirmationForm): + """ + A generic confirmation form. The form is not valid unless the `confirm` field is checked. + """ + name = forms.CharField( + widget=forms.HiddenInput() + ) diff --git a/netbox_cloud_pilot/views.py b/netbox_cloud_pilot/views.py index dcb6f58..2c3e0ba 100644 --- a/netbox_cloud_pilot/views.py +++ b/netbox_cloud_pilot/views.py @@ -431,14 +431,13 @@ def get(self, request, *args, **kwargs): messages.error(request, "Plugin not found.") return redirect("plugins:netbox_cloud_pilot:netboxplugin_list") - form = ConfirmationForm(initial=request.GET) + form = forms.ConfirmationForm(initial=request.GET) return render( request, self.template_name, { "object": obj, - "plugin": plugin, "form": form, "return_url": self.get_return_url(request, obj), **self.get_extra_context(request, obj), @@ -447,11 +446,10 @@ def get(self, request, *args, **kwargs): def post(self, request, *args, **kwargs): obj = self.get_object(**kwargs) - form = ConfirmationForm(request.POST) - - plugin = utils.get_plugins_list().get(request.POST.get("name")) + form = forms.ConfirmationForm(request.POST) if form.is_valid(): + plugin = utils.get_plugins_list().get(form.cleaned_data["name"]) job = obj.enqueue(obj.get_env().uninstall_plugin, request, plugin=plugin) messages.success(request, utils.job_msg(job)) @@ -462,7 +460,6 @@ def post(self, request, *args, **kwargs): self.template_name, { "object": obj, - "plugin": plugin, "form": form, "return_url": self.get_return_url(request, obj), **self.get_extra_context(request, obj), From bfda91724b6c9265787141bfe271cf9f76ebaa08 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 23:19:21 +0530 Subject: [PATCH 15/17] Update setup.py Signed-off-by: Abhimanyu Saharan --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4bcdb4a..b3c4dce 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name="netbox-cloud-pilot", - version="0.0.1", + version="0.0.2", description="Enhances NetBox on CloudMyDC's VAP with advanced management and control features.", long_description=long_description, long_description_content_type="text/markdown", From dd6bd273f5404b64cc3181199493cba827d983fc Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 23:36:39 +0530 Subject: [PATCH 16/17] lint fixes --- netbox_cloud_pilot/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/netbox_cloud_pilot/forms.py b/netbox_cloud_pilot/forms.py index 280a8bc..2ee2ca2 100644 --- a/netbox_cloud_pilot/forms.py +++ b/netbox_cloud_pilot/forms.py @@ -367,6 +367,5 @@ class ConfirmationForm(_ConfirmationForm): """ A generic confirmation form. The form is not valid unless the `confirm` field is checked. """ - name = forms.CharField( - widget=forms.HiddenInput() - ) + + name = forms.CharField(widget=forms.HiddenInput()) From 2200ccc3b74a5b39d2f49dfec8eba6c129b44441 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Thu, 4 Jan 2024 23:37:33 +0530 Subject: [PATCH 17/17] Updated ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bef95d..5e18717 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: test: runs-on: ubuntu-latest - name: Runs plugin tests - NetBox v3.5 + name: Runs plugin tests steps: - name: Checkout