From b46d5d4df7b6911b56ac0a6d95ab86c5d12e45c0 Mon Sep 17 00:00:00 2001 From: Ardavan Behnia Date: Mon, 30 Sep 2024 19:19:10 -0700 Subject: [PATCH 1/5] feat: add cloud-init message --- example.conf | 7 +- landscape/client/manager/cloudinit.py | 66 +++++++++++ landscape/client/manager/config.py | 1 + .../client/manager/tests/test_cloud_init.py | 107 ++++++++++++++++++ landscape/client/manager/tests/test_config.py | 3 +- landscape/message_schemas/server_bound.py | 7 ++ 6 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 landscape/client/manager/cloudinit.py create mode 100644 landscape/client/manager/tests/test_cloud_init.py diff --git a/example.conf b/example.conf index 842809596..f26635249 100644 --- a/example.conf +++ b/example.conf @@ -93,9 +93,6 @@ ignore_sigusr1 = False # SwiftUsage - Swift cluster usage # CephUsage - Ceph usage information # ComputerTags - changes in computer tags -# UbuntuProInfo - Ubuntu Pro registration information -# LivePatch - Livepath status information -# UbuntuProRebootRequired - informs if the system needs to be rebooted # SnapServicesMonitor - manage snap services # # The special value "ALL" is an alias for the full list of plugins. @@ -176,10 +173,14 @@ cloud = True # UserManager # ShutdownManager # AptSources +# CloudInit - Relevant information for cloud init # HardwareInfo # KeystoneToken # SnapManager # SnapServicesManager +# UbuntuProInfo - Ubuntu Pro registration information +# LivePatch - Livepath status information +# UbuntuProRebootRequired - informs if the system needs to be rebooted # # The special value "ALL" is an alias for the entire list of plugins above and is the default. manager_plugins = ALL diff --git a/landscape/client/manager/cloudinit.py b/landscape/client/manager/cloudinit.py new file mode 100644 index 000000000..75711cc40 --- /dev/null +++ b/landscape/client/manager/cloudinit.py @@ -0,0 +1,66 @@ +import json +import subprocess +from typing import Any + +from landscape.client.manager.plugin import DataWatcherManager + + +class CloudInit(DataWatcherManager): + + message_type = "cloud-init" + message_key = message_type + scope = "cloud-init" + persist_name = message_type + run_immediately = True + run_interval = 3600 * 24 # 24h + + def get_data(self) -> str: + return json.dumps(get_cloud_init(), sort_keys=True) + + +def get_cloud_init() -> dict[str, Any]: + """ + cloud-init returns all the information the instance has been initialized + with, in JSON format. This function takes the the output and parses it + into a python dictionary and sticks it in "output" along with error and + return code information. + """ + + data: dict[str, Any] = {} + output: dict[str, Any] = {} + + try: + completed_process = subprocess.run( + ["cloud-init", "query", "-a"], + encoding="utf8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError as exc: + data["return_code"] = -1 + data["error"] = str(exc) + data["output"] = "" + except Exception as exc: + data["return_code"] = -2 + data["error"] = str(exc) + data["output"] = "" + else: + string_output = completed_process.stdout.strip() + try: + # INFO: We don't want to parse an empty string. + if string_output: + json_output = json.loads(string_output) + # INFO: Only return relevant information from cloud init. + output["availability_zone"] = json_output.get( + "availability_zone", + "", + ) or json_output.get("availability-zone", "") + data["return_code"] = completed_process.returncode + data["error"] = completed_process.stderr + data["output"] = output + except json.decoder.JSONDecodeError as exc: + data["return_code"] = completed_process.returncode + data["error"] = str(exc) + data["output"] = output + + return data diff --git a/landscape/client/manager/config.py b/landscape/client/manager/config.py index 6b23484b3..0b1fa0b08 100644 --- a/landscape/client/manager/config.py +++ b/landscape/client/manager/config.py @@ -17,6 +17,7 @@ "UbuntuProInfo", "LivePatch", "UbuntuProRebootRequired", + "CloudInit", ] diff --git a/landscape/client/manager/tests/test_cloud_init.py b/landscape/client/manager/tests/test_cloud_init.py new file mode 100644 index 000000000..9929b3851 --- /dev/null +++ b/landscape/client/manager/tests/test_cloud_init.py @@ -0,0 +1,107 @@ +import json +from unittest import mock + +from landscape.client.manager.cloudinit import CloudInit +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper + + +def subprocess_cloud_init_mock(*args, **kwargs): + """Mock a cloud-init subprocess output.""" + data = {"availability_zone": "us-east-1"} + output = json.dumps(data) + return mock.Mock(stdout=output, stderr="", returncode=0) + + +class CloudInitTest(LandscapeTest): + """Cloud init plugin tests.""" + + helpers = [ManagerHelper] + + def setUp(self): + super(CloudInitTest, self).setUp() + self.mstore = self.broker_service.message_store + self.mstore.set_accepted_types(["cloud-init"]) + + def test_cloud_init(self): + """Test calling cloud-init.""" + plugin = CloudInit() + + with mock.patch("subprocess.run") as run_mock: + run_mock.side_effect = subprocess_cloud_init_mock + self.manager.add(plugin) + plugin.run() + + messages = self.mstore.get_pending_messages() + self.assertTrue(len(messages) > 0) + message = json.loads(messages[0]["cloud-init"]) + self.assertEqual(message["output"]["availability_zone"], "us-east-1") + self.assertEqual(message["return_code"], 0) + self.assertFalse(message["error"]) + + def test_cloud_init_when_not_installed(self): + """Tests calling cloud-init when it is not installed.""" + plugin = CloudInit() + + with mock.patch("subprocess.run") as run_mock: + run_mock.side_effect = FileNotFoundError("Not found!") + self.manager.add(plugin) + plugin.run() + + messages = self.mstore.get_pending_messages() + message = json.loads(messages[0]["cloud-init"]) + self.assertTrue(len(messages) > 0) + self.assertTrue(message["error"]) + self.assertEqual(message["return_code"], -1) + + def test_undefined_exception(self): + """Test calling cloud-init when a random exception occurs.""" + plugin = CloudInit() + + with mock.patch("subprocess.run") as run_mock: + run_mock.side_effect = ValueError("Not found!") + self.manager.add(plugin) + plugin.run() + + messages = self.mstore.get_pending_messages() + message = json.loads(messages[0]["cloud-init"]) + self.assertTrue(len(messages) > 0) + self.assertTrue(message["error"]) + self.assertEqual(message["return_code"], -2) + + def test_json_parse_error(self): + """ + If a Json parsing error occurs, show the exception and unparsed data. + """ + plugin = CloudInit() + + with mock.patch("subprocess.run") as run_mock: + run_mock.return_value = mock.Mock(stdout="'") + run_mock.return_value.returncode = 0 + self.manager.add(plugin) + plugin.run() + + messages = self.mstore.get_pending_messages() + message = json.loads(messages[0]["cloud-init"]) + self.assertTrue(len(messages) > 0) + self.assertTrue(message["error"]) + self.assertEqual({}, message["output"]) + + def test_empty_string(self): + """ + If cloud-init is disabled, stdout is an empty string. + """ + plugin = CloudInit() + + with mock.patch("subprocess.run") as run_mock: + run_mock.return_value = mock.Mock(stdout="", stderr="Error") + run_mock.return_value.returncode = 1 + self.manager.add(plugin) + plugin.run() + + messages = self.mstore.get_pending_messages() + message = json.loads(messages[0]["cloud-init"]) + self.assertTrue(len(messages) > 0) + self.assertTrue(message["error"]) + self.assertEqual(message["return_code"], 1) + self.assertEqual({}, message["output"]) diff --git a/landscape/client/manager/tests/test_config.py b/landscape/client/manager/tests/test_config.py index bb322a578..68ab641f2 100644 --- a/landscape/client/manager/tests/test_config.py +++ b/landscape/client/manager/tests/test_config.py @@ -24,7 +24,8 @@ def test_plugin_factories(self): "SnapServicesManager", "UbuntuProInfo", "LivePatch", - "UbuntuProRebootRequired" + "UbuntuProRebootRequired", + "CloudInit", ], ALL_PLUGINS, ) diff --git a/landscape/message_schemas/server_bound.py b/landscape/message_schemas/server_bound.py index c810a7467..d82260ef4 100644 --- a/landscape/message_schemas/server_bound.py +++ b/landscape/message_schemas/server_bound.py @@ -17,6 +17,7 @@ "ACTIVE_PROCESS_INFO", "COMPUTER_UPTIME", "CLIENT_UPTIME", + "CLOUD_INIT", "OPERATION_RESULT", "COMPUTER_INFO", "DISTRIBUTION_INFO", @@ -768,6 +769,11 @@ {"ubuntu-pro-reboot-required": Unicode()}, ) +CLOUD_INIT = Message( + "cloud-init", + {"cloud-init": Unicode()}, +) + SNAPS = Message( "snaps", { @@ -849,6 +855,7 @@ ACTIVE_PROCESS_INFO, COMPUTER_UPTIME, CLIENT_UPTIME, + CLOUD_INIT, OPERATION_RESULT, COMPUTER_INFO, DISTRIBUTION_INFO, From 9791cfd3305fc5aa5864efc28636865156bdcf54 Mon Sep 17 00:00:00 2001 From: Ardavan Behnia Date: Fri, 4 Oct 2024 00:56:40 -0700 Subject: [PATCH 2/5] manager -> monitor --- example.conf | 2 +- landscape/client/manager/config.py | 1 - landscape/client/manager/tests/test_config.py | 1 - .../client/{manager => monitor}/cloudinit.py | 17 ++++++++++------- landscape/client/monitor/config.py | 1 + .../tests/test_cloud_init.py | 17 ++++++++--------- 6 files changed, 20 insertions(+), 19 deletions(-) rename landscape/client/{manager => monitor}/cloudinit.py (86%) rename landscape/client/{manager => monitor}/tests/test_cloud_init.py (89%) diff --git a/example.conf b/example.conf index f26635249..81a5d369e 100644 --- a/example.conf +++ b/example.conf @@ -94,6 +94,7 @@ ignore_sigusr1 = False # CephUsage - Ceph usage information # ComputerTags - changes in computer tags # SnapServicesMonitor - manage snap services +# CloudInit - Relevant information for cloud init # # The special value "ALL" is an alias for the full list of plugins. monitor_plugins = ALL @@ -173,7 +174,6 @@ cloud = True # UserManager # ShutdownManager # AptSources -# CloudInit - Relevant information for cloud init # HardwareInfo # KeystoneToken # SnapManager diff --git a/landscape/client/manager/config.py b/landscape/client/manager/config.py index 0b1fa0b08..6b23484b3 100644 --- a/landscape/client/manager/config.py +++ b/landscape/client/manager/config.py @@ -17,7 +17,6 @@ "UbuntuProInfo", "LivePatch", "UbuntuProRebootRequired", - "CloudInit", ] diff --git a/landscape/client/manager/tests/test_config.py b/landscape/client/manager/tests/test_config.py index 68ab641f2..1fdd57af4 100644 --- a/landscape/client/manager/tests/test_config.py +++ b/landscape/client/manager/tests/test_config.py @@ -25,7 +25,6 @@ def test_plugin_factories(self): "UbuntuProInfo", "LivePatch", "UbuntuProRebootRequired", - "CloudInit", ], ALL_PLUGINS, ) diff --git a/landscape/client/manager/cloudinit.py b/landscape/client/monitor/cloudinit.py similarity index 86% rename from landscape/client/manager/cloudinit.py rename to landscape/client/monitor/cloudinit.py index 75711cc40..488740bc1 100644 --- a/landscape/client/manager/cloudinit.py +++ b/landscape/client/monitor/cloudinit.py @@ -1,15 +1,14 @@ import json import subprocess -from typing import Any -from landscape.client.manager.plugin import DataWatcherManager +from landscape.client.monitor.plugin import DataWatcher -class CloudInit(DataWatcherManager): +class CloudInit(DataWatcher): message_type = "cloud-init" message_key = message_type - scope = "cloud-init" + scope = message_type persist_name = message_type run_immediately = True run_interval = 3600 * 24 # 24h @@ -17,8 +16,12 @@ class CloudInit(DataWatcherManager): def get_data(self) -> str: return json.dumps(get_cloud_init(), sort_keys=True) + def register(self, monitor): + super().register(monitor) + self.call_on_accepted("cloud-init", self.send_message) -def get_cloud_init() -> dict[str, Any]: + +def get_cloud_init(): """ cloud-init returns all the information the instance has been initialized with, in JSON format. This function takes the the output and parses it @@ -26,8 +29,8 @@ def get_cloud_init() -> dict[str, Any]: return code information. """ - data: dict[str, Any] = {} - output: dict[str, Any] = {} + data = {} + output = {} try: completed_process = subprocess.run( diff --git a/landscape/client/monitor/config.py b/landscape/client/monitor/config.py index 63d278ac7..bd353e7e2 100644 --- a/landscape/client/monitor/config.py +++ b/landscape/client/monitor/config.py @@ -21,6 +21,7 @@ "CephUsage", "ComputerTags", "SnapServicesMonitor", + "CloudInit", ] diff --git a/landscape/client/manager/tests/test_cloud_init.py b/landscape/client/monitor/tests/test_cloud_init.py similarity index 89% rename from landscape/client/manager/tests/test_cloud_init.py rename to landscape/client/monitor/tests/test_cloud_init.py index 9929b3851..95fa25cca 100644 --- a/landscape/client/manager/tests/test_cloud_init.py +++ b/landscape/client/monitor/tests/test_cloud_init.py @@ -1,9 +1,9 @@ import json from unittest import mock -from landscape.client.manager.cloudinit import CloudInit +from landscape.client.monitor.cloudinit import CloudInit from landscape.client.tests.helpers import LandscapeTest -from landscape.client.tests.helpers import ManagerHelper +from landscape.client.tests.helpers import MonitorHelper def subprocess_cloud_init_mock(*args, **kwargs): @@ -16,11 +16,10 @@ def subprocess_cloud_init_mock(*args, **kwargs): class CloudInitTest(LandscapeTest): """Cloud init plugin tests.""" - helpers = [ManagerHelper] + helpers = [MonitorHelper] def setUp(self): super(CloudInitTest, self).setUp() - self.mstore = self.broker_service.message_store self.mstore.set_accepted_types(["cloud-init"]) def test_cloud_init(self): @@ -29,7 +28,7 @@ def test_cloud_init(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = subprocess_cloud_init_mock - self.manager.add(plugin) + self.monitor.add(plugin) plugin.run() messages = self.mstore.get_pending_messages() @@ -45,7 +44,7 @@ def test_cloud_init_when_not_installed(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = FileNotFoundError("Not found!") - self.manager.add(plugin) + self.monitor.add(plugin) plugin.run() messages = self.mstore.get_pending_messages() @@ -60,7 +59,7 @@ def test_undefined_exception(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = ValueError("Not found!") - self.manager.add(plugin) + self.monitor.add(plugin) plugin.run() messages = self.mstore.get_pending_messages() @@ -78,7 +77,7 @@ def test_json_parse_error(self): with mock.patch("subprocess.run") as run_mock: run_mock.return_value = mock.Mock(stdout="'") run_mock.return_value.returncode = 0 - self.manager.add(plugin) + self.monitor.add(plugin) plugin.run() messages = self.mstore.get_pending_messages() @@ -96,7 +95,7 @@ def test_empty_string(self): with mock.patch("subprocess.run") as run_mock: run_mock.return_value = mock.Mock(stdout="", stderr="Error") run_mock.return_value.returncode = 1 - self.manager.add(plugin) + self.monitor.add(plugin) plugin.run() messages = self.mstore.get_pending_messages() From b604e6a0ce68b9df2cc2d6edd46675ffbbfabe0e Mon Sep 17 00:00:00 2001 From: Ardavan Behnia Date: Fri, 4 Oct 2024 00:59:12 -0700 Subject: [PATCH 3/5] add cloud-init to snap default monitors --- snap/hooks/default-configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/hooks/default-configure b/snap/hooks/default-configure index 304b559db..d5a6c60a3 100644 --- a/snap/hooks/default-configure +++ b/snap/hooks/default-configure @@ -29,7 +29,7 @@ if [ -z "$_manager_plugins" ]; then fi if [ -z "$_monitor_plugins" ]; then - _monitor_plugins="ActiveProcessInfo,ComputerInfo,LoadAverage,MemoryInfo,MountInfo,ProcessorInfo,Temperature,UserMonitor,RebootRequired,NetworkActivity,NetworkDevice,CPUUsage,SwiftUsage,CephUsage,ComputerTags,SnapServicesMonitor" + _monitor_plugins="ActiveProcessInfo,ComputerInfo,LoadAverage,MemoryInfo,MountInfo,ProcessorInfo,Temperature,UserMonitor,RebootRequired,NetworkActivity,NetworkDevice,CPUUsage,SwiftUsage,CephUsage,ComputerTags,SnapServicesMonitor,CloudInit" fi cat > "$CLIENT_CONF" << EOF From 27b57bd8528f1c91395c1d08cbe8a41fb752e1e5 Mon Sep 17 00:00:00 2001 From: Ardavan Behnia Date: Fri, 4 Oct 2024 01:05:17 -0700 Subject: [PATCH 4/5] fix tests --- landscape/client/monitor/tests/test_cloud_init.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/landscape/client/monitor/tests/test_cloud_init.py b/landscape/client/monitor/tests/test_cloud_init.py index 95fa25cca..cf87fad09 100644 --- a/landscape/client/monitor/tests/test_cloud_init.py +++ b/landscape/client/monitor/tests/test_cloud_init.py @@ -29,7 +29,7 @@ def test_cloud_init(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = subprocess_cloud_init_mock self.monitor.add(plugin) - plugin.run() + plugin.exchange() messages = self.mstore.get_pending_messages() self.assertTrue(len(messages) > 0) @@ -45,7 +45,7 @@ def test_cloud_init_when_not_installed(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = FileNotFoundError("Not found!") self.monitor.add(plugin) - plugin.run() + plugin.exchange() messages = self.mstore.get_pending_messages() message = json.loads(messages[0]["cloud-init"]) @@ -60,7 +60,7 @@ def test_undefined_exception(self): with mock.patch("subprocess.run") as run_mock: run_mock.side_effect = ValueError("Not found!") self.monitor.add(plugin) - plugin.run() + plugin.exchange() messages = self.mstore.get_pending_messages() message = json.loads(messages[0]["cloud-init"]) @@ -78,7 +78,7 @@ def test_json_parse_error(self): run_mock.return_value = mock.Mock(stdout="'") run_mock.return_value.returncode = 0 self.monitor.add(plugin) - plugin.run() + plugin.exchange() messages = self.mstore.get_pending_messages() message = json.loads(messages[0]["cloud-init"]) @@ -96,7 +96,7 @@ def test_empty_string(self): run_mock.return_value = mock.Mock(stdout="", stderr="Error") run_mock.return_value.returncode = 1 self.monitor.add(plugin) - plugin.run() + plugin.exchange() messages = self.mstore.get_pending_messages() message = json.loads(messages[0]["cloud-init"]) From 61c86eaca7bde8068fda22b47833ec8927a021eb Mon Sep 17 00:00:00 2001 From: Ardavan Behnia Date: Fri, 4 Oct 2024 02:55:29 -0700 Subject: [PATCH 5/5] improve error handling and fix bug (urgent) --- landscape/client/monitor/cloudinit.py | 6 +++--- landscape/client/monitor/tests/test_cloud_init.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/landscape/client/monitor/cloudinit.py b/landscape/client/monitor/cloudinit.py index 488740bc1..727892eb6 100644 --- a/landscape/client/monitor/cloudinit.py +++ b/landscape/client/monitor/cloudinit.py @@ -18,7 +18,7 @@ def get_data(self) -> str: def register(self, monitor): super().register(monitor) - self.call_on_accepted("cloud-init", self.send_message) + self.call_on_accepted("cloud-init", self.exchange, True) def get_cloud_init(): @@ -42,11 +42,11 @@ def get_cloud_init(): except FileNotFoundError as exc: data["return_code"] = -1 data["error"] = str(exc) - data["output"] = "" + data["output"] = output except Exception as exc: data["return_code"] = -2 data["error"] = str(exc) - data["output"] = "" + data["output"] = output else: string_output = completed_process.stdout.strip() try: diff --git a/landscape/client/monitor/tests/test_cloud_init.py b/landscape/client/monitor/tests/test_cloud_init.py index cf87fad09..2238c463c 100644 --- a/landscape/client/monitor/tests/test_cloud_init.py +++ b/landscape/client/monitor/tests/test_cloud_init.py @@ -52,6 +52,7 @@ def test_cloud_init_when_not_installed(self): self.assertTrue(len(messages) > 0) self.assertTrue(message["error"]) self.assertEqual(message["return_code"], -1) + self.assertEqual({}, message["output"]) def test_undefined_exception(self): """Test calling cloud-init when a random exception occurs.""" @@ -67,6 +68,7 @@ def test_undefined_exception(self): self.assertTrue(len(messages) > 0) self.assertTrue(message["error"]) self.assertEqual(message["return_code"], -2) + self.assertEqual({}, message["output"]) def test_json_parse_error(self): """