Add loop around dcnm_send to wait for successful response.

We've seen cases were a 500 response is returned if install-options is called too early.  Add a loop to try for self.timeout  seconds with 5 second poll time.  Update unit tests to bypass the sleep so that the tests run faster.

plugins/module_utils/image_mgmt/install_options.py

@@ -21,6 +21,7 @@
 import inspect
 import json
 import logging
+import time
 from typing import Any, Dict
 
 from ansible_collections.cisco.dcnm.plugins.module_utils.image_mgmt.api_endpoints import \
@@ -158,14 +159,16 @@ def __init__(self, module) -> None:
     def _init_properties(self):
         # self.properties is already initialized in the parent class
         self.properties["epld"] = False
+        self.properties["epld_modules"] = None
         self.properties["issu"] = True
-        self.properties["response_data"] = None
-        self.properties["response"] = None
-        self.properties["result"] = None
         self.properties["package_install"] = False
         self.properties["policy_name"] = None
+        self.properties["response"] = None
+        self.properties["response_data"] = None
+        self.properties["result"] = {}
         self.properties["serial_number"] = None
-        self.properties["epld_modules"] = None
+        self.properties["timeout"] = 300
+        self.properties["unit_test"] = False
 
     def refresh(self) -> None:
         """
@@ -185,6 +188,10 @@ def refresh(self) -> None:
             msg += "calling refresh()"
             self.module.fail_json(msg, **self.failed_result)
 
+        msg = f"self.epld {self.epld}, "
+        msg += f"self.issu {self.issu}, "
+        msg += f"self.package_install {self.package_install}"
+        self.log.debug(msg)
         # At least one of epld, issu, or package_install must be True
         # before calling refresh() or the controller will return an error.
         # Mock the response such that the caller knows nothing needs to be
@@ -204,11 +211,34 @@ def refresh(self) -> None:
 
         self._build_payload()
 
-        self.properties["response"] = dcnm_send(
-            self.module, self.verb, self.path, data=json.dumps(self.payload)
-        )
-        self.properties["response_data"] = self.response.get("DATA", {})
-        self.properties["result"] = self._handle_response(self.response, self.verb)
+        timeout = self.timeout
+        sleep_time = 5
+        self.result["success"] = False
+
+        msg = f"Entering dcnm_send loop. timeout {timeout}, sleep_time {sleep_time}"
+        self.log.debug(msg)
+
+        while timeout > 0 and self.result.get("success") is False:
+            msg = "Calling dcnm_send with payload: "
+            msg += f"{json.dumps(self.payload, indent=4, sort_keys=True)}"
+            self.log.debug(msg)
+
+            self.properties["response"] = dcnm_send(
+                self.module, self.verb, self.path, data=json.dumps(self.payload)
+            )
+
+            msg = "Calling dcnm_send DONE"
+            self.log.debug(msg)
+
+            self.properties["response_data"] = self.response.get("DATA", {})
+            self.properties["result"] = self._handle_response(self.response, self.verb)
+
+            msg = f"self.response {self.response}"
+            self.log.debug(msg)
+
+            if self.result.get("success") is False and self.unit_test is False:
+                time.sleep(sleep_time)
+            timeout -= sleep_time
 
         if self.result["success"] is False:
             msg = f"{self.class_name}.{method_name}: "
@@ -255,6 +285,9 @@ def _build_payload(self) -> None:
         self.payload["epld"] = self.epld
         self.payload["packageInstall"] = self.package_install
 
+        msg = f"self.payload {self.payload}"
+        self.log.debug(msg)
+
     def _get(self, item):
         return self.make_boolean(self.make_none(self.response_data.get(item)))
 
@@ -287,6 +320,7 @@ def serial_number(self, value):
         self.properties["serial_number"] = value
 
     # Optional properties
+
     @property
     def issu(self):
         """
@@ -352,6 +386,42 @@ def package_install(self, value):
             self.module.fail_json(msg, **self.failed_result)
         self.properties["package_install"] = value
 
+    @property
+    def timeout(self):
+        """
+        Timeout, in seconds, for retrieving install-options from the controller.
+        Valid values: int()
+        Default: 300
+        """
+        return self.properties.get("timeout")
+
+    @timeout.setter
+    def timeout(self, value):
+        method_name = inspect.stack()[0][3]
+        if not isinstance(value, int):
+            msg = f"{self.class_name}.{method_name}: "
+            msg += f"{method_name} must be an int(). Got {value}."
+            self.module.fail_json(msg, **self.failed_result)
+        self.properties["timeout"] = value
+
+    @property
+    def unit_test(self):
+        """
+        Is the class running under a unit test.
+        Set this to True in unit tests to speed the test up.
+        Default: False
+        """
+        return self.properties.get("unit_test")
+
+    @unit_test.setter
+    def unit_test(self, value):
+        method_name = inspect.stack()[0][3]
+        if not isinstance(value, bool):
+            msg = f"{self.class_name}.{method_name}: "
+            msg += f"{method_name} must be a bool(). Got {value}."
+            self.module.fail_json(msg, **self.failed_result)
+        self.properties["unit_test"] = value
+
     # Retrievable properties
     @property
     def comp_disp(self):

tests/unit/modules/dcnm/dcnm_image_upgrade/test_image_upgrade_image_install_options.py

@@ -82,8 +82,10 @@ def test_image_mgmt_install_options_00002(image_install_options) -> None:
     assert instance.properties.get("policy_name") is None
     assert instance.properties.get("response") is None
     assert instance.properties.get("response_data") is None
-    assert instance.properties.get("result") is None
+    assert instance.properties.get("result") == {}
     assert instance.properties.get("serial_number") is None
+    assert instance.properties.get("timeout") == 300
+    assert instance.properties.get("unit_test") is False
 
 
 def test_image_mgmt_install_options_00003(image_install_options) -> None:
@@ -142,6 +144,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     monkeypatch.setattr(DCNM_SEND_INSTALL_OPTIONS, mock_dcnm_send_install_options)
 
     instance = image_install_options
+    instance.unit_test = True
     instance.policy_name = "KRM5"
     instance.serial_number = "BAR"
     instance.refresh()
@@ -188,6 +191,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     match += "the controller. Controller response:"
 
     instance = image_install_options
+    instance.unit_test = True
     instance.policy_name = "KRM5"
     instance.serial_number = "BAR"
     with pytest.raises(AnsibleFailJson, match=rf"{match}"):
@@ -223,6 +227,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     instance = image_install_options
     instance.policy_name = "KRM5"
     instance.serial_number = "FDO21120U5D"
+    instance.unit_test = True
     instance.refresh()
     assert isinstance(instance.response, dict)
     assert instance.device_name == "leaf1"
@@ -279,6 +284,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     instance.epld = True
     instance.issu = True
     instance.package_install = False
+    instance.unit_test = True
     instance.refresh()
     assert isinstance(instance.response, dict)
     assert instance.device_name == "leaf1"
@@ -336,6 +342,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     instance.epld = True
     instance.issu = False
     instance.package_install = False
+    instance.unit_test = True
     instance.refresh()
     assert isinstance(instance.response, dict)
     assert instance.device_name is None
@@ -397,6 +404,7 @@ def mock_dcnm_send_install_options(*args, **kwargs) -> Dict[str, Any]:
     instance.epld = True
     instance.issu = True
     instance.package_install = True
+    instance.unit_test = True
     with pytest.raises(AnsibleFailJson, match=match):
         instance.refresh()
 
@@ -431,6 +439,7 @@ def test_image_mgmt_install_options_00011(image_install_options) -> None:
         instance.epld = False
         instance.issu = False
         instance.package_install = False
+        instance.unit_test = True
         instance.refresh()
 
     assert isinstance(instance.response_data, dict)
@@ -456,6 +465,7 @@ def test_image_mgmt_install_options_00020(image_install_options) -> None:
     instance = image_install_options
     instance.policy_name = "KRM5"
     instance.serial_number = "BAR"
+    instance.unit_test = True
     instance._build_payload()  # pylint: disable=protected-access
     assert instance.payload.get("devices")[0].get("policyName") == "KRM5"
     assert instance.payload.get("devices")[0].get("serialNumber") == "BAR"
@@ -482,6 +492,7 @@ def test_image_mgmt_install_options_00021(image_install_options) -> None:
     instance.issu = False
     instance.epld = True
     instance.package_install = True
+    instance.unit_test = True
     instance._build_payload()  # pylint: disable=protected-access
     assert instance.payload.get("devices")[0].get("policyName") == "KRM5"
     assert instance.payload.get("devices")[0].get("serialNumber") == "BAR"
@@ -502,6 +513,7 @@ def test_image_mgmt_install_options_00022(image_install_options) -> None:
     match += "boolean value"
 
     instance = image_install_options
+    instance.unit_test = True
     with pytest.raises(AnsibleFailJson, match=match):
         instance.issu = "FOO"
 
@@ -518,6 +530,7 @@ def test_image_mgmt_install_options_00023(image_install_options) -> None:
     match += "boolean value"
 
     instance = image_install_options
+    instance.unit_test = True
     with pytest.raises(AnsibleFailJson, match=match):
         instance.epld = "FOO"
 
@@ -534,5 +547,6 @@ def test_image_mgmt_install_options_00024(image_install_options) -> None:
     match += "package_install must be a boolean value"
 
     instance = image_install_options
+    instance.unit_test = True
     with pytest.raises(AnsibleFailJson, match=match):
         instance.package_install = "FOO"