diff --git a/src/test_workflow/dependency_installer.py b/src/test_workflow/dependency_installer.py index 046e0d30d8..b7ad413390 100644 --- a/src/test_workflow/dependency_installer.py +++ b/src/test_workflow/dependency_installer.py @@ -25,24 +25,28 @@ def __init__(self, root_url, build_manifest, bundle_manifest): self.bundle_manifest = bundle_manifest def download_dist(self, dest): - local_path = os.path.join(dest, os.path.basename(self.bundle_manifest.build.location)) + local_path = os.path.realpath(os.path.join(dest, os.path.basename(self.bundle_manifest.build.location))) return self.download_or_copy(self.bundle_manifest.build.location, local_path) + def __source_dest(self, path, category, dest): + source = "/".join([self.root_url, category, self.build_manifest.build.filename, path]) + dest = os.path.realpath(os.path.join(dest, "/".join(path.split("/")[1:]))) + return (source, dest) + def download(self, paths, category, dest): logging.info(f"Downloading to {dest} ...") with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - for path in paths: - url = "/".join([self.root_url, category, path]) - # paths are prefixed by category, remove - local_path = os.path.join(dest, "/".join(path.split("/")[1:])) - executor.submit(self.download_or_copy, url, local_path) + for result in executor.map( + lambda args: self.download_or_copy(*args), + map(lambda path: self.__source_dest(path, category, dest), + paths) + ): + logging.debug(f"Written {result}") def download_or_copy(self, source, dest): - dest = os.path.realpath(dest) os.makedirs(os.path.dirname(dest), exist_ok=True) if validators.url(source): logging.info(f"Downloading {source} into {dest} ...") - urllib.request.urlretrieve(source, dest) else: logging.info(f"Copying {source} into {dest} ...") diff --git a/tests/tests_test_workflow/test_dependency_installer_opensearch.py b/tests/tests_test_workflow/test_dependency_installer_opensearch.py index ffa3ebef78..a4abf345e2 100644 --- a/tests/tests_test_workflow/test_dependency_installer_opensearch.py +++ b/tests/tests_test_workflow/test_dependency_installer_opensearch.py @@ -1,6 +1,7 @@ import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import call, patch +from urllib.error import HTTPError from manifests.build_manifest import BuildManifest from manifests.bundle_manifest import BundleManifest @@ -13,25 +14,17 @@ class DependencyInstallerOpenSearchTests(unittest.TestCase): DIST_MANIFEST_LOCAL = os.path.join(DATA, "local", "dist", "opensearch", "manifest.yml") DIST_MANIFEST_REMOTE = os.path.join(DATA, "remote", "dist", "opensearch", "manifest.yml") - @patch("concurrent.futures.ThreadPoolExecutor", return_value=MagicMock()) @patch("os.makedirs") @patch("shutil.copyfile") @patch("urllib.request.urlretrieve") - def test_install_maven_dependencies_local(self, mock_request, mock_copyfile, mock_makedirs, mock_threadpool): - def submit_and_run(callable, source, dest): - # Filter down the actual calls to make analyzing failures easier - if 'alerting-notification-1.2.0.0.jar' in source: - return callable(source, dest) - return None - mock_threadpool.return_value.__enter__().submit.side_effect = submit_and_run + def test_install_maven_dependencies_local(self, mock_request, mock_copyfile, mock_makedirs): dependency_installer = DependencyInstallerOpenSearch( self.DATA, BuildManifest.from_path(self.BUILD_MANIFEST), BundleManifest.from_path(self.DIST_MANIFEST_LOCAL) ) + dependency_installer.install_maven_dependencies() - self.assertEqual(mock_threadpool.call_count, 4) - self.assertEqual(mock_threadpool().__enter__().submit.call_count, 2375) mock_makedirs.assert_called_with( os.path.realpath( @@ -42,30 +35,26 @@ def submit_and_run(callable, source, dest): exist_ok=True ) mock_request.assert_not_called() - mock_copyfile.assert_called_with( - os.path.join(self.DATA, "builds", "maven", "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar"), - os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar")) - ) + self.assertEqual(mock_copyfile.call_count, 2375) + mock_copyfile.assert_has_calls([ + call( + os.path.join(self.DATA, "builds", "opensearch", "maven", "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar"), + os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar")) + ) + ]) - @patch("concurrent.futures.ThreadPoolExecutor", return_value=MagicMock()) @patch("os.makedirs") @patch("shutil.copyfile") @patch("urllib.request.urlretrieve") - def test_install_maven_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs, mock_threadpool): - def submit_and_run(callable, source, dest): - # Filter down the actual calls to make analyzing failures easier - if 'alerting-notification-1.2.0.0.jar' in source: - return callable(source, dest) - return None - mock_threadpool.return_value.__enter__().submit.side_effect = submit_and_run + def test_install_maven_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs): dependency_installer = DependencyInstallerOpenSearch( "https://ci.opensearch.org/x/y", BuildManifest.from_path(self.BUILD_MANIFEST), BundleManifest.from_path(self.DIST_MANIFEST_REMOTE) ) + dependency_installer.install_maven_dependencies() - self.assertEqual(mock_threadpool.call_count, 4) - self.assertEqual(mock_threadpool().__enter__().submit.call_count, 2375) + self.assertEqual(mock_request.call_count, 2375) mock_makedirs.assert_called_with( os.path.realpath( os.path.join( @@ -75,11 +64,32 @@ def submit_and_run(callable, source, dest): exist_ok=True ) mock_copyfile.assert_not_called() - mock_request.assert_called_with( - "https://ci.opensearch.org/x/y/builds/maven/org/opensearch/notification/alerting-notification-1.2.0.0.jar", - os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar")) + mock_request.assert_has_calls([ + call( + "https://ci.opensearch.org/x/y/builds/opensearch/maven/org/opensearch/notification/alerting-notification-1.2.0.0.jar", + os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar")) + ) + ]) + + @patch("os.makedirs") + @patch("shutil.copyfile") + @patch("urllib.request.urlretrieve") + def test_install_maven_dependencies_remote_failure(self, mock_request, mock_copyfile, mock_makedirs): + def mock_retrieve(source, dest): + raise HTTPError(url=source, hdrs={}, fp=None, msg="Not Found", code=404) + + mock_request.side_effect = mock_retrieve + + dependency_installer = DependencyInstallerOpenSearch( + "https://ci.opensearch.org/x/y", + BuildManifest.from_path(self.BUILD_MANIFEST), + BundleManifest.from_path(self.DIST_MANIFEST_REMOTE) ) + with self.assertRaises(HTTPError) as ctx: + dependency_installer.install_maven_dependencies() + self.assertEqual(str(ctx.exception), "HTTP Error 404: Not Found") + @patch("os.makedirs") @patch("shutil.copyfile") @patch("urllib.request.urlretrieve")