From bafad626087bccdc7cb1032cca68a5bcddff1ff2 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 12 Jun 2024 20:09:39 +0200 Subject: [PATCH 01/12] Add default manifest entries to maven jars. --- src/mx/_impl/mx.py | 2 +- src/mx/_impl/mx_jardistribution.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index 60f7ad99..b7e32dd3 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -18173,7 +18173,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.25.14") # mergetool-suite-import conflict marker +version = VersionSpec("7.26.0") # GR-54502 _mx_start_datetime = datetime.utcnow() diff --git a/src/mx/_impl/mx_jardistribution.py b/src/mx/_impl/mx_jardistribution.py index e151c5ce..ea30f702 100644 --- a/src/mx/_impl/mx_jardistribution.py +++ b/src/mx/_impl/mx_jardistribution.py @@ -779,6 +779,19 @@ def stage_archive(self): + self.manifest['Main-Class'] + " of the " + dist.name + " distribution. There should be only one definition.") self.manifest['Main-Class'] = mainClass + if self.dist.maven: + developer = self.dist.suite.developer + release_version = self.dist.suite.release_version() + + self.manifest.setdefault('Name', self.dist.maven_artifact_id()) + + for group in 'Specification', 'Implementation': + if hasattr(dist, 'description'): + self.manifest.setdefault(f'{group}-Title', dist.description) + self.manifest.setdefault(f'{group}-Version', release_version) + if 'organization' in developer: + self.manifest.setdefault(f'{group}-Vendor', developer['organization']) + for dep in head + tail: self.stage_dep(dep) From 292d734c97ef29518b7a1eb7e6f7679c9ea8b6b0 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Wed, 12 Jun 2024 20:53:58 +0200 Subject: [PATCH 02/12] Do not attempt to infer a Title. For example, `TRUFFLE_API` has a new line in the module description, which makes the Title illegal. --- src/mx/_impl/mx_jardistribution.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mx/_impl/mx_jardistribution.py b/src/mx/_impl/mx_jardistribution.py index ea30f702..6c3b5d57 100644 --- a/src/mx/_impl/mx_jardistribution.py +++ b/src/mx/_impl/mx_jardistribution.py @@ -786,8 +786,6 @@ def stage_archive(self): self.manifest.setdefault('Name', self.dist.maven_artifact_id()) for group in 'Specification', 'Implementation': - if hasattr(dist, 'description'): - self.manifest.setdefault(f'{group}-Title', dist.description) self.manifest.setdefault(f'{group}-Version', release_version) if 'organization' in developer: self.manifest.setdefault(f'{group}-Vendor', developer['organization']) From e79ec6b9d74697337e721807634c7bbaae668bf6 Mon Sep 17 00:00:00 2001 From: Danilo Ansaloni Date: Mon, 17 Jun 2024 13:29:26 +0200 Subject: [PATCH 03/12] Sync common.json. --- common.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/common.json b/common.json index 482ad596..02233e65 100644 --- a/common.json +++ b/common.json @@ -4,11 +4,11 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.25.5", + "mx_version": "7.25.14", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { - "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+1-1", "platformspecific": true, "extrabundles": ["static-libs"]}, + "galahad-jdk": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+2-53", "platformspecific": true, "extrabundles": ["static-libs"]}, "oraclejdk11": {"name": "jpg-jdk", "version": "11.0.11", "build_id": "jdk-11.0.11+9", "platformspecific": true, "extrabundles": ["static-libs"] }, @@ -45,13 +45,13 @@ "labsjdk-ee-21-llvm": {"name": "labsjdk", "version": "ee-21.0.2+13-jvmci-23.1-b33-sulong", "platformspecific": true }, "graalvm-ee-21": {"name": "graalvm-java21", "version": "23.1.3", "platformspecific": true }, - "oraclejdk-latest": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+1", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-24+1-jvmci-b01", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-24+1-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-24+1-jvmci-b01-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-24+1-jvmci-b01", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-24+1-jvmci-b01-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-24+1-jvmci-b01-sulong", "platformspecific": true } + "oraclejdk-latest": {"name": "jpg-jdk", "version": "24", "build_id": "jdk-24+2", "platformspecific": true, "extrabundles": ["static-libs"]}, + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-24+2-jvmci-b01", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-24+2-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-24+2-jvmci-b01-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-24+2-jvmci-b01", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-24+2-jvmci-b01-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-24+2-jvmci-b01-sulong", "platformspecific": true } }, "eclipse": { From 04a91bcee6c8a3a6ec8e7022aca96f3e93d8ffa4 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 13 Jun 2024 10:47:20 +0200 Subject: [PATCH 04/12] bump default max-java-compliance to 24 --- src/mx/_impl/mx_ide_intellij.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mx/_impl/mx_ide_intellij.py b/src/mx/_impl/mx_ide_intellij.py index bfa26fbf..28726a98 100644 --- a/src/mx/_impl/mx_ide_intellij.py +++ b/src/mx/_impl/mx_ide_intellij.py @@ -60,7 +60,7 @@ class IntellijConfig: external_projects: bool = True java_modules: bool = True native_projects: bool = False - max_java_compliance: int = 21 + max_java_compliance: int = 99 import_inner_classes: bool = False on_save_actions: bool = False refresh_only: bool = False From 696a34b43dcce047aac2e4c1dd020b231e4a9d4e Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 13 Jun 2024 10:47:33 +0200 Subject: [PATCH 05/12] Warn if max compliance is lower than requested one --- src/mx/_impl/mx_ide_intellij.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mx/_impl/mx_ide_intellij.py b/src/mx/_impl/mx_ide_intellij.py index 28726a98..83f91ff8 100644 --- a/src/mx/_impl/mx_ide_intellij.py +++ b/src/mx/_impl/mx_ide_intellij.py @@ -324,6 +324,8 @@ def _complianceToIntellijLanguageLevel(compliance): # they changed the name format starting with JDK_10 if compliance.value >= 10: # latest Idea 2021.2 requires the acceptance of a legal notice for beta Java specification to enable support for JDK17. Clamp at JDK16 by default + if config.max_java_compliance < compliance.value: + mx.warn(f"Requested Java compliance {compliance.value} is higher than maximum ({config.max_java_compliance}). Consider passing '--max-java-compliance'.") return 'JDK_' + str(min(compliance.value, config.max_java_compliance)) return 'JDK_1_' + str(compliance.value) From 764d4d420dcbdfc7812751952bf4889ea6e86d8e Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 13 Jun 2024 10:50:58 +0200 Subject: [PATCH 06/12] Remove obsolete comment --- src/mx/_impl/mx_ide_intellij.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mx/_impl/mx_ide_intellij.py b/src/mx/_impl/mx_ide_intellij.py index 83f91ff8..c8aee522 100644 --- a/src/mx/_impl/mx_ide_intellij.py +++ b/src/mx/_impl/mx_ide_intellij.py @@ -323,7 +323,6 @@ def _intellij_exclude_if_exists(xml, p, name, output=False): def _complianceToIntellijLanguageLevel(compliance): # they changed the name format starting with JDK_10 if compliance.value >= 10: - # latest Idea 2021.2 requires the acceptance of a legal notice for beta Java specification to enable support for JDK17. Clamp at JDK16 by default if config.max_java_compliance < compliance.value: mx.warn(f"Requested Java compliance {compliance.value} is higher than maximum ({config.max_java_compliance}). Consider passing '--max-java-compliance'.") return 'JDK_' + str(min(compliance.value, config.max_java_compliance)) From 9009ab49a4be60a08ece79a5ed5f5da7b65ffb84 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 18 Jun 2024 09:55:03 +0200 Subject: [PATCH 07/12] Update version --- src/mx/_impl/mx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index b7e32dd3..bb79a4d3 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -18173,7 +18173,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.26.0") # GR-54502 +version = VersionSpec("7.26.1") # GR-54784 _mx_start_datetime = datetime.utcnow() From 641ab5b7bb34001e3a3a9dfd3830cac5358ba735 Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Thu, 13 Jun 2024 15:27:15 +0200 Subject: [PATCH 08/12] Enforce strong hash algorithms on library and source downloads. --- src/mx/_impl/mx.py | 17 +++++++++++++---- src/mx/_impl/mx_compat.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index bb79a4d3..6bb276b4 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -3700,7 +3700,7 @@ def download_file_exists(urls): def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, ext=None, sources=False, canSymlink=True): return download_file_with_digest(name, path, urls, sha1, resolve, mustExist, ext, sources, canSymlink) -def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True): +def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext=None, sources=False, canSymlink=True, supported_hash_algorithms=None): """ Downloads an entity from a URL in the list `urls` (tried in order) to `path`, checking the digest of the result against `digest` (if not "NOCHECK") @@ -3710,6 +3710,12 @@ def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext= check_digest = digest and digest.value != 'NOCHECK' canSymlink = canSymlink and can_symlink() + if supported_hash_algorithms is not None: + if not check_digest: + abort(f'Refusing download of {name} without checking digest.') + if digest.name not in supported_hash_algorithms: + abort(f'Refusing download of {name} with weak hash algorithm {digest.name}.') + if len(urls) == 0 and not check_digest: return path @@ -8306,7 +8312,8 @@ def get_urls(self): return self.urls def get_path(self, resolve): - return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, ext=self.ext, canSymlink=True, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def getArchivableResults(self, use_relpath=True, single=False): path = realpath(self.get_path(False)) @@ -8636,7 +8643,8 @@ def is_available(self): def get_path(self, resolve): bootClassPathAgent = hasattr(self, 'bootClassPathAgent') and getattr(self, 'bootClassPathAgent').lower() == 'true' - return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.path, self.urls, self.digest, resolve, not self.optional, canSymlink=not bootClassPathAgent, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def _check_download_needed(self): if not _check_file_with_digest(self.path, self.digest): @@ -8649,7 +8657,8 @@ def _check_download_needed(self): def get_source_path(self, resolve): if self.sourcePath is None: return None - return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True) + compat = self.suite.getMxCompatibility() + return download_file_with_digest(self.name, self.sourcePath, self.sourceUrls, self.sourceDigest, resolve, len(self.sourceUrls) != 0, sources=True, supported_hash_algorithms=compat.get_supported_hash_algorithms()) def classpath_repr(self, resolve=True): path = self.get_path(resolve) diff --git a/src/mx/_impl/mx_compat.py b/src/mx/_impl/mx_compat.py index 25762860..9dc76539 100644 --- a/src/mx/_impl/mx_compat.py +++ b/src/mx/_impl/mx_compat.py @@ -300,6 +300,13 @@ def gate_run_pyformat(self) -> bool: """ return False + def get_supported_hash_algorithms(self): + """ + Enforce a strong hash on all downloads. + Returns a list of acceptable hash algorithms, or None to accept all (the behavior of old mx versions). + """ + return None + class MxCompatibility520(MxCompatibility500): @staticmethod @@ -765,6 +772,15 @@ def gate_run_pyformat(self) -> bool: return True +class MxCompatibility727(MxCompatibility713): + @staticmethod + def version(): + return mx.VersionSpec("7.27.0") + + def get_supported_hash_algorithms(self): + return ['sha256', 'sha512', 'sha3_384', 'sha3_512'] + + def minVersion(): _ensureCompatLoaded() return list(_versionsMap)[0] From a4940a00fd327faa9bf433815529a4a550c1556d Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Mon, 17 Jun 2024 12:51:31 +0200 Subject: [PATCH 09/12] Warning for usages of download_file_with_digest that don't enforce a strong hash. --- src/mx/_impl/mx.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index 6bb276b4..9c1dce0a 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -3710,11 +3710,19 @@ def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext= check_digest = digest and digest.value != 'NOCHECK' canSymlink = canSymlink and can_symlink() - if supported_hash_algorithms is not None: + if supported_hash_algorithms is None: + # Legacy usage of download_file_with_digest without enforcing strong hash. + # Check algorithm against the newest allowlist, but warn only for backwards compatibility. + algos = mx_compat.getMxCompatibility(version).get_supported_hash_algorithms() + if digest.name in algos: + warn(f'Deprecated use of dowload_file_with_digest without supported_hash_algorithms argument.\nVerifying download of {name} with strong hash {digest.name}, but this is not enforced.\nConsider bumping mxversion in suite.py to at least 7.27.0 to get rid of this warning.') + else: + warn(f'Verifying download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {algos}.') + else: if not check_digest: abort(f'Refusing download of {name} without checking digest.') if digest.name not in supported_hash_algorithms: - abort(f'Refusing download of {name} with weak hash algorithm {digest.name}.') + abort(f'Refusing download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {supported_hash_algorithms}.') if len(urls) == 0 and not check_digest: return path From 37ee5f26e71f9e1d62621608eb699ccfedc80dc3 Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Thu, 13 Jun 2024 15:32:11 +0200 Subject: [PATCH 10/12] Bump version. --- src/mx/_impl/mx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index 9c1dce0a..3a94df15 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -18190,7 +18190,7 @@ def alarm_handler(signum, frame): _CACHE_DIR = get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')) # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue -version = VersionSpec("7.26.1") # GR-54784 +version = VersionSpec("7.27.0") # strong digest check _mx_start_datetime = datetime.utcnow() From 0f86ddb1c660afaedc7c52917d9e46457a401b79 Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Wed, 19 Jun 2024 10:34:38 +0200 Subject: [PATCH 11/12] No strong hash check needed in `mx fetch-jdk`. --- src/mx/_impl/mx.py | 4 ++++ src/mx/_impl/mx_fetchjdk.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mx/_impl/mx.py b/src/mx/_impl/mx.py index 3a94df15..67d94943 100755 --- a/src/mx/_impl/mx.py +++ b/src/mx/_impl/mx.py @@ -3718,6 +3718,10 @@ def download_file_with_digest(name, path, urls, digest, resolve, mustExist, ext= warn(f'Deprecated use of dowload_file_with_digest without supported_hash_algorithms argument.\nVerifying download of {name} with strong hash {digest.name}, but this is not enforced.\nConsider bumping mxversion in suite.py to at least 7.27.0 to get rid of this warning.') else: warn(f'Verifying download of {name} with unsupported or weak hash algorithm {digest.name}.\nThe recommended algorithms are {algos}.') + elif 'all' in supported_hash_algorithms: + # Concious decision of the programmer that the hash check in this usage of `download_file_with_digest` is not security relevant. + # So we don't need to enforce a strong hash algorithm here. + pass else: if not check_digest: abort(f'Refusing download of {name} without checking digest.') diff --git a/src/mx/_impl/mx_fetchjdk.py b/src/mx/_impl/mx_fetchjdk.py index ee8c85e1..00aa8f80 100644 --- a/src/mx/_impl/mx_fetchjdk.py +++ b/src/mx/_impl/mx_fetchjdk.py @@ -104,7 +104,9 @@ def fetch_jdk(args): except Exception as e: #pylint: disable=broad-except mx.abort(f'Error retrieving {sha_url}: {e}') - mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False) + # Disable enforcment of strong hash algorithm for this download. The hash comes from the same server as the file, + # so there is no extra security benefit from this check. This is only checking for corrupted downloads. + mx.download_file_with_digest(artifact, archive_location, [url], digest, resolve=True, mustExist=True, sources=False, supported_hash_algorithms=['all']) extractor = mx.Extractor.create(archive_location) From 9c80a284b5813f3e55e2dca6c05ff17560e20c9e Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Wed, 19 Jun 2024 11:37:27 +0200 Subject: [PATCH 12/12] Sync common files. --- common.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.json b/common.json index 02233e65..f2ac2a82 100644 --- a/common.json +++ b/common.json @@ -4,7 +4,7 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.25.14", + "mx_version": "7.26.0", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": {