Merge remote-tracking branch 'spack/develop' into fi-nixpack

lib/spack/llnl/path.py

@@ -0,0 +1,105 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Path primitives that just require Python standard library."""
+import functools
+import sys
+from typing import List, Optional
+from urllib.parse import urlparse
+
+
+class Path:
+    """Enum to identify the path-style."""
+
+    unix: int = 0
+    windows: int = 1
+    platform_path: int = windows if sys.platform == "win32" else unix
+
+
+def format_os_path(path: str, mode: int = Path.unix) -> str:
+    """Formats the input path to use consistent, platform specific separators.
+
+    Absolute paths are converted between drive letters and a prepended '/' as per platform
+    requirement.
+
+    Parameters:
+        path: the path to be normalized, must be a string or expose the replace method.
+        mode: the path file separator style to normalize the passed path to.
+            Default is unix style, i.e. '/'
+    """
+    if not path:
+        return path
+    if mode == Path.windows:
+        path = path.replace("/", "\\")
+    else:
+        path = path.replace("\\", "/")
+    return path
+
+
+def convert_to_posix_path(path: str) -> str:
+    """Converts the input path to POSIX style."""
+    return format_os_path(path, mode=Path.unix)
+
+
+def convert_to_windows_path(path: str) -> str:
+    """Converts the input path to Windows style."""
+    return format_os_path(path, mode=Path.windows)
+
+
+def convert_to_platform_path(path: str) -> str:
+    """Converts the input path to the current platform's native style."""
+    return format_os_path(path, mode=Path.platform_path)
+
+
+def path_to_os_path(*parameters: str) -> List[str]:
+    """Takes an arbitrary number of positional parameters, converts each argument of type
+    string to use a normalized filepath separator, and returns a list of all values.
+    """
+
+    def _is_url(path_or_url: str) -> bool:
+        if "\\" in path_or_url:
+            return False
+        url_tuple = urlparse(path_or_url)
+        return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1
+
+    result = []
+    for item in parameters:
+        if isinstance(item, str) and not _is_url(item):
+            item = convert_to_platform_path(item)
+        result.append(item)
+    return result
+
+
+def system_path_filter(_func=None, arg_slice: Optional[slice] = None):
+    """Filters function arguments to account for platform path separators.
+    Optional slicing range can be specified to select specific arguments
+
+    This decorator takes all (or a slice) of a method's positional arguments
+    and normalizes usage of filepath separators on a per platform basis.
+
+    Note: `**kwargs`, urls, and any type that is not a string are ignored
+    so in such cases where path normalization is required, that should be
+    handled by calling path_to_os_path directly as needed.
+
+    Parameters:
+        arg_slice: a slice object specifying the slice of arguments
+            in the decorated method over which filepath separators are
+            normalized
+    """
+
+    def holder_func(func):
+        @functools.wraps(func)
+        def path_filter_caller(*args, **kwargs):
+            args = list(args)
+            if arg_slice:
+                args[arg_slice] = path_to_os_path(*args[arg_slice])
+            else:
+                args = path_to_os_path(*args)
+            return func(*args, **kwargs)
+
+        return path_filter_caller
+
+    if _func:
+        return holder_func(_func)
+    return holder_func

lib/spack/llnl/string.py

@@ -0,0 +1,67 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""String manipulation functions that do not have other dependencies than Python
+standard library
+"""
+from typing import List, Optional
+
+
+def comma_list(sequence: List[str], article: str = "") -> str:
+    if type(sequence) is not list:
+        sequence = list(sequence)
+
+    if not sequence:
+        return ""
+    if len(sequence) == 1:
+        return sequence[0]
+
+    out = ", ".join(str(s) for s in sequence[:-1])
+    if len(sequence) != 2:
+        out += ","  # oxford comma
+    out += " "
+    if article:
+        out += article + " "
+    out += str(sequence[-1])
+    return out
+
+
+def comma_or(sequence: List[str]) -> str:
+    """Return a string with all the elements of the input joined by comma, but the last
+    one (which is joined by 'or').
+    """
+    return comma_list(sequence, "or")
+
+
+def comma_and(sequence: List[str]) -> str:
+    """Return a string with all the elements of the input joined by comma, but the last
+    one (which is joined by 'and').
+    """
+    return comma_list(sequence, "and")
+
+
+def quote(sequence: List[str], q: str = "'") -> List[str]:
+    """Quotes each item in the input list with the quote character passed as second argument."""
+    return [f"{q}{e}{q}" for e in sequence]
+
+
+def plural(n: int, singular: str, plural: Optional[str] = None, show_n: bool = True) -> str:
+    """Pluralize <singular> word by adding an s if n != 1.
+
+    Arguments:
+        n: number of things there are
+        singular: singular form of word
+        plural: optional plural form, for when it's not just singular + 's'
+        show_n: whether to include n in the result string (default True)
+
+    Returns:
+        "1 thing" if n == 1 or "n things" if n != 1
+    """
+    number = f"{n} " if show_n else ""
+    if n == 1:
+        return f"{number}{singular}"
+    elif plural is not None:
+        return f"{number}{plural}"
+    else:
+        return f"{number}{singular}s"

lib/spack/llnl/util/filesystem.py

@@ -28,7 +28,8 @@
 from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink
 
 from spack.util.executable import Executable, which
-from spack.util.path import path_to_os_path, system_path_filter
+
+from ..path import path_to_os_path, system_path_filter
 
 if sys.platform != "win32":
     import grp
@@ -336,8 +337,7 @@ def groupid_to_group(x):
 
     if string:
         regex = re.escape(regex)
-    filenames = path_to_os_path(*filenames)
-    for filename in filenames:
+    for filename in path_to_os_path(*filenames):
         msg = 'FILTER FILE: {0} [replacing "{1}"]'
         tty.debug(msg.format(filename, regex))
 

lib/spack/llnl/util/lock.py

@@ -14,7 +14,7 @@
 
 from llnl.util import lang, tty
 
-import spack.util.string
+from ..string import plural
 
 if sys.platform != "win32":
     import fcntl
@@ -169,7 +169,7 @@ def _attempts_str(wait_time, nattempts):
     if nattempts <= 1:
         return ""
 
-    attempts = spack.util.string.plural(nattempts, "attempt")
+    attempts = plural(nattempts, "attempt")
     return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts)
 
 

lib/spack/llnl/util/symlink.py

@@ -11,8 +11,7 @@
 
 from llnl.util import lang, tty
 
-from spack.error import SpackError
-from spack.util.path import system_path_filter
+from ..path import system_path_filter
 
 if sys.platform == "win32":
     from win32file import CreateHardLink
@@ -338,7 +337,7 @@ def resolve_link_target_relative_to_the_link(link):
     return os.path.join(link_dir, target)
 
 
-class SymlinkError(SpackError):
+class SymlinkError(RuntimeError):
     """Exception class for errors raised while creating symlinks,
     junctions and hard links
     """

lib/spack/spack/binary_distribution.py

@@ -647,8 +647,7 @@ class BuildManifestVisitor(BaseDirectoryVisitor):
     directories."""
 
     def __init__(self):
-        # Save unique identifiers of files to avoid
-        # relocating hardlink files for each path.
+        # Save unique identifiers of hardlinks to avoid relocating them multiple times
         self.visited = set()
 
         # Lists of files we will check
@@ -657,6 +656,8 @@ def __init__(self):
 
     def seen_before(self, root, rel_path):
         stat_result = os.lstat(os.path.join(root, rel_path))
+        if stat_result.st_nlink == 1:
+            return False
         identifier = (stat_result.st_dev, stat_result.st_ino)
         if identifier in self.visited:
             return True
@@ -1581,9 +1582,10 @@ def dedupe_hardlinks_if_necessary(root, buildinfo):
         for rel_path in buildinfo[key]:
             stat_result = os.lstat(os.path.join(root, rel_path))
             identifier = (stat_result.st_dev, stat_result.st_ino)
-            if identifier in visited:
-                continue
-            visited.add(identifier)
+            if stat_result.st_nlink > 1:
+                if identifier in visited:
+                    continue
+                visited.add(identifier)
             new_list.append(rel_path)
         buildinfo[key] = new_list
 

lib/spack/spack/build_environment.py

@@ -43,6 +43,7 @@
 from typing import List, Tuple
 
 import llnl.util.tty as tty
+from llnl.string import plural
 from llnl.util.filesystem import join_path
 from llnl.util.lang import dedupe
 from llnl.util.symlink import symlink
@@ -82,7 +83,6 @@
 from spack.util.executable import Executable
 from spack.util.log_parse import make_log_context, parse_log_events
 from spack.util.module_cmd import load_module, module, path_from_modules
-from spack.util.string import plural
 
 #
 # This can be set by the user to globally disable parallel builds.

lib/spack/spack/build_systems/oneapi.py

@@ -61,6 +61,11 @@ def component_prefix(self):
         """Path to component <prefix>/<component>/<version>."""
         return self.prefix.join(join_path(self.component_dir, self.spec.version))
 
+    @property
+    def env_script_args(self):
+        """Additional arguments to pass to vars.sh script."""
+        return ()
+
     def install(self, spec, prefix):
         self.install_component(basename(self.url_for_version(spec.version)))
 
@@ -124,7 +129,7 @@ def setup_run_environment(self, env):
         if "~envmods" not in self.spec:
             env.extend(
                 EnvironmentModifications.from_sourcing_file(
-                    join_path(self.component_prefix, "env", "vars.sh")
+                    join_path(self.component_prefix, "env", "vars.sh"), *self.env_script_args
                 )
             )
 

lib/spack/spack/cmd/__init__.py

@@ -11,6 +11,7 @@
 from textwrap import dedent
 from typing import List, Match, Tuple
 
+import llnl.string
 import llnl.util.tty as tty
 from llnl.util.filesystem import join_path
 from llnl.util.lang import attr_setdefault, index_by
@@ -29,7 +30,6 @@
 import spack.user_environment as uenv
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
-import spack.util.string
 
 # cmd has a submodule called "list" so preserve the python list module
 python_list = list
@@ -516,7 +516,7 @@ def print_how_many_pkgs(specs, pkg_type=""):
             category, e.g. if pkg_type is "installed" then the message
             would be "3 installed packages"
     """
-    tty.msg("%s" % spack.util.string.plural(len(specs), pkg_type + " package"))
+    tty.msg("%s" % llnl.string.plural(len(specs), pkg_type + " package"))
 
 
 def spack_is_git_repo():

lib/spack/spack/cmd/buildcache.py

@@ -13,6 +13,7 @@
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as clr
+from llnl.string import plural
 from llnl.util.lang import elide_list
 
 import spack.binary_distribution as bindist
@@ -32,7 +33,6 @@
 from spack.cmd import display_specs
 from spack.spec import Spec, save_dependency_specfiles
 from spack.stage import Stage
-from spack.util.string import plural
 
 description = "create, download and install binary packages"
 section = "packaging"

lib/spack/spack/cmd/env.py

@@ -9,6 +9,7 @@
 import sys
 import tempfile
 
+import llnl.string as string
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
@@ -28,7 +29,6 @@
 import spack.schema.env
 import spack.spec
 import spack.tengine
-import spack.util.string as string
 from spack.util.environment import EnvironmentModifications
 
 description = "manage virtual environments"

lib/spack/spack/cmd/make_installer.py

@@ -6,10 +6,11 @@
 import posixpath
 import sys
 
+from llnl.path import convert_to_posix_path
+
 import spack.paths
 import spack.util.executable
 from spack.spec import Spec
-from spack.util.path import convert_to_posix_path
 
 description = "generate Windows installer"
 section = "admin"

lib/spack/spack/cmd/tags.py

@@ -5,6 +5,7 @@
 import io
 import sys
 
+import llnl.string
 import llnl.util.tty as tty
 import llnl.util.tty.colify as colify
 
@@ -24,7 +25,7 @@ def report_tags(category, tags):
     if isatty:
         num = len(tags)
         fmt = "{0} package tag".format(category)
-        buffer.write("{0}:\n".format(spack.util.string.plural(num, fmt)))
+        buffer.write("{0}:\n".format(llnl.string.plural(num, fmt)))
 
     if tags:
         colify.colify(tags, output=buffer, tty=isatty, indent=4)