From dc81548cdb46897cbb462e10f8ed044babf7a159 Mon Sep 17 00:00:00 2001 From: Marco Hutter Date: Mon, 29 Aug 2016 21:32:49 +0200 Subject: [PATCH] Improved support for native library deployment See https://github.com/jcuda/jcuda-main/issues/8 --- .gitignore | 7 +- JCudaDriverJNI/CMakeLists.txt | 2 +- JCudaJava/pom.xml | 97 +--- JCudaJava/src/main/java/jcuda/LibUtils.java | 481 +++++++++++++----- .../main/java/jcuda/driver/JCudaDriver.java | 11 +- .../src/main/java/jcuda/runtime/JCuda.java | 16 +- JCudaRuntimeJNI/CMakeLists.txt | 2 +- JNvrtcJNI/CMakeLists.txt | 2 +- 8 files changed, 399 insertions(+), 219 deletions(-) diff --git a/.gitignore b/.gitignore index ab4673f..e7c62a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ -JCudaJava/target/ \ No newline at end of file +/JCudaJava/target/ +/JCudaJava/.settings +/JCudaJava/.classpath +/JCudaJava/.project +/target +/nativeLibraries diff --git a/JCudaDriverJNI/CMakeLists.txt b/JCudaDriverJNI/CMakeLists.txt index ce7787e..f8fd147 100644 --- a/JCudaDriverJNI/CMakeLists.txt +++ b/JCudaDriverJNI/CMakeLists.txt @@ -36,4 +36,4 @@ target_link_libraries(${PROJECT_NAME} ) set_target_properties(${PROJECT_NAME} - PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${CMAKE_HOST}-${CMAKE_ARCH}) + PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${JCUDA_VERSION}-${JCUDA_OS}-${JCUDA_ARCH}) diff --git a/JCudaJava/pom.xml b/JCudaJava/pom.xml index adf366d..41e7e53 100644 --- a/JCudaJava/pom.xml +++ b/JCudaJava/pom.xml @@ -5,13 +5,14 @@ org.jcuda jcuda-parent - 0.7.5b + 0.7.5c ../../jcuda-common/JCudaParentPOM/pom.xml jcuda + junit junit @@ -21,103 +22,11 @@ org.jcuda - JCudaRuntime-native - ${project.version} - ${jcuda.os}-${jcuda.arch} - ${jcuda.ext} - - - - org.jcuda - JCudaDriver-native + jcuda-natives ${project.version} ${jcuda.os}-${jcuda.arch} - ${jcuda.ext} - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - - process-test-resources - copy-natives - - copy - - - - - org.jcuda - JCudaRuntime-native - ${jcuda.os}-${jcuda.arch} - ${jcuda.ext} - true - ${project.build.directory}/lib - ${jcuda.prefix}JCudaRuntime-${jcuda.os}-${jcuda.arch}.${jcuda.ext} - - - org.jcuda - JCudaDriver-native - ${jcuda.os}-${jcuda.arch} - ${jcuda.ext} - true - ${project.build.directory}/lib - ${jcuda.prefix}JCudaDriver-${jcuda.os}-${jcuda.arch}.${jcuda.ext} - - - - - - - - - - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - - org.apache.maven.plugins - - - maven-dependency-plugin - - - [2.10,) - - - - copy-dependencies - - copy - - - - - - - - - - - - - - \ No newline at end of file diff --git a/JCudaJava/src/main/java/jcuda/LibUtils.java b/JCudaJava/src/main/java/jcuda/LibUtils.java index 39d9e65..dc6ef0a 100644 --- a/JCudaJava/src/main/java/jcuda/LibUtils.java +++ b/JCudaJava/src/main/java/jcuda/LibUtils.java @@ -27,135 +27,307 @@ package jcuda; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.Locale; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Utility class for detecting the operating system and architecture * types, and automatically loading the matching native library - * as a resource or from a file.
- *
- * The architecture and OS detection has been adapted from - * http://javablog.co.uk/2007/05/19/making-jni-cross-platform/ - * and extended with http://lopica.sourceforge.net/os.html + * as a resource or from a file.
+ *
+ * This class is not intended to be used by clients.
+ *
*/ public final class LibUtils { + // The architecture and OS detection has been adapted from + // http://javablog.co.uk/2007/05/19/making-jni-cross-platform/ + // and extended with http://lopica.sourceforge.net/os.html + + /** + * The logger used in this class + */ + private static final Logger logger = + Logger.getLogger(LibUtils.class.getName()); + + /** + * The default log level + */ + private static final Level level = Level.FINE; + + /** + * The directory where libraries are expected in JAR files, + * when they are loaded as resources + */ + private static final String LIBRARY_PATH_IN_JAR = "/lib"; + /** * Enumeration of common operating systems, independent of version * or architecture. */ public static enum OSType { - APPLE, LINUX, SUN, WINDOWS, UNKNOWN + ANDROID, APPLE, LINUX, SUN, WINDOWS, UNKNOWN } /** * Enumeration of common CPU architectures. */ - public static enum ARCHType + public static enum ArchType { - PPC, PPC_64, SPARC, X86, X86_64, ARM, MIPS, RISC, UNKNOWN + PPC, PPC_64, SPARC, X86, X86_64, ARM, ARM64, MIPS, MIPS64, RISC, UNKNOWN } /** - * Loads the specified library. The full name of the library - * is created by calling {@link LibUtils#createLibName(String)} - * with the given argument. The method will attempt to load - * the library as a as a resource (for usage within a JAR), - * and, if this fails, using the usual System.loadLibrary - * call. - * - * @param baseName The base name of the library - * @throws UnsatisfiedLinkError if the native library + * Private constructor to prevent instantiation. + */ + private LibUtils() + { + // Private constructor to prevent instantiation. + } + + /** + * Loads the specified library.
+ *
+ * The method will attempt to load the library using the usual + * System.loadLibrary call. In this case, the specified + * dependent libraries are ignored, because they are assumed to be + * loaded automatically in the same way as the main library.
+ *
+ * If the library can not be loaded with the + * System.loadLibrary call, then this method will attempt + * to load the file as a resource (usually one that is contained in + * a JAR file). In this case, the library is assumed to be located + * in subdirectory called "/lib" inside the JAR file. + * The method will try to load a resource that has the platform-specific + * {@link #createLibraryFileName(String) library file name} from + * this directory, extract it into the default directory for temporary + * files, and load the library from there.
+ *
+ * In this case, the specified dependent libraries may also be loaded + * as resources. They are assumed to be located in subdirectories + * that are named according to the {@link #osString()} and + * {@link #archString()} of the executing platform. For example, such + * a library may be located in a directory inside the JAR that is + * called "/lib/windows/x86_64". These dependent libraries + * will be extracted and loaded before the main library is loaded. + * + * @param libraryName The name of the library (without a platform specific + * prefix or file extension) + * @param dependentLibraryNames The names of libraries that the library + * to load depends on. If the library is loaded as a resource, then + * it will be attempted to also load these libraries as resources, as + * described above + * @throws UnsatisfiedLinkError if the native library * could not be loaded. */ - public static void loadLibrary(String baseName) + public static void loadLibrary( + String libraryName, String ... dependentLibraryNames) { - String libName = LibUtils.createLibName(baseName); + logger.log(level, "Loading library: " + libraryName); - Throwable throwable = null; - final boolean tryResource = true; - if (tryResource) + // First, try to load the specified library as a file + // that is visible in the default search path + Throwable throwableFromFile; + try { - try - { - loadLibraryResource(libName); - return; - } - catch (Throwable t) - { - throwable = t; - } + logger.log(level, "Loading library as a file"); + System.loadLibrary(libraryName); + logger.log(level, "Loading library as a file DONE"); + return; + } + catch (Throwable t) + { + logger.log(level, "Loading library as a file FAILED"); + throwableFromFile = t; } + // Now try to load the library by extracting the + // corresponding resource from the JAR file try { - System.loadLibrary(libName); + logger.log(level, "Loading library as a resource"); + loadLibraryResource(LIBRARY_PATH_IN_JAR, + libraryName, "", dependentLibraryNames); + logger.log(level, "Loading library as a resource DONE"); return; } - catch (Throwable t) + catch (Throwable throwableFromResource) { + logger.log(level, "Loading library as a resource FAILED", + throwableFromResource); + StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); pw.println("Error while loading native library \"" + - libName + "\" with base name \""+baseName+"\""); + libraryName + "\""); pw.println("Operating system name: "+ - System.getProperty("os.name")); + System.getProperty("os.name")); pw.println("Architecture : "+ - System.getProperty("os.arch")); + System.getProperty("os.arch")); pw.println("Architecture bit size: "+ - System.getProperty("sun.arch.data.model")); - - if (throwable != null) - { - pw.println( - "Stack trace from the attempt to " + - "load the library as a resource:"); - throwable.printStackTrace(pw); - } + System.getProperty("sun.arch.data.model")); - pw.println( - "Stack trace from the attempt to " + + pw.println("---(start of nested stack traces)---"); + + pw.println("Stack trace from the attempt to " + "load the library as a file:"); - t.printStackTrace(pw); + throwableFromFile.printStackTrace(pw); + + pw.println("Stack trace from the attempt to " + + "load the library as a resource:"); + throwableFromResource.printStackTrace(pw); + + pw.println("---(end of nested stack traces)---"); - pw.flush(); pw.close(); - throw new UnsatisfiedLinkError( - "Could not load the native library.\n"+ - sw.toString()); + throw new UnsatisfiedLinkError(sw.toString()); } } + /** - * Load the library with the given name from a resource. - * The extension for the current OS will be appended. - * - * @param libName The library name + * Load the library with the given name from a resource. + * + * @param resourceSubdirectoryName The subdirectory where the resource + * is expected + * @param libraryName The library name, e.g. "EXAMPLE-windows-x86" + * @param tempSubdirectoryName The name for the subdirectory in the + * temp directory, where the temporary files for dependent libraries + * should be stored + * @param dependentLibraryNames The names of libraries that the library + * to load depends on, and that may have to be loaded as resources and + * stored as temporary files as well * @throws Throwable If the library could not be loaded */ - private static void loadLibraryResource(String libName) throws Throwable + private static void loadLibraryResource( + String resourceSubdirectoryName, + String libraryName, + String tempSubdirectoryName, + String ... dependentLibraryNames) throws Throwable + { + // First try to load all dependent libraries, recursively + for (String dependentLibraryName : dependentLibraryNames) + { + logger.log(level, + "Library " + libraryName + + " depends on " + dependentLibraryName); + + String dependentResourceSubdirectoryName = + resourceSubdirectoryName + "/" + + osString() + "/" + + archString(); + + String dependentLibraryTempSubDirectoryName = + libraryName + "_dependents" + File.separator + + osString() + File.separator + + archString() + File.separator; + + loadLibraryResource( + dependentResourceSubdirectoryName, + dependentLibraryName, + dependentLibraryTempSubDirectoryName); + } + + // Now, prepare loading the actual library + String libraryFileName = createLibraryFileName(libraryName); + File libraryTempFile = createTempFile( + tempSubdirectoryName, libraryFileName); + + // If the temporary file for the library does not exist, create it + if (!libraryTempFile.exists()) + { + String libraryResourceName = + resourceSubdirectoryName + "/" + libraryFileName; + logger.log(level, + "Writing resource " + libraryResourceName); + logger.log(level, + "to temporary file " + libraryTempFile); + writeResourceToFile(libraryResourceName, libraryTempFile); + } + + // Finally, try to load the library from the temporary file + logger.log(level, "Loading library " + libraryTempFile); + System.load(libraryTempFile.toString()); + logger.log(level, "Loading library " + libraryTempFile + " DONE"); + } + + + /** + * Create a file object representing the file with the given name + * in the specified subdirectory of the default "temp" directory. + * If the specified subdirectory does not exist yet, it is created. + * + * @param name The file name + * @return The file + * @throws IOException If the subdirectory can not be created + */ + private static File createTempFile( + String tempSubdirectoryName, String name) throws IOException { - String libPrefix = createLibPrefix(); - String libExtension = createLibExtension(); - String fullName = libPrefix + libName; - String resourceName = "/lib/" + fullName + "." + libExtension; - InputStream inputStream = + String tempDirName = System.getProperty("java.io.tmpdir"); + File tempSubDirectory = + new File(tempDirName + File.separator + tempSubdirectoryName); + if (!tempSubDirectory.exists()) + { + boolean createdDirectory = tempSubDirectory.mkdirs(); + if (!createdDirectory) + { + throw new IOException( + "Could not create directory for temporary file: " + + tempSubDirectory); + } + } + String tempFileName = tempSubDirectory + File.separator + name; + File tempFile = new File(tempFileName); + return tempFile; + } + + + /** + * Obtain an input stream to the resource with the given name, and write + * it to the specified file (which may not be null, and + * may not exist yet) + * + * @param resourceName The name of the resource + * @param file The file to write to + * @throws NullPointerException If the given file is null + * @throws IllegalArgumentException If the given file already exists + * @throws IOException If an IO error occurs + */ + private static void writeResourceToFile( + String resourceName, File file) throws IOException + { + if (file == null) + { + throw new NullPointerException("Target file may not be null"); + } + if (file.exists()) + { + throw new IllegalArgumentException( + "Target file already exists: "+file); + } + InputStream inputStream = LibUtils.class.getResourceAsStream(resourceName); if (inputStream == null) { - throw new NullPointerException( - "No resource found with name '"+resourceName+"'"); + throw new IOException( + "No resource found with name '"+resourceName+"'"); } - File tempFile = File.createTempFile(fullName, "."+libExtension); - tempFile.deleteOnExit(); OutputStream outputStream = null; try { - outputStream = new FileOutputStream(tempFile); - byte[] buffer = new byte[8192]; + outputStream = new FileOutputStream(file); + byte[] buffer = new byte[32768]; while (true) { int read = inputStream.read(buffer); @@ -163,90 +335,157 @@ private static void loadLibraryResource(String libName) throws Throwable { break; } - outputStream.write(buffer, 0, read); + outputStream.write(buffer, 0, read); } outputStream.flush(); - outputStream.close(); - outputStream = null; - System.load(tempFile.toString()); } - finally + finally { if (outputStream != null) { - outputStream.close(); + try + { + outputStream.close(); + } + catch (IOException e) + { + logger.log(Level.SEVERE, e.getMessage(), e); + } + } + try + { + inputStream.close(); + } + catch (IOException e) + { + logger.log(Level.SEVERE, e.getMessage(), e); } } } + + /** + /** + * Create the full library file name, including the extension + * and prefix, for the given library name. For example, the + * name "EXAMPLE" will become
+ * EXAMPLE.dll on Windows
+ * libEXAMPLE.so on Linux
+ * EXAMPLE.dylib on MacOS
+ * + * @param libraryName The library name + * @return The full library name, with extension + */ + public static String createLibraryFileName(String libraryName) + { + String libPrefix = createLibraryPrefix(); + String libExtension = createLibraryExtension(); + String fullName = libPrefix + libraryName + "." + libExtension; + return fullName; + } /** * Returns the extension for dynamically linked libraries on the - * current OS. That is, returns "dylib" on Apple, "so" on Linux - * and Sun, and "dll" on Windows. + * current OS. That is, returns "dylib" on Apple, + * "so" on Linux and Sun, and "dll" + * on Windows. * * @return The library extension */ - private static String createLibExtension() + private static String createLibraryExtension() { OSType osType = calculateOS(); switch (osType) { case APPLE: return "dylib"; + case ANDROID: case LINUX: - return "so"; case SUN: return "so"; case WINDOWS: return "dll"; + default: + break; } return ""; } /** * Returns the prefix for dynamically linked libraries on the - * current OS. That is, returns "lib" on Apple, Linux and Sun, - * and the empty String on Windows. + * current OS. That is, returns "lib" on Apple, + * Linux and Sun, and the empty String on Windows. * * @return The library prefix */ - private static String createLibPrefix() + private static String createLibraryPrefix() { OSType osType = calculateOS(); switch (osType) { + case ANDROID: case APPLE: case LINUX: case SUN: return "lib"; case WINDOWS: return ""; + default: + break; } return ""; } /** - * Creates the name for the native library with the given base - * name for the current operating system and architecture. - * The resulting name will be of the form
- * baseName-OSType-ARCHType
- * where OSType and ARCHType are the lower case Strings - * of the respective enum constants. Example:
- * jcuda-windows-x86
- * + * Creates the name for the native library with the given base name for + * the current platform, by appending strings that indicate the current + * operating system and architecture.
+ *
+ * The resulting name will be of the form
+ * baseName-OSType-ArchType
+ * where OSType and ArchType are the lower case Strings + * of the respective {@link LibUtils.OSType OSType} and + * {@link LibUtils.ArchType ArcType} enum constants.
+ *
+ * For example, the library name with the base name "EXAMPLE" may be
+ * EXAMPLE-windows-x86
+ *
+ * Note that the resulting name will not include any platform specific + * prefixes or extensions for the actual name. + * * @param baseName The base name of the library * @return The library name */ - public static String createLibName(String baseName) + public static String createPlatformLibraryName(String baseName) + { + return baseName + "-" + osString() + "-" + archString(); + } + + /** + * Returns a the lower case String representation of + * the {@link #calculateOS() OSType} of this platform. E.g. + * "windows". + * + * @return The string describing the operating system + */ + private static String osString() { OSType osType = calculateOS(); - ARCHType archType = calculateArch(); - String libName = baseName; - libName += "-" + osType.toString().toLowerCase(Locale.ENGLISH); - libName += "-" + archType.toString().toLowerCase(Locale.ENGLISH); - return libName; + return osType.toString().toLowerCase(Locale.ENGLISH); + } + + /** + * Returns a the lower case String representation of + * the {@link #calculateArch() ArchType} of this platform. E.g. + * "x86_64". + * + * @return The string describing the architecture + */ + private static String archString() + { + ArchType archType = calculateArch(); + return archType.toString().toLowerCase(Locale.ENGLISH); } /** @@ -256,6 +495,11 @@ public static String createLibName(String baseName) */ public static OSType calculateOS() { + String vendor = System.getProperty("java.vendor"); + if ("The Android Project".equals(vendor)) + { + return OSType.ANDROID; + } String osName = System.getProperty("os.name"); osName = osName.toLowerCase(Locale.ENGLISH); if (osName.startsWith("mac os")) @@ -283,51 +527,52 @@ public static OSType calculateOS() * * @return The current ARCHType */ - public static ARCHType calculateArch() + public static ArchType calculateArch() { String osArch = System.getProperty("os.arch"); osArch = osArch.toLowerCase(Locale.ENGLISH); - if (osArch.equals("i386") || - osArch.equals("x86") || - osArch.equals("i686")) + if ("i386".equals(osArch) || + "x86".equals(osArch) || + "i686".equals(osArch)) { - return ARCHType.X86; + return ArchType.X86; } if (osArch.startsWith("amd64") || osArch.startsWith("x86_64")) { - return ARCHType.X86_64; + return ArchType.X86_64; } - if (osArch.equals("ppc") || osArch.equals("powerpc")) + if (osArch.startsWith("arm64")) { - return ARCHType.PPC; + return ArchType.ARM64; + } + if (osArch.startsWith("arm")) + { + return ArchType.ARM; + } + if ("ppc".equals(osArch) || "powerpc".equals(osArch)) + { + return ArchType.PPC; } if (osArch.startsWith("ppc")) { - return ARCHType.PPC_64; + return ArchType.PPC_64; } if (osArch.startsWith("sparc")) { - return ARCHType.SPARC; + return ArchType.SPARC; } - if (osArch.startsWith("arm")) + if (osArch.startsWith("mips64")) { - return ARCHType.ARM; + return ArchType.MIPS64; } if (osArch.startsWith("mips")) { - return ARCHType.MIPS; + return ArchType.MIPS; } if (osArch.contains("risc")) { - return ARCHType.RISC; + return ArchType.RISC; } - return ARCHType.UNKNOWN; - } - - /** - * Private constructor to prevent instantiation. - */ - private LibUtils() - { + return ArchType.UNKNOWN; } } diff --git a/JCudaJava/src/main/java/jcuda/driver/JCudaDriver.java b/JCudaJava/src/main/java/jcuda/driver/JCudaDriver.java index cc9e0f1..f77e63e 100644 --- a/JCudaJava/src/main/java/jcuda/driver/JCudaDriver.java +++ b/JCudaJava/src/main/java/jcuda/driver/JCudaDriver.java @@ -29,7 +29,11 @@ import java.util.Arrays; -import jcuda.*; +import jcuda.CudaException; +import jcuda.LibUtils; +import jcuda.LogLevel; +import jcuda.Pointer; +import jcuda.runtime.JCuda; /** * Java bindings for the NVidia CUDA driver API.
@@ -272,7 +276,10 @@ private ConstantCUstream(long value) static { - LibUtils.loadLibrary("JCudaDriver"); + String libraryBaseName = "JCudaDriver-" + JCuda.getJCudaVersion(); + String libraryName = + LibUtils.createPlatformLibraryName(libraryBaseName); + LibUtils.loadLibrary(libraryName); } /* Private constructor to prevent instantiation */ diff --git a/JCudaJava/src/main/java/jcuda/runtime/JCuda.java b/JCudaJava/src/main/java/jcuda/runtime/JCuda.java index 3d5ba12..fb5395f 100644 --- a/JCudaJava/src/main/java/jcuda/runtime/JCuda.java +++ b/JCudaJava/src/main/java/jcuda/runtime/JCuda.java @@ -42,6 +42,17 @@ public class JCuda */ public static final int CUDART_VERSION = 7050; + /** + * Returns an unspecified string that will be appended to native + * library names for disambiguation + * + * @return The JCuda version + */ + public static String getJCudaVersion() + { + return "0.7.5c"; + } + /** * Default page-locked allocation flag */ @@ -380,7 +391,10 @@ public static void initialize() { if (!initialized) { - LibUtils.loadLibrary("JCudaRuntime"); + String libraryBaseName = "JCudaRuntime-" + getJCudaVersion(); + String libraryName = + LibUtils.createPlatformLibraryName(libraryBaseName); + LibUtils.loadLibrary(libraryName); initialized = true; } } diff --git a/JCudaRuntimeJNI/CMakeLists.txt b/JCudaRuntimeJNI/CMakeLists.txt index fdf5b31..594c12a 100644 --- a/JCudaRuntimeJNI/CMakeLists.txt +++ b/JCudaRuntimeJNI/CMakeLists.txt @@ -36,4 +36,4 @@ target_link_libraries(${PROJECT_NAME} set_target_properties(${PROJECT_NAME} - PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${CMAKE_HOST}-${CMAKE_ARCH}) + PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${JCUDA_VERSION}-${JCUDA_OS}-${JCUDA_ARCH}) diff --git a/JNvrtcJNI/CMakeLists.txt b/JNvrtcJNI/CMakeLists.txt index b9c9c6b..2748995 100644 --- a/JNvrtcJNI/CMakeLists.txt +++ b/JNvrtcJNI/CMakeLists.txt @@ -37,4 +37,4 @@ target_link_libraries(${PROJECT_NAME} ) set_target_properties(${PROJECT_NAME} - PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${CMAKE_HOST}-${CMAKE_ARCH}) + PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${JCUDA_VERSION}-${JCUDA_OS}-${JCUDA_ARCH})