diff --git a/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgFactory.java b/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgFactory.java new file mode 100644 index 0000000..dd7f87a --- /dev/null +++ b/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgFactory.java @@ -0,0 +1,190 @@ +/* + * #%L + * ImgLib2: a general-purpose, multidimensional image processing library. + * %% + * Copyright (C) 2009 - 2016 Tobias Pietzsch, Stephan Preibisch, Stephan Saalfeld, + * John Bogovic, Albert Cardona, Barry DeZonia, Christian Dietz, Jan Funke, + * Aivar Grislis, Jonathan Hale, Grant Harris, Stefan Helfrich, Mark Hiner, + * Martin Horn, Steffen Jaensch, Lee Kamentsky, Larry Lindsey, Melissa Linkert, + * Mark Longair, Brian Northan, Nick Perry, Curtis Rueden, Johannes Schindelin, + * Jean-Yves Tinevez and Michael Zinsmaier. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package net.imglib2.cache.img; + +import net.imglib2.cache.Cache; +import net.imglib2.cache.CacheLoader; +import net.imglib2.cache.IoSync; +import net.imglib2.cache.LoaderRemoverCache; +import net.imglib2.cache.ref.GuardedStrongRefLoaderRemoverCache; +import net.imglib2.cache.ref.SoftRefLoaderRemoverCache; +import net.imglib2.img.NativeImgFactory; +import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory; +import net.imglib2.img.basictypeaccess.array.ArrayDataAccess; +import net.imglib2.img.cell.Cell; +import net.imglib2.img.cell.CellGrid; +import net.imglib2.type.NativeType; +import net.imglib2.type.NativeTypeFactory; +import net.imglib2.util.Fraction; + +/** + * Abstract factory for creating {@link CachedCellImg}s. Holds functionality shared by read-write-caches, + * but leaves the implementation of the cell writing to specialized implementations. + * + * See {@link DiskCachedCellImgFactory} for a specialized example. + * + * @param Element type of the images that can be created by this factory + * + * @author Tobias Pietzsch + * @author Carsten Haubold, KNIME GmbH, Konstanz, Germany + */ +public abstract class AbstractReadWriteCachedCellImgFactory> extends NativeImgFactory { + /** + * Create a new {@link AbstractReadWriteCachedCellImgFactory} that can create images of the provided type. + * + * @param type Element type of the images that can be created by this factory + */ + public AbstractReadWriteCachedCellImgFactory(final T type) { + super(type); + } + + /** + * Merge this factory's default options (which can be specified in the constructor) with the user provided options. + * User provided option values that differ from their default value take precedence over the factory's default options + * + * @param userProvidedOptions The options that were provided by the user when calling one of the methods of this factory. + * @return a new options object created by merging this factory's default options with the provided ones + */ + abstract AbstractReadWriteCachedCellImgOptions mergeWithFactoryOptions(final AbstractReadWriteCachedCellImgOptions userProvidedOptions); + + /** + * Create a cached cell img with the provided settings. Much of the work is deferred to abstract methods that + * must be implemented for the specific writer-backend in specialized classes. + * + * @param dimensions Dimensions of the image that should be created + * @param cacheLoader Loader for already cached cells (optional) + * @param cellLoader Loader for Cells that are not cached yet (optional, cache will provide empty cells if this is null) + * @param type Instance of the element type of the image that should be created + * @param typeFactory A native type factory + * @param additionalOptions Cache options that extend this cache factory's options + * @param Access Type + * @return A CachedCellImg with the given cache configuration + */ + protected > CachedCellImg create(final long[] dimensions, + final CacheLoader> cacheLoader, + final CellLoader cellLoader, + final T type, + final NativeTypeFactory typeFactory, + final AbstractReadWriteCachedCellImgOptions additionalOptions) { + + final AbstractReadWriteCachedCellImgOptions options = mergeWithFactoryOptions(additionalOptions); + + final Fraction entitiesPerPixel = type.getEntitiesPerPixel(); + + final CellGrid grid = ReadOnlyCachedCellImgFactory.createCellGrid(dimensions, options.values().cellDimensions(), entitiesPerPixel); + + @SuppressWarnings("unchecked") + CacheLoader> backingLoader = (CacheLoader>) cacheLoader; + if (backingLoader == null) { + if (cellLoader != null) { + final CellLoader actualCellLoader = options.values().initializeCellsAsDirty() ? cell -> { + cellLoader.load(cell); + cell.setDirty(); + } : cellLoader; + backingLoader = LoadedCellCacheLoader.get(grid, actualCellLoader, type, options.values().accessFlags()); + } else + backingLoader = EmptyCellCacheLoader.get(grid, type, options.values().accessFlags()); + } + + final ReadWriteCellCache cellCache = createCellCache(options, grid, backingLoader, type, entitiesPerPixel); + + final IoSync> iosync = new IoSync<>(cellCache, options.values().numIoThreads(), options.values().maxIoQueueSize()); + + LoaderRemoverCache> listenableCache; + switch (options.values().cacheType()) { + case BOUNDED: + listenableCache = new GuardedStrongRefLoaderRemoverCache<>(options.values().maxCacheSize()); + break; + case SOFTREF: + default: + listenableCache = new SoftRefLoaderRemoverCache<>(); + break; + } + + final Cache> cache = listenableCache.withRemover(iosync).withLoader(iosync); + final A accessType = ArrayDataAccessFactory.get(typeFactory, options.values().accessFlags()); + + final CachedCellImg img = createCachedCellImg(grid, entitiesPerPixel, cache, accessType); + img.setLinkedType(typeFactory.createLinkedType(img)); + return img; + } + + /** + * Derived classes should create an instance of the CachedCellImg type that they support, given the provided cache and grid + * E.g. a {@link DiskCachedCellImgFactory} would create and return a {@link DiskCachedCellImg}. + * + * @param grid The grid structure of the CellCache + * @param entitiesPerPixel + * @param cache The configured cache to use as backing for the image + * @param accessType + * @return A {@link CachedCellImg} + */ + protected abstract > CachedCellImg createCachedCellImg( + final CellGrid grid, + final Fraction entitiesPerPixel, + final Cache> cache, + final A accessType + ); + + /** + * Derived classes should create a read-write cell cache with the given options, cell grid and backing loader. + * @param options cache creation options + * @param grid cell grid + * @param backingLoader the backing loader for cache cells + * @param type element type + * @param entitiesPerPixel + * @return A {@link ReadWriteCellCache} + */ + protected abstract > ReadWriteCellCache createCellCache( + final AbstractReadWriteCachedCellImgOptions options, + final CellGrid grid, + final CacheLoader> backingLoader, + final T type, + final Fraction entitiesPerPixel); + + /* + * ----------------------------------------------------------------------- + * + * Deprecated API. + * + * Supports backwards compatibility with ImgFactories that are constructed + * without a type instance or supplier. + * + * ----------------------------------------------------------------------- + */ + @Deprecated + public AbstractReadWriteCachedCellImgFactory() { + } +} diff --git a/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgOptions.java b/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgOptions.java new file mode 100644 index 0000000..990aa93 --- /dev/null +++ b/src/main/java/net/imglib2/cache/img/AbstractReadWriteCachedCellImgOptions.java @@ -0,0 +1,271 @@ +/* + * #%L + * BigDataViewer core classes with minimal dependencies + * %% + * Copyright (C) 2012 - 2015 BigDataViewer authors + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package net.imglib2.cache.img; + +import java.lang.ref.SoftReference; + +import net.imglib2.Dirty; +import net.imglib2.cache.img.ReadOnlyCachedCellImgOptions.CacheType; + +/** + * Options base class used when constructing a cell image factory derived from {@link AbstractReadWriteCachedCellImgFactory}. + * This holds a set of default options shared by read-write-caches. Specialized writer-backends should provide their + * own CachedCellImgFactory and -Options. + * + * Must be subclassed, which usually also requires a subclass of Values that adds + * specific options depending on the writer-backend for the read-write-cache. + * + * @author Tobias Pietzsch + * @author Carsten Haubold, KNIME GmbH, Konstanz, Germany + */ +public abstract class AbstractReadWriteCachedCellImgOptions { + AbstractReadWriteCachedCellImgOptions() { + } + + /** + * @return The values of the (derived) options object + */ + public abstract Values values(); + + /** + * @param other Options that should be merged with this option object + * @return New {@link AbstractReadWriteCachedCellImgOptions} containing the options of this object, + * overwritten by the non-default settings in the provided other options object + */ + public abstract AbstractReadWriteCachedCellImgOptions merge(final AbstractReadWriteCachedCellImgOptions other); + + /** + * Specify whether the image should use {@link Dirty} accesses. Dirty accesses + * track whether cells were written to. Only cells that were written to are + * (potentially) cached to disk. + *

+ * This is {@code true} by default. + *

+ * + * @param dirty whether the image should use {@link Dirty} accesses. + */ + public abstract AbstractReadWriteCachedCellImgOptions dirtyAccesses(final boolean dirty); + + public abstract AbstractReadWriteCachedCellImgOptions volatileAccesses(final boolean volatil); + + /** + * The specified number of threads is started to handle asynchronous writing of + * values that are evicted from the memory cache. + * + * @param numIoThreads how many writer threads to start (default is 1). + */ + public abstract AbstractReadWriteCachedCellImgOptions numIoThreads(final int numIoThreads); + + /** + * Set the maximum size of the disk write queue. When the queue is full, + * removing entries from the cache will block until earlier values have been + * written. + *

+ * Because processing of removed entries is done whenever the cache is accessed, + * this may also block accesses to the cache. (This is a good thing, because it + * avoids running out of memory because entries cannot be cleared fast + * enough...) + *

+ * + * @param maxIoQueueSize the maximum size of the write queue (default is 10). + */ + public abstract AbstractReadWriteCachedCellImgOptions maxIoQueueSize(final int maxIoQueueSize); + + /** + * Which in-memory cache type to use. The options are + *
    + *
  • {@link CacheType#SOFTREF SOFTREF}: The cache keeps SoftReferences to + * values (cells), basically relying on GC for removal. The advantage of this is + * that many caches can be created without needing to put a limit on the size of + * any of them. GC will take care of balancing that. The downside is that + * {@link OutOfMemoryError} may occur because {@link SoftReference}s are cleared + * too slow. SoftReferences are not collected for a certain time after they have + * been used. If there is heavy thrashing with cells being constantly swapped in + * and out from disk then OutOfMemory may happen because of this. This sounds + * worse than it is in practice and should only happen in pathological + * situations. Tuning the {@code -XX:SoftRefLRUPolicyMSPerMB} JVM flag does + * often help.
    • + *
  • {@link CacheType#BOUNDED BOUNDED}: The cache keeps strong references to a + * limited number of values (cells). The advantage is that there is never + * OutOfMemory because of the issues described above (fingers crossed). The + * downside is that the number of cells that should be cached needs to be + * specified beforehand. So {@link OutOfMemoryError} may occur if many caches + * are opened and consume too much memory in total.
    • + *
+ * + * @param cacheType which cache type to use (default is {@code SOFTREF}). + */ + public abstract AbstractReadWriteCachedCellImgOptions cacheType(final CacheType cacheType); + + /** + * Set the maximum number of values (cells) to keep in the cache. This is only + * used if {@link #cacheType(CacheType)} is {@link CacheType#BOUNDED}. + * + * @param maxCacheSize maximum number of values in the cache (default is 1000). + */ + public abstract AbstractReadWriteCachedCellImgOptions maxCacheSize(final long maxCacheSize); + + /** + * Set the dimensions of a cell. This is extended or truncated as necessary. For + * example if {@code cellDimensions=[64,32]} then for creating a 3D image it + * will be augmented to {@code [64,32,32]}. For creating a 1D image it will be + * truncated to {@code [64]}. + * + * @param cellDimensions dimensions of a cell (default is 10). + */ + public abstract AbstractReadWriteCachedCellImgOptions cellDimensions(final int... cellDimensions); + + /** + * Specify whether cells initialized by a {@link CellLoader} should be marked as + * dirty. It is useful to set this to {@code true} if initialization is a costly + * operation. By this, it is made sure that cells are initialized only once, and + * then written and retrieve from the disk cache when they are next required. + *

+ * This is {@code false} by default. + *

+ *

+ * This option only has an effect for {@link DiskCachedCellImg} that are created + * with a {@link CellLoader} + * ({@link DiskCachedCellImgFactory#create(long[], net.imglib2.type.NativeType, CellLoader)}). + *

+ * + * @param initializeAsDirty whether cells initialized by a {@link CellLoader} + * should be marked as dirty. + */ + public abstract AbstractReadWriteCachedCellImgOptions initializeCellsAsDirty(final boolean initializeAsDirty); + + /** + * Read-only {@link AbstractReadWriteCachedCellImgOptions} values. + */ + protected static class Values extends ReadOnlyCachedCellImgOptions.Values { + /** + * Copy constructor. + */ + Values(final Values that) { + super(that); + if (!that.dirtyAccessesModified) { + this.dirtyAccesses = true; // RW default value differs from readonly cache + } + this.numIoThreads = that.numIoThreads; + this.numIoThreadsModified = that.numIoThreadsModified; + this.maxIoQueueSize = that.maxIoQueueSize; + this.maxIoQueueSizeModified = that.maxIoQueueSizeModified; + this.initializeCellsAsDirty = that.initializeCellsAsDirty; + this.initializeCellsAsDirtyModified = that.initializeCellsAsDirtyModified; + } + + Values() { + super(); + dirtyAccesses = true; // RW default value differs from readonly cache + } + + Values(final Values base, final Values aug) { + super(base, aug); + numIoThreads = aug.numIoThreadsModified ? aug.numIoThreads : base.numIoThreads; + maxIoQueueSize = aug.maxIoQueueSizeModified ? aug.maxIoQueueSize : base.maxIoQueueSize; + initializeCellsAsDirty = aug.initializeCellsAsDirtyModified ? aug.initializeCellsAsDirty + : base.initializeCellsAsDirty; + } + + protected int numIoThreads = 1; + + protected int maxIoQueueSize = 10; + + protected boolean initializeCellsAsDirty = false; + + public int numIoThreads() { + return numIoThreads; + } + + public int maxIoQueueSize() { + return maxIoQueueSize; + } + + public boolean initializeCellsAsDirty() { + return initializeCellsAsDirty; + } + + protected boolean numIoThreadsModified = false; + + protected boolean maxIoQueueSizeModified = false; + + protected boolean initializeCellsAsDirtyModified = false; + + Values setNumIoThreads(final int n) { + numIoThreads = n; + numIoThreadsModified = true; + return this; + } + + Values setMaxIoQueueSize(final int n) { + maxIoQueueSize = n; + maxIoQueueSizeModified = true; + return this; + } + + Values setInitializeCellsAsDirty(final boolean b) { + initializeCellsAsDirty = b; + initializeCellsAsDirtyModified = true; + return this; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(super.toString()); + sb.append("AbstractReadWriteCachedCellImgOptions = {"); + + sb.append("numIoThreads = "); + sb.append(numIoThreads); + if (numIoThreadsModified) + sb.append(" [m]"); + sb.append(", "); + + sb.append("maxIoQueueSize = "); + sb.append(maxIoQueueSize); + if (maxIoQueueSizeModified) + sb.append(" [m]"); + sb.append(", "); + + sb.append("initializeCellsAsDirty = "); + sb.append(Boolean.toString(initializeCellsAsDirty)); + if (initializeCellsAsDirtyModified) + sb.append(" [m]"); + + sb.append("}"); + + return sb.toString(); + } + + @Override + Values copy() { + return new Values(this); + } + } +} diff --git a/src/main/java/net/imglib2/cache/img/DiskCachedCellImgFactory.java b/src/main/java/net/imglib2/cache/img/DiskCachedCellImgFactory.java index 612d40e..bf09471 100644 --- a/src/main/java/net/imglib2/cache/img/DiskCachedCellImgFactory.java +++ b/src/main/java/net/imglib2/cache/img/DiskCachedCellImgFactory.java @@ -41,18 +41,11 @@ import net.imglib2.Dimensions; import net.imglib2.cache.Cache; import net.imglib2.cache.CacheLoader; -import net.imglib2.cache.IoSync; -import net.imglib2.cache.LoaderRemoverCache; -import net.imglib2.cache.ref.GuardedStrongRefLoaderRemoverCache; -import net.imglib2.cache.ref.SoftRefLoaderRemoverCache; import net.imglib2.exception.IncompatibleTypeException; import net.imglib2.img.ImgFactory; -import net.imglib2.img.NativeImgFactory; -import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory; import net.imglib2.img.basictypeaccess.array.ArrayDataAccess; import net.imglib2.img.cell.Cell; import net.imglib2.img.cell.CellGrid; -import net.imglib2.img.cell.CellImgFactory; import net.imglib2.type.NativeType; import net.imglib2.type.NativeTypeFactory; import net.imglib2.util.Fraction; @@ -65,9 +58,9 @@ * defaults. * * @author Tobias Pietzsch + * @author Carsten Haubold, KNIME GmbH, Konstanz, Germany */ -public class DiskCachedCellImgFactory< T extends NativeType< T > > extends NativeImgFactory< T > -{ +public class DiskCachedCellImgFactory> extends AbstractReadWriteCachedCellImgFactory { private DiskCachedCellImgOptions factoryOptions; /** @@ -175,111 +168,25 @@ public < A > DiskCachedCellImg< T, A > createWithCacheLoader( final Dimensions d * additional options that partially override general factory * options, or {@code null}. */ - private < A > DiskCachedCellImg< T, A > create( - final long[] dimensions, - final CacheLoader< Long, ? extends Cell< ? extends A > > cacheLoader, - final CellLoader< T > cellLoader, - final T type, - final DiskCachedCellImgOptions additionalOptions ) - { - @SuppressWarnings( { "unchecked", "rawtypes" } ) - final DiskCachedCellImg< T, A > img = create( dimensions, cacheLoader, cellLoader, type, ( NativeTypeFactory ) type.getNativeTypeFactory(), additionalOptions ); - return img; - } - - private < A extends ArrayDataAccess< A > > DiskCachedCellImg< T, ? extends A > create( - final long[] dimensions, - final CacheLoader< Long, ? extends Cell< ? > > cacheLoader, - final CellLoader< T > cellLoader, - final T type, - final NativeTypeFactory< T, A > typeFactory, - final DiskCachedCellImgOptions additionalOptions ) - { - final DiskCachedCellImgOptions.Values options = ( additionalOptions == null ) - ? factoryOptions.values - : new DiskCachedCellImgOptions.Values( factoryOptions.values, additionalOptions.values ); - - - final Fraction entitiesPerPixel = type.getEntitiesPerPixel(); - - final CellGrid grid = createCellGrid( dimensions, entitiesPerPixel, options ); - - @SuppressWarnings( "unchecked" ) - CacheLoader< Long, Cell< A > > backingLoader = ( CacheLoader< Long, Cell< A > > ) cacheLoader; - if ( backingLoader == null ) - { - if ( cellLoader != null ) - { - final CellLoader< T > actualCellLoader = options.initializeCellsAsDirty() - ? cell -> { - cellLoader.load( cell ); - cell.setDirty(); - } - : cellLoader; - backingLoader = LoadedCellCacheLoader.get( grid, actualCellLoader, type, options.accessFlags() ); - } - else - backingLoader = EmptyCellCacheLoader.get( grid, type, options.accessFlags() ); - } - - final Path blockcache = createBlockCachePath( options ); - - @SuppressWarnings( { "rawtypes", "unchecked" } ) - final DiskCellCache< A > diskcache = options.dirtyAccesses() - ? new DirtyDiskCellCache( - blockcache, grid, backingLoader, - AccessIo.get( type, options.accessFlags() ), - entitiesPerPixel ) - : new DiskCellCache<>( - blockcache, grid, backingLoader, - AccessIo.get( type, options.accessFlags() ), - entitiesPerPixel ); - - final IoSync< Long, Cell< A > > iosync = new IoSync<>( - diskcache, - options.numIoThreads(), - options.maxIoQueueSize() ); - - LoaderRemoverCache< Long, Cell< A > > listenableCache; - switch ( options.cacheType() ) - { - case BOUNDED: - listenableCache = new GuardedStrongRefLoaderRemoverCache<>( options.maxCacheSize() ); - break; - case SOFTREF: - default: - listenableCache = new SoftRefLoaderRemoverCache<>(); - break; - } - - final Cache< Long, Cell< A > > cache = listenableCache - .withRemover( iosync ) - .withLoader( iosync ); - - final A accessType = ArrayDataAccessFactory.get( typeFactory, options.accessFlags() ); - final DiskCachedCellImg< T, ? extends A > img = new DiskCachedCellImg<>( this, grid, entitiesPerPixel, cache, accessType ); - img.setLinkedType( typeFactory.createLinkedType( img ) ); + private DiskCachedCellImg create(final long[] dimensions, + final CacheLoader> cacheLoader, final CellLoader cellLoader, + final T type, final DiskCachedCellImgOptions additionalOptions) { + @SuppressWarnings({ "unchecked", "rawtypes" }) + final DiskCachedCellImg img = (DiskCachedCellImg)create(dimensions, cacheLoader, cellLoader, type, + (NativeTypeFactory) type.getNativeTypeFactory(), additionalOptions); return img; } @SuppressWarnings( { "unchecked", "rawtypes" } ) @Override - public < S > ImgFactory< S > imgFactory( final S type ) throws IncompatibleTypeException - { - if ( NativeType.class.isInstance( type ) ) - return new DiskCachedCellImgFactory( factoryOptions ); - throw new IncompatibleTypeException( this, type.getClass().getCanonicalName() + " does not implement NativeType." ); + public ImgFactory imgFactory(final S type) throws IncompatibleTypeException { + if (NativeType.class.isInstance(type)) + return new DiskCachedCellImgFactory(factoryOptions); + throw new IncompatibleTypeException(this, + type.getClass().getCanonicalName() + " does not implement NativeType."); } - private CellGrid createCellGrid( final long[] dimensions, final Fraction entitiesPerPixel, final DiskCachedCellImgOptions.Values options ) - { - CellImgFactory.verifyDimensions( dimensions ); - final int n = dimensions.length; - final int[] cellDimensions = CellImgFactory.getCellDimensions( options.cellDimensions(), n, entitiesPerPixel ); - return new CellGrid( dimensions, cellDimensions ); - } - - private Path createBlockCachePath( final DiskCachedCellImgOptions.Values options ) + static Path createBlockCachePath( final DiskCachedCellImgOptions.Values options ) { try { @@ -308,6 +215,46 @@ else if ( dir != null ) } } + @Override + AbstractReadWriteCachedCellImgOptions mergeWithFactoryOptions( + AbstractReadWriteCachedCellImgOptions userProvidedOptions) { + return ( userProvidedOptions == null ) ? factoryOptions : factoryOptions.merge(userProvidedOptions); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Override + protected > ReadWriteCellCache createCellCache( + AbstractReadWriteCachedCellImgOptions options, + CellGrid grid, + CacheLoader> backingLoader, + T type, + Fraction entitiesPerPixel) { + DiskCachedCellImgOptions.Values diskCacheOptions; + if (options instanceof DiskCachedCellImgOptions) { + diskCacheOptions = ((DiskCachedCellImgOptions)options).values(); + } else { + // If the given options are no DiskCachedCellImgOptions, we create default options for + // the disk cache specifics, and merge them with the provided values. + diskCacheOptions = DiskCachedCellImgOptions.options().merge(options).values(); + } + + final Path blockcache = createBlockCachePath(diskCacheOptions); + return diskCacheOptions.dirtyAccesses() + ? (ReadWriteCellCache)new DirtyDiskCellCache(blockcache, grid, backingLoader, AccessIo.get(type, diskCacheOptions.accessFlags()), + entitiesPerPixel) + : new DiskCellCache<>(blockcache, grid, backingLoader, AccessIo.get(type, diskCacheOptions.accessFlags()), + entitiesPerPixel); + } + + @Override + protected > CachedCellImg createCachedCellImg( + final CellGrid grid, + final Fraction entitiesPerPixel, + final Cache> cache, + final A accessType + ) { + return new DiskCachedCellImg<>(this, grid, entitiesPerPixel, cache, accessType); + } /* * ----------------------------------------------------------------------- diff --git a/src/main/java/net/imglib2/cache/img/DiskCachedCellImgOptions.java b/src/main/java/net/imglib2/cache/img/DiskCachedCellImgOptions.java index a73ed56..90b4e83 100644 --- a/src/main/java/net/imglib2/cache/img/DiskCachedCellImgOptions.java +++ b/src/main/java/net/imglib2/cache/img/DiskCachedCellImgOptions.java @@ -30,24 +30,24 @@ import java.lang.ref.SoftReference; import java.nio.file.Path; -import java.util.Set; import net.imglib2.Dirty; -import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.cache.img.ReadOnlyCachedCellImgOptions.CacheType; import net.imglib2.img.cell.CellImgFactory; -import net.imglib2.util.Util; /** * Optional parameters for constructing a {@link DiskCachedCellImgFactory}. * * @author Tobias Pietzsch + * @author Carsten Haubold, KNIME GmbH, Konstanz, Germany */ -public class DiskCachedCellImgOptions +public class DiskCachedCellImgOptions extends AbstractReadWriteCachedCellImgOptions { - public final Values values; + private final Values values; - DiskCachedCellImgOptions( final Values values ) + DiskCachedCellImgOptions(final Values values) { + super(); this.values = values; } @@ -56,6 +56,16 @@ public DiskCachedCellImgOptions() this( new Values() ); } + @Override + public Values values() { + return values; + } + + @Override + public DiskCachedCellImgOptions merge(final AbstractReadWriteCachedCellImgOptions other) { + return new DiskCachedCellImgOptions(new Values(values, other.values())); + } + /** * Create default {@link DiskCachedCellImgOptions}. * @@ -115,40 +125,7 @@ public DiskCachedCellImgOptions numIoThreads( final int numIoThreads ) */ public DiskCachedCellImgOptions maxIoQueueSize( final int maxIoQueueSize ) { - return new DiskCachedCellImgOptions( values.copy().setMaxIoQueueSize( maxIoQueueSize ) ); - } - - /** - * Rough in-memory cache types. - * - * @author Tobias Pietzsch - */ - public static enum CacheType - { - /** - * The cache keeps SoftReferences to values (cells), basically relying - * on GC for removal. The advantage of this is that many caches can be - * created without needing to put a limit on the size of any of them. GC - * will take care of balancing that. The downside is that - * {@link OutOfMemoryError} may occur because {@link SoftReference}s are - * cleared too slow. SoftReferences are not collected for a certain time - * after they have been used. If there is heavy thrashing with cells - * being constantly swapped in and out from disk then OutOfMemory may - * happen because of this. This sounds worse than it is in practice and - * should only happen in pathological situations. Tuning the - * {@code -XX:SoftRefLRUPolicyMSPerMB} JVM flag does often help. - */ - SOFTREF, - - /** - * The cache keeps strong references to a limited number of values - * (cells). The advantage is that there is never OutOfMemory because of - * the issues described above (fingers crossed). The downside is that - * the number of cells that should be cached needs to be specified - * beforehand. So {@link OutOfMemoryError} may occur if many caches are - * opened and consume too much memory in total. - */ - BOUNDED + return new DiskCachedCellImgOptions(values.copy().setMaxIoQueueSize( maxIoQueueSize )); } /** @@ -284,7 +261,7 @@ public DiskCachedCellImgOptions tempDirectoryPrefix( final String prefix ) */ public DiskCachedCellImgOptions deleteCacheDirectoryOnExit( final boolean deleteOnExit ) { - return new DiskCachedCellImgOptions( values.copy().setDeleteCacheDirectoryOnExit( deleteOnExit ) ); + return new DiskCachedCellImgOptions(values.copy().setDeleteCacheDirectoryOnExit( deleteOnExit )); } /** @@ -298,8 +275,7 @@ public DiskCachedCellImgOptions deleteCacheDirectoryOnExit( final boolean delete *

*

* This option only has an effect for {@link DiskCachedCellImg} that are - * created with a {@link CellLoader} - * ({@link DiskCachedCellImgFactory#create(long[], net.imglib2.type.NativeType, CellLoader)}). + * created with a {@link CellLoader} ({@link DiskCachedCellImgFactory#create(net.imglib2.Dimensions, CellLoader)}}) *

* * @param initializeAsDirty @@ -308,33 +284,20 @@ public DiskCachedCellImgOptions deleteCacheDirectoryOnExit( final boolean delete */ public DiskCachedCellImgOptions initializeCellsAsDirty( final boolean initializeAsDirty ) { - return new DiskCachedCellImgOptions( values.copy().setInitializeCellsAsDirty( initializeAsDirty ) ); + return new DiskCachedCellImgOptions(values.copy().setInitializeCellsAsDirty( initializeAsDirty )); } /** * Read-only {@link DiskCachedCellImgOptions} values. */ - public static class Values + protected static class Values extends AbstractReadWriteCachedCellImgOptions.Values { /** * Copy constructor. */ Values( final Values that ) { - this.dirtyAccesses = that.dirtyAccesses; - this.dirtyAccessesModified = that.dirtyAccessesModified; - this.volatileAccesses = that.volatileAccesses; - this.volatileAccessesModified = that.volatileAccessesModified; - this.numIoThreads = that.numIoThreads; - this.numIoThreadsModified = that.numIoThreadsModified; - this.maxIoQueueSize = that.maxIoQueueSize; - this.maxIoQueueSizeModified = that.maxIoQueueSizeModified; - this.cacheType = that.cacheType; - this.cacheTypeModified = that.cacheTypeModified; - this.maxCacheSize = that.maxCacheSize; - this.maxCacheSizeModified = that.maxCacheSizeModified; - this.cellDimensions = that.cellDimensions; - this.cellDimensionsModified = that.cellDimensionsModified; + super(that); this.cacheDirectory = that.cacheDirectory; this.cacheDirectoryModified = that.cacheDirectoryModified; this.tempDirectory = that.tempDirectory; @@ -343,8 +306,6 @@ public static class Values this.tempDirectoryPrefixModified = that.tempDirectoryPrefixModified; this.deleteCacheDirectoryOnExit = that.deleteCacheDirectoryOnExit; this.deleteCacheDirectoryOnExitModified = that.deleteCacheDirectoryOnExitModified; - this.initializeCellsAsDirty = that.initializeCellsAsDirty; - this.initializeCellsAsDirtyModified = that.initializeCellsAsDirtyModified; } Values() @@ -352,27 +313,7 @@ public static class Values Values( final Values base, final Values aug ) { - dirtyAccesses = aug.dirtyAccessesModified - ? aug.dirtyAccesses - : base.dirtyAccesses; - volatileAccesses = aug.volatileAccessesModified - ? aug.volatileAccesses - : base.volatileAccesses; - numIoThreads = aug.numIoThreadsModified - ? aug.numIoThreads - : base.numIoThreads; - maxIoQueueSize = aug.maxIoQueueSizeModified - ? aug.maxIoQueueSize - : base.maxIoQueueSize; - cacheType = aug.cacheTypeModified - ? aug.cacheType - : base.cacheType; - maxCacheSize = aug.maxCacheSizeModified - ? aug.maxCacheSize - : base.maxCacheSize; - cellDimensions = aug.cellDimensionsModified - ? aug.cellDimensions - : base.cellDimensions; + super(base, aug); cacheDirectory = aug.cacheDirectoryModified ? aug.cacheDirectory : base.cacheDirectory; @@ -385,30 +326,17 @@ public static class Values deleteCacheDirectoryOnExit = aug.deleteCacheDirectoryOnExitModified ? aug.deleteCacheDirectoryOnExit : base.deleteCacheDirectoryOnExit; - initializeCellsAsDirty = aug.initializeCellsAsDirtyModified - ? aug.initializeCellsAsDirty - : base.initializeCellsAsDirty; } - public DiskCachedCellImgOptions optionsFromValues() + Values( final Values base, final AbstractReadWriteCachedCellImgOptions.Values aug ) { - return new DiskCachedCellImgOptions( new Values( this ) ); + super(base, aug); + cacheDirectory = base.cacheDirectory; + tempDirectory = base.tempDirectory; + tempDirectoryPrefix = base.tempDirectoryPrefix; + deleteCacheDirectoryOnExit = base.deleteCacheDirectoryOnExit; } - private boolean dirtyAccesses = true; - - private boolean volatileAccesses = true; - - private int numIoThreads = 1; - - private int maxIoQueueSize = 10; - - private CacheType cacheType = CacheType.SOFTREF; - - private long maxCacheSize = 1000; - - private int[] cellDimensions = new int[] { 10 }; - private Path cacheDirectory = null; private Path tempDirectory = null; @@ -417,48 +345,6 @@ public DiskCachedCellImgOptions optionsFromValues() private boolean deleteCacheDirectoryOnExit = true; - private boolean initializeCellsAsDirty = false; - - public boolean dirtyAccesses() - { - return dirtyAccesses; - } - - public boolean volatileAccesses() - { - return volatileAccesses; - } - - public Set< AccessFlags > accessFlags() - { - return AccessFlags.fromBooleansDirtyVolatile( dirtyAccesses, volatileAccesses ); - } - - public int numIoThreads() - { - return numIoThreads; - } - - public int maxIoQueueSize() - { - return maxIoQueueSize; - } - - public CacheType cacheType() - { - return cacheType; - } - - public long maxCacheSize() - { - return maxCacheSize; - } - - public int[] cellDimensions() - { - return cellDimensions; - } - public Path cacheDirectory() { return cacheDirectory; @@ -479,25 +365,6 @@ public boolean deleteCacheDirectoryOnExit() return deleteCacheDirectoryOnExit; } - public boolean initializeCellsAsDirty() - { - return initializeCellsAsDirty; - } - - private boolean dirtyAccessesModified = false; - - private boolean volatileAccessesModified = false; - - private boolean numIoThreadsModified = false; - - private boolean maxIoQueueSizeModified = false; - - private boolean cacheTypeModified = false; - - private boolean maxCacheSizeModified = false; - - private boolean cellDimensionsModified = false; - private boolean cacheDirectoryModified = false; private boolean tempDirectoryModified = false; @@ -506,145 +373,93 @@ public boolean initializeCellsAsDirty() private boolean deleteCacheDirectoryOnExitModified = false; - private boolean initializeCellsAsDirtyModified = false; - - Values setDirtyAccesses( final boolean b ) + Values setCacheDirectory( final Path dir ) { - dirtyAccesses = b; - dirtyAccessesModified = true; + cacheDirectory = dir; + cacheDirectoryModified = true; return this; } - Values setVolatileAccesses( final boolean b ) + Values setTempDirectory( final Path dir ) { - volatileAccesses = b; - volatileAccessesModified = true; + tempDirectory = dir; + tempDirectoryModified = true; return this; } - Values setNumIoThreads( final int n ) + Values setTempDirectoryPrefix( final String prefix ) { - numIoThreads = n; - numIoThreadsModified = true; + tempDirectoryPrefix = prefix; + tempDirectoryPrefixModified = true; return this; } - Values setMaxIoQueueSize( final int n ) + Values setDeleteCacheDirectoryOnExit( final boolean b ) { - maxIoQueueSize = n; - maxIoQueueSizeModified = true; + deleteCacheDirectoryOnExit = b; + deleteCacheDirectoryOnExitModified = true; return this; } - Values setCacheType( final CacheType t ) + @Override + Values setDirtyAccesses( final boolean b ) { - cacheType = t; - cacheTypeModified = true; + super.setDirtyAccesses(b); return this; } - Values setMaxCacheSize( final long n ) + @Override + Values setVolatileAccesses( final boolean b ) { - maxCacheSize = n; - maxCacheSizeModified = true; + super.setVolatileAccesses(b); return this; } - Values setCellDimensions( final int[] dims ) + @Override + Values setCacheType( final CacheType t ) { - cellDimensions = dims; - cellDimensionsModified = true; + super.setCacheType(t); return this; } - Values setCacheDirectory( final Path dir ) + @Override + Values setMaxCacheSize( final long n ) { - cacheDirectory = dir; - cacheDirectoryModified = true; + super.setMaxCacheSize(n); return this; } - Values setTempDirectory( final Path dir ) + @Override + Values setCellDimensions( final int[] dims ) { - tempDirectory = dir; - tempDirectoryModified = true; + super.setCellDimensions(dims); return this; } - Values setTempDirectoryPrefix( final String prefix ) - { - tempDirectoryPrefix = prefix; - tempDirectoryPrefixModified = true; + @Override + Values setNumIoThreads(final int n) { + super.setNumIoThreads(n); return this; } - Values setDeleteCacheDirectoryOnExit( final boolean b ) - { - deleteCacheDirectoryOnExit = b; - deleteCacheDirectoryOnExitModified = true; + @Override + Values setMaxIoQueueSize(final int n) { + super.setMaxIoQueueSize(n); return this; } - Values setInitializeCellsAsDirty( final boolean b ) - { - initializeCellsAsDirty = b; - initializeCellsAsDirtyModified = true; + @Override + Values setInitializeCellsAsDirty(final boolean b) { + super.setInitializeCellsAsDirty(b); return this; } - Values copy() - { - return new Values( this ); - } - @Override public String toString() { final StringBuilder sb = new StringBuilder(); - - sb.append( "{" ); - - sb.append( "dirtyAccesses = " ); - sb.append( Boolean.toString( dirtyAccesses ) ); - if ( dirtyAccessesModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "volatileAccesses = " ); - sb.append( Boolean.toString( volatileAccesses ) ); - if ( volatileAccessesModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "numIoThreads = " ); - sb.append( numIoThreads ); - if ( numIoThreadsModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "maxIoQueueSize = " ); - sb.append( maxIoQueueSize ); - if ( maxIoQueueSizeModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "cacheType = " ); - sb.append( cacheType ); - if ( cacheTypeModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "maxCacheSize = " ); - sb.append( maxCacheSize ); - if ( maxCacheSizeModified ) - sb.append( " [m]" ); - sb.append( ", " ); - - sb.append( "cellDimensions = " ); - sb.append( Util.printCoordinates( cellDimensions ) ); - if ( cellDimensionsModified ) - sb.append( " [m]" ); - sb.append( ", " ); + sb.append(super.toString()); + sb.append( "DiskCachedCellImgOptions = {" ); sb.append( "cacheDirectory = " ); sb.append( cacheDirectory ); @@ -669,14 +484,14 @@ public String toString() if ( deleteCacheDirectoryOnExitModified ) sb.append( " [m]" ); - sb.append( "initializeCellsAsDirty = " ); - sb.append( Boolean.toString( initializeCellsAsDirty ) ); - if ( initializeCellsAsDirtyModified ) - sb.append( " [m]" ); - sb.append( "}" ); return sb.toString(); } + + @Override + Values copy() { + return new Values(this); + } } } diff --git a/src/main/java/net/imglib2/cache/img/DiskCellCache.java b/src/main/java/net/imglib2/cache/img/DiskCellCache.java index f8b4de5..a82ac0e 100644 --- a/src/main/java/net/imglib2/cache/img/DiskCellCache.java +++ b/src/main/java/net/imglib2/cache/img/DiskCellCache.java @@ -40,8 +40,7 @@ * * @author Tobias Pietzsch */ -public class DiskCellCache< A > implements CacheRemover< Long, Cell< A > >, CacheLoader< Long, Cell< A > > -{ +public class DiskCellCache implements ReadWriteCellCache { private final Path blockcache; private final CellGrid grid; diff --git a/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgFactory.java b/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgFactory.java index cd7e9d3..ef5ce4e 100644 --- a/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgFactory.java +++ b/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgFactory.java @@ -123,12 +123,12 @@ public < T extends NativeType< T >, A > CachedCellImg< T, A > createWithCacheLoa final ReadOnlyCachedCellImgOptions additionalOptions ) { final ReadOnlyCachedCellImgOptions.Values options = ( additionalOptions == null ) - ? factoryOptions.values - : new ReadOnlyCachedCellImgOptions.Values( factoryOptions.values, additionalOptions.values ); + ? factoryOptions.values() + : factoryOptions.merge(additionalOptions).values(); final PrimitiveType primitiveType = type.getNativeTypeFactory().getPrimitiveType(); final Fraction entitiesPerPixel = type.getEntitiesPerPixel(); - final CellGrid grid = createCellGrid( dimensions, entitiesPerPixel, options ); + final CellGrid grid = createCellGrid( dimensions, options.cellDimensions(), entitiesPerPixel ); final A accessType = ArrayDataAccessFactory.get( primitiveType, options.accessFlags() ); @SuppressWarnings( "unchecked" ) @@ -152,11 +152,21 @@ public < T extends NativeType< T >, A > CachedCellImg< T, A > createWithCacheLoa return new CachedCellImg<>( grid, type, cache, accessType ); } - private CellGrid createCellGrid( final long[] dimensions, final Fraction entitiesPerPixel, final ReadOnlyCachedCellImgOptions.Values options ) + /** + * Create a {@link CellGrid} with the given image dimensions, desired cell dimensions, and entitiesPerPixel + * @param dimensions image dimensions + * @param defaultCellDimensions desired cell dimensions + * @param entitiesPerPixel + * @return + */ + static CellGrid createCellGrid( + final long[] dimensions, + final int[] defaultCellDimensions, + final Fraction entitiesPerPixel) { - CellImgFactory.verifyDimensions( dimensions ); + CellImgFactory.verifyDimensions(dimensions); final int n = dimensions.length; - final int[] cellDimensions = CellImgFactory.getCellDimensions( options.cellDimensions(), n, entitiesPerPixel ); - return new CellGrid( dimensions, cellDimensions ); + final int[] cellDimensions = CellImgFactory.getCellDimensions(defaultCellDimensions, n, entitiesPerPixel); + return new CellGrid(dimensions, cellDimensions); } } diff --git a/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgOptions.java b/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgOptions.java index d4b10bf..46e219e 100644 --- a/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgOptions.java +++ b/src/main/java/net/imglib2/cache/img/ReadOnlyCachedCellImgOptions.java @@ -32,7 +32,6 @@ import java.util.Set; import net.imglib2.Dirty; -import net.imglib2.cache.img.DiskCachedCellImgOptions.CacheType; import net.imglib2.img.basictypeaccess.AccessFlags; import net.imglib2.img.cell.CellImgFactory; import net.imglib2.util.Util; @@ -44,7 +43,7 @@ */ public class ReadOnlyCachedCellImgOptions { - public final Values values; + private final Values values; ReadOnlyCachedCellImgOptions( final Values values ) { @@ -66,6 +65,22 @@ public static ReadOnlyCachedCellImgOptions options() return new ReadOnlyCachedCellImgOptions(); } + /** + * @return The values of this options object + */ + public Values values() { + return values; + } + + /** + * @param other Options that should be merged with this option object + * @return New {@link ReadOnlyCachedCellImgOptions} containing the options of this object, + * overwritten by the non-default settings in the provided other options object + */ + public ReadOnlyCachedCellImgOptions merge(final ReadOnlyCachedCellImgOptions other) { + return new ReadOnlyCachedCellImgOptions(new Values(values, other.values)); + } + /** * Specify whether the image should use {@link Dirty} accesses. Dirty * accesses track whether cells were written to. @@ -86,6 +101,39 @@ public ReadOnlyCachedCellImgOptions volatileAccesses( final boolean volatil ) return new ReadOnlyCachedCellImgOptions( values.copy().setVolatileAccesses( volatil ) ); } + /** + * Rough in-memory cache types. + * + * @author Tobias Pietzsch + */ + public static enum CacheType + { + /** + * The cache keeps SoftReferences to values (cells), basically relying + * on GC for removal. The advantage of this is that many caches can be + * created without needing to put a limit on the size of any of them. GC + * will take care of balancing that. The downside is that + * {@link OutOfMemoryError} may occur because {@link SoftReference}s are + * cleared too slow. SoftReferences are not collected for a certain time + * after they have been used. If there is heavy thrashing with cells + * being constantly swapped in and out from disk then OutOfMemory may + * happen because of this. This sounds worse than it is in practice and + * should only happen in pathological situations. Tuning the + * {@code -XX:SoftRefLRUPolicyMSPerMB} JVM flag does often help. + */ + SOFTREF, + + /** + * The cache keeps strong references to a limited number of values + * (cells). The advantage is that there is never OutOfMemory because of + * the issues described above (fingers crossed). The downside is that + * the number of cells that should be cached needs to be specified + * beforehand. So {@link OutOfMemoryError} may occur if many caches are + * opened and consume too much memory in total. + */ + BOUNDED + } + /** * Which in-memory cache type to use. The options are *