T getAttribute(\n\t\t\tfinal String pathName,\n\t\t\tfinal String key,\n\t\t\tfinal Type type) throws N5Exception {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String normalizedAttributePath = N5URI.normalizeAttributePath(key);\n\t\tJsonElement attributes;\n\t\tif (cacheMeta()) {\n\t\t\tattributes = getCache().getAttributes(normalPathName, getAttributesKey());\n\t\t} else {\n\t\t\tattributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);\n\t\t}\n\t\ttry {\n\t\t\treturn GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson());\n\t\t} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {\n\t\t\tthrow new N5Exception.N5ClassCastException(e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault boolean exists(final String pathName) {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tif (cacheMeta())\n\t\t\treturn getCache().isGroup(normalPathName, getAttributesKey());\n\t\telse {\n\t\t\treturn existsFromContainer(normalPathName, null);\n\t\t}\n\t}\n\n\t@Override\n\tdefault boolean existsFromContainer(final String normalPathName, final String normalCacheKey) {\n\n\t\tfinal KeyValueAccess kva = getKeyValueAccess();\n\t\tif (normalCacheKey == null)\n\t\t\treturn kva.isDirectory(kva.compose(getURI(), normalPathName));\n\t\telse\n\t\t\treturn kva.isFile(kva.compose(getURI(), normalPathName, normalCacheKey));\n\t}\n\n\t@Override\n\tdefault boolean groupExists(final String pathName) {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tif (cacheMeta())\n\t\t\treturn getCache().isGroup(normalPathName, null);\n\t\telse {\n\t\t\treturn isGroupFromContainer(normalPathName);\n\t\t}\n\t}\n\n\t@Override\n\tdefault boolean isGroupFromContainer(final String normalPathName) {\n\n\t\treturn GsonKeyValueN5Reader.super.groupExists(normalPathName);\n\t}\n\n\t@Override\n\tdefault boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) {\n\n\t\treturn true;\n\t}\n\n\t@Override\n\tdefault boolean datasetExists(final String pathName) throws N5IOException {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tif (cacheMeta()) {\n\t\t\treturn getCache().isDataset(normalPathName, getAttributesKey());\n\t\t}\n\t\treturn isDatasetFromContainer(normalPathName);\n\t}\n\n\t@Override\n\tdefault boolean isDatasetFromContainer(final String normalPathName) throws N5IOException {\n\n\t\treturn normalGetDatasetAttributes(normalPathName) != null;\n\t}\n\n\t@Override\n\tdefault boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) {\n\n\t\treturn isGroupFromAttributes(normalCacheKey, attributes) && createDatasetAttributes(attributes) != null;\n\t}\n\n\t/**\n\t * Reads or creates the attributes map of a group or dataset.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return the attribute\n\t * @throws N5IOException if an IO error occurs while reading the attribute\n\t */\n\t@Override\n\tdefault JsonElement getAttributes(final String pathName) throws N5IOException {\n\n\t\tfinal String groupPath = N5URI.normalizeGroupPath(pathName);\n\n\t\t/* If cached, return the cache */\n\t\tif (cacheMeta()) {\n\t\t\treturn getCache().getAttributes(groupPath, getAttributesKey());\n\t\t} else {\n\t\t\treturn GsonKeyValueN5Reader.super.getAttributes(groupPath);\n\t\t}\n\t}\n\n\t@Override\n\tdefault String[] list(final String pathName) throws N5IOException {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\tif (cacheMeta()) {\n\t\t\treturn getCache().list(normalPath);\n\t\t} else {\n\t\t\treturn GsonKeyValueN5Reader.super.list(normalPath);\n\t\t}\n\t}\n\n\t@Override\n\tdefault String[] listFromContainer(final String normalPathName) {\n\n\t\t// this implementation doesn't use cache key, but rather depends on\n\t\treturn GsonKeyValueN5Reader.super.list(normalPathName);\n\t}\n\n\t/**\n\t * Check for attributes that are required for a group to be a dataset.\n\t *\n\t * @param attributes\n\t * to check for dataset attributes\n\t * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and\n\t * {@link DatasetAttributes#DATA_TYPE_KEY} are present\n\t */\n\tstatic boolean hasDatasetAttributes(final JsonElement attributes) {\n\n\t\tif (attributes == null || !attributes.isJsonObject()) {\n\t\t\treturn false;\n\t\t}\n\n\t\tfinal JsonObject metadataCache = attributes.getAsJsonObject();\n\t\treturn metadataCache.has(DatasetAttributes.DIMENSIONS_KEY)\n\t\t\t\t&& metadataCache.has(DatasetAttributes.DATA_TYPE_KEY);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\n\n/**\n * Cached default implementation of {@link N5Writer} with JSON attributes parsed\n * with {@link Gson}.\n */\npublic interface CachedGsonKeyValueN5Writer extends CachedGsonKeyValueN5Reader, GsonKeyValueN5Writer {\n\n\t@Override\n\tdefault void setVersion(final String path) throws N5Exception {\n\n\t\tfinal Version version = getVersion();\n\t\tif (!VERSION.isCompatible(version))\n\t\t\tthrow new N5IOException(\"Incompatible version \" + version + \" (this is \" + VERSION + \").\");\n\n\t\tif (!VERSION.equals(version))\n\t\t\tsetAttribute(\"/\", VERSION_KEY, VERSION.toString());;\n\t}\n\n\t@Override\n\tdefault void createGroup(final String path) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\t// avoid hitting the backend if this path is already a group according to the cache\n\t\t// else if exists is true (then a dataset is present) so throw an exception to avoid\n\t\t// overwriting / invalidating existing data\n\t\tif (groupExists(normalPath))\n\t\t\treturn;\n\t\telse if (datasetExists(normalPath))\n\t\t\tthrow new N5Exception(\"Can't make a group on existing dataset.\");\n\n\t\tgetKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));\n\n\t\tif (cacheMeta()) {\n\t\t\t// check all nodes that are parents of the added node, if they have\n\t\t\t// a children set, add the new child to it\n\t\t\tgetKeyValueAccess().parent(normalPath);\n\t\t\tString[] pathParts = getKeyValueAccess().components(normalPath);\n\t\t\tString parent = N5URI.normalizeGroupPath(\"/\");\n\t\t\tif (pathParts.length == 0) {\n\t\t\t\tpathParts = new String[]{\"\"};\n\t\t\t}\n\t\t\tfor (final String child : pathParts) {\n\n\t\t\t\tfinal String childPath = parent.isEmpty() ? child : parent + \"/\" + child;\n\t\t\t\tgetCache().initializeNonemptyCache(childPath, getAttributesKey());\n\t\t\t\tgetCache().updateCacheInfo(childPath, getAttributesKey());\n\n\t\t\t\t// only add if the parent exists and has children cached already\n\t\t\t\tif (parent != null && !child.isEmpty())\n\t\t\t\t\tgetCache().addChildIfPresent(parent, child);\n\n\t\t\t\tparent = childPath;\n\t\t\t}\n\t\t}\n\t}\n\n\t@Override\n\tdefault void writeAttributes(\n\t\t\tfinal String normalGroupPath,\n\t\t\tfinal JsonElement attributes) throws N5Exception {\n\n\t\twriteAndCacheAttributes(normalGroupPath, attributes);\n\t}\n\n\tdefault void writeAndCacheAttributes(\n\t\t\tfinal String normalGroupPath,\n\t\t\tfinal JsonElement attributes) throws N5Exception {\n\n\t\tGsonKeyValueN5Writer.super.writeAttributes(normalGroupPath, attributes);\n\n\t\tif (cacheMeta()) {\n\t\t\tJsonElement nullRespectingAttributes = attributes;\n\t\t\t/*\n\t\t\t * Gson only filters out nulls when you write the JsonElement. This\n\t\t\t * means it doesn't filter them out when caching.\n\t\t\t * To handle this, we explicitly writer the existing JsonElement to\n\t\t\t * a new JsonElement.\n\t\t\t * The output is identical to the input if:\n\t\t\t * - serializeNulls is true\n\t\t\t * - no null values are present\n\t\t\t * - caching is turned off\n\t\t\t */\n\t\t\tif (!getGson().serializeNulls()) {\n\t\t\t\tnullRespectingAttributes = getGson().toJsonTree(attributes);\n\t\t\t}\n\t\t\t/* Update the cache, and write to the writer */\n\t\t\tgetCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes);\n\t\t}\n\t}\n\n\t@Override\n\tdefault boolean remove(final String path) throws N5Exception {\n\n\t\t// GsonKeyValueN5Writer.super.remove(path)\n\t\t/*\n\t\t * the lines below duplicate the single line above but would have to call\n\t\t * normalizeGroupPath again the below duplicates code, but avoids extra work\n\t\t */\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\tfinal String groupPath = absoluteGroupPath(normalPath);\n\n\t\tif (getKeyValueAccess().isDirectory(groupPath))\n\t\t\tgetKeyValueAccess().delete(groupPath);\n\n\t\tif (cacheMeta()) {\n\t\t\tfinal String parentPath = getKeyValueAccess().parent(normalPath);\n\t\t\tgetCache().removeCache(parentPath, normalPath);\n\t\t}\n\n\t\t/* an IOException should have occurred if anything had failed midway */\n\t\treturn true;\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.io.UncheckedIOException;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.FileLock;\nimport java.nio.channels.OverlappingFileLockException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardOpenOption;\n\n/**\n * Holds a channel and system-level file lock (shared for writing, non-shared\n * for reading) and keeps it open until this {@code ChannelLock} is {@link\n * #close() closed}.\n */\nclass ChannelLock implements Closeable {\n\n\tprivate final FileChannel channel;\n\n\t/**\n\t * Hold a hard reference to the {@code FileLock} to make sure it is not\n\t * prematurely released.\n\t * \n\t * NB: We do not call {@code lock.release()} in {@link #close}, because at\n\t * this point the channel might be already closed (by an external writer).\n\t * {@code lock.release()} will throw an exception if the channel is already\n\t * closed. Instead, we just close the channel which will automatically\n\t * release the lock.\n\t */\n\t@SuppressWarnings({\"unused\", \"FieldCanBeLocal\"})\n\tprivate final FileLock lock;\n\n\tprivate ChannelLock(final FileChannel channel, final FileLock lock) {\n\t\tthis.channel = channel;\n\t\tthis.lock = lock;\n\t}\n\n\tpublic void close() throws IOException {\n\n\t\t// NB: We do not call lock.release() here, because it may throw an\n\t\t// exception if the channel is already closed. Instead, we just close\n\t\t// the channel. This will automatically release the lock. (And it is ok\n\t\t// to close an already closed channel.)\n\n\t\tchannel.close();\n\t}\n\n\tFileChannel getChannel() {\n\t\treturn channel;\n\t}\n\n\t/**\n\t * Create a {@link FileChannel} on the given {@code path} and lock it with a\n\t * system-level {@link FileLock}. If there is an existing overlapping file\n\t * lock, this method will block until the existing lock is released and the\n\t * channel could be locked (by us).\n\t *
\n\t * The {@code FileLock} is exclusive if the {@code path} is locked {@code\n\t * forWriting}, and shared otherwise.\n\t *
\n\t * If the {@code path} is locked {@code forWriting} non-existing file and\n\t * the parent directories are created as needed.\n\t *\n\t * @throws IOException if an error occurs while opening the channel, or if\n\t * the calling thread is interrupted while waiting for the {@code FileLock}.\n\t */\n\tstatic ChannelLock lock(final Path path, final boolean forWriting, final LockingPolicy policy) throws IOException {\n\n\t\tfinal FileChannel channel = openFileChannel(path, forWriting);\n\t\tif (policy == LockingPolicy.UNSAFE) {\n\t\t\treturn new ChannelLock(channel, null);\n\t\t}\n\t\ttry {\n\t\t\twhile (true) {\n\t\t\t\ttry {\n\t\t\t\t\tfinal FileLock lock = channel.lock(0, Long.MAX_VALUE, !forWriting);\n\t\t\t\t\treturn new ChannelLock(channel, lock);\n\t\t\t\t} catch (final OverlappingFileLockException e) {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tThread.sleep(100);\n\t\t\t\t\t} catch (final InterruptedException ie) {\n\t\t\t\t\t\tThread.currentThread().interrupt();\n\t\t\t\t\t\tthrow new IOException(\"Interrupted while waiting for file lock\", ie);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} catch (Exception e) {\n\t\t\tif (policy == LockingPolicy.STRICT) {\n\t\t\t\tcloseQuietly(channel);\n\t\t\t\tthrow e;\n\t\t\t} else {\n\t\t\t\treturn new ChannelLock(channel, null);\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Opens a file channel. If the channel is opened {@code forWriting},\n\t * then this may create the file and the parent directories as needed.\n\t *\n\t * @throws IOException\n\t * \t\tif the channel cannot be opened\n\t */\n\tprivate static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException {\n\n\t\tif (forWriting) {\n\t\t\tfinal Path parent = path.getParent();\n\t\t\tif (parent != null) {\n\t\t\t\tFiles.createDirectories(parent);\n\t\t\t}\n\t\t\treturn FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE);\n\t\t} else {\n\t\t\treturn FileChannel.open(path, StandardOpenOption.READ);\n\t\t}\n\t}\n\n\tprivate static void closeQuietly(final FileChannel fileChannel) {\n\t\tif (fileChannel != null) {\n\t\t\ttry {\n\t\t\t\tfileChannel.close();\n\t\t\t} catch (final IOException | UncheckedIOException ignored) {\n\t\t\t}\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/Compression.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.Serializable;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Inherited;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\nimport org.janelia.saalfeldlab.n5.codec.DataCodec;\nimport org.janelia.saalfeldlab.n5.codec.CodecInfo;\nimport org.janelia.saalfeldlab.n5.codec.DataCodecInfo;\nimport org.scijava.annotations.Indexable;\n\n/**\n * This interface is used to indicate that a {@link DataCodec} can be\n * serialized as a \"compression\" for the N5 format (using the N5 API).\n *
\n * N5Readers and N5Writers for the N5 format can declare DataCodecs that\n * implement this interface so that the {@link CompressionAdapter} is used for\n * serialization.\n *
\n * See also: an alternative method for serializing general {@link CodecInfo}s is\n * with the {@link NameConfigAdapter}. This interface remains for legacy\n * (de)serialization.\n *\n * @author Stephan Saalfeld\n */\npublic interface Compression extends Serializable, DataCodec, DataCodecInfo {\n\n\t/**\n\t * Annotation for runtime discovery of compression schemes.\n\t *\n\t */\n\t@Retention(RetentionPolicy.RUNTIME)\n\t@Inherited\n\t@Target(ElementType.TYPE)\n\t@Indexable\n\t@interface CompressionType {\n\n\t\tString value();\n\t}\n\n\t/**\n\t * Annotation for runtime discovery of compression schemes.\n\t *\n\t */\n\t@Retention(RetentionPolicy.RUNTIME)\n\t@Inherited\n\t@Target(ElementType.FIELD)\n\t@interface CompressionParameter {}\n\n\tdefault String getType() {\n\n\t\tfinal CompressionType compressionType = getClass().getAnnotation(CompressionType.class);\n\t\tif (compressionType == null)\n\t\t\treturn null;\n\t\telse\n\t\t\treturn compressionType.value();\n\t}\n\n\t@Override\n\tdefault DataCodec create() {\n\t\treturn this;\n\t}\n\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Type;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.Map.Entry;\n\nimport org.janelia.saalfeldlab.n5.Compression.CompressionParameter;\nimport org.janelia.saalfeldlab.n5.Compression.CompressionType;\nimport org.scijava.annotations.Index;\nimport org.scijava.annotations.IndexItem;\n\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonSerializationContext;\nimport com.google.gson.JsonSerializer;\n\n/**\n * Compression adapter, auto-discovers annotated compression implementations\n * in the classpath.\n *\n * @author Stephan Saalfeld\n */\npublic class CompressionAdapter implements JsonDeserializer, JsonSerializer {\n\n\tprivate static CompressionAdapter instance = null;\n\n\tprivate final HashMap> compressionConstructors = new HashMap<>();\n\tprivate final HashMap>> compressionParameters = new HashMap<>();\n\n\tprivate static ArrayList getDeclaredFields(Class clazz) {\n\n\t\tfinal ArrayList fields = new ArrayList<>();\n\t\tfields.addAll(Arrays.asList(clazz.getDeclaredFields()));\n\t\tfor (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass())\n\t\t\tfields.addAll(Arrays.asList(clazz.getDeclaredFields()));\n\t\treturn fields;\n\t}\n\n\t@SuppressWarnings(\"unchecked\")\n\tpublic static synchronized void update(final boolean override) {\n\n\t\tif (override || instance == null) {\n\n\t\t\tfinal CompressionAdapter newInstance = new CompressionAdapter();\n\n\t\t\tfinal ClassLoader classLoader = Thread.currentThread().getContextClassLoader();\n\t\t\tfinal Index annotationIndex = Index.load(CompressionType.class, classLoader);\n\t\t\tfor (final IndexItem item : annotationIndex) {\n\t\t\t\tClass clazz;\n\t\t\t\ttry {\n\t\t\t\t\tclazz = (Class)Class.forName(item.className());\n\t\t\t\t\tfinal String type = clazz.getAnnotation(CompressionType.class).value();\n\n\t\t\t\t\tfinal Constructor constructor = clazz.getDeclaredConstructor();\n\n\t\t\t\t\tfinal HashMap> parameters = new HashMap<>();\n\t\t\t\t\tfinal ArrayList fields = getDeclaredFields(clazz);\n\t\t\t\t\tfor (final Field field : fields) {\n\t\t\t\t\t\tif (field.getAnnotation(CompressionParameter.class) != null) {\n\t\t\t\t\t\t\tparameters.put(field.getName(), field.getType());\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tnewInstance.compressionConstructors.put(type, constructor);\n\t\t\t\t\tnewInstance.compressionParameters.put(type, parameters);\n\t\t\t\t} catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException\n\t\t\t\t\t\t| UnsatisfiedLinkError e) {\n\t\t\t\t\tSystem.err.println(\"Compression '\" + item.className() + \"' could not be registered\");\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tinstance = newInstance;\n\t\t}\n\t}\n\n\tpublic static void update() {\n\n\t\tupdate(false);\n\t}\n\n\t@Override\n\tpublic JsonElement serialize(\n\t\t\tfinal Compression compression,\n\t\t\tfinal Type typeOfSrc,\n\t\t\tfinal JsonSerializationContext context) {\n\n\t\tfinal String type = compression.getType();\n\t\tfinal Class clazz = compression.getClass();\n\n\t\tfinal JsonObject json = new JsonObject();\n\t\tjson.addProperty(\"type\", type);\n\n\t\tfinal HashMap> parameterTypes = compressionParameters.get(type);\n\t\ttry {\n\t\t\tfor (final Entry> parameterType : parameterTypes.entrySet()) {\n\t\t\t\tfinal String name = parameterType.getKey();\n\t\t\t\tfinal Field field = clazz.getDeclaredField(name);\n\t\t\t\tfinal boolean isAccessible = field.isAccessible();\n\t\t\t\tfield.setAccessible(true);\n\t\t\t\tfinal Object value = field.get(compression);\n\t\t\t\tfield.setAccessible(isAccessible);\n\t\t\t\tjson.add(parameterType.getKey(), context.serialize(value));\n\t\t\t}\n\t\t} catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {\n\t\t\te.printStackTrace(System.err);\n\t\t\treturn null;\n\t\t}\n\n\t\treturn json;\n\t}\n\n\t@Override\n\tpublic Compression deserialize(\n\t\t\tfinal JsonElement json,\n\t\t\tfinal Type typeOfT,\n\t\t\tfinal JsonDeserializationContext context) throws JsonParseException {\n\n\t\tfinal JsonObject jsonObject = json.getAsJsonObject();\n\t\tfinal JsonElement jsonType = jsonObject.get(\"type\");\n\t\tif (jsonType == null)\n\t\t\treturn null;\n\n\t\tfinal String type = jsonType.getAsString();\n\t\tfinal Constructor constructor = compressionConstructors.get(type);\n\t\tfinal Compression compression;\n\t\ttry {\n\t\t\tcompression = constructor.newInstance();\n\t\t\tfinal HashMap> parameterTypes = compressionParameters.get(type);\n\t\t\tfor (final Entry> parameterType : parameterTypes.entrySet()) {\n\t\t\t\tfinal String name = parameterType.getKey();\n\t\t\t\tif (jsonObject.has(name)) {\n\t\t\t\t\tfinal Object parameter = context.deserialize(jsonObject.get(name), parameterType.getValue());\n\t\t\t\t\tReflectionUtils.setFieldValue(compression, name, parameter);\n\t\t\t\t}\n\t\t\t}\n\t\t} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException\n\t\t\t\t| SecurityException | NoSuchFieldException e) {\n\t\t\te.printStackTrace(System.err);\n\t\t\treturn null;\n\t\t}\n\n\t\treturn compression;\n\t}\n\n\tpublic static CompressionAdapter getJsonAdapter() {\n\n\t\tif (instance == null)\n\t\t\tupdate();\n\t\treturn instance;\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\n/**\n * Interface for data blocks. A data block has data, a position on the block\n * grid, and a size.\n *\n * @param type of the data contained in the DataBlock\n *\n * @author Stephan Saalfeld\n */\npublic interface DataBlock {\n\n\t/**\n\t * Returns the size of this data block.\n\t * \n\t * The size of a data block is expected to be smaller than or equal to the\n\t * spacing of the block grid. The dimensionality of size is expected to be\n\t * equal to the dimensionality of the dataset. Consistency is not enforced.\n\t *\n\t * @return size of the data block\n\t */\n\tint[] getSize();\n\n\t/**\n\t * Returns the position of this data block on the block grid relative to dataset.\n\t *
\n\t * The dimensionality of the grid position is expected to be equal to the\n\t * dimensionality of the dataset. Consistency is not enforced.\n\t *\n\t * @return position on the block grid\n\t */\n\tlong[] getGridPosition();\n\n\t/**\n\t * Returns the data object held by this data block.\n\t *\n\t * @return data object\n\t */\n\tT getData();\n\n\t/**\n\t * Returns the number of elements in this {@link DataBlock}. This number is\n\t * not necessarily equal {@link #getNumElements(int[])\n\t * getNumElements(getSize())}.\n\t *\n\t * @return the number of elements\n\t */\n\tint getNumElements();\n\n\t/**\n\t * Returns the number of elements in a box of given size.\n\t *\n\t * @param size\n\t * the size\n\t * @return the number of elements\n\t */\n\tstatic int getNumElements(final int[] size) {\n\n\t\tint n = size[0];\n\t\tfor (int i = 1; i < size.length; ++i)\n\t\t\tn *= size[i];\n\t\treturn n;\n\t}\n\n\t/**\n\t * Factory for creating {@code DataBlock}.\n\t *\n\t * @param \n\t * \t\ttype of the data contained in the DataBlock\n\t */\n\tinterface DataBlockFactory {\n\n\t\t/**\n\t\t * Create a new {@link DataBlock} with the given {@code blockSize}, {@code gridPosition}, and {@code data} content.\n\t\t *\n\t\t * @param blockSize\n\t\t * \t\tthe block size\n\t\t * @param gridPosition\n\t\t * \t\tthe grid position\n\t\t * @param data\n\t\t * \t\tthe data object\n\t\t *\n\t\t * @return a new DataBlock\n\t\t */\n\t\tDataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/DataType.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.lang.reflect.Type;\n\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.JsonSerializationContext;\nimport com.google.gson.JsonSerializer;\n\n/**\n * Enumerates available data types.\n *\n * @author Stephan Saalfeld\n */\npublic enum DataType {\n\n\tUINT8(\n\t\t\t\"uint8\",\n\t\t\t(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew byte[numElements])),\n\tUINT16(\n\t\t\t\"uint16\",\n\t\t\t(blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew short[numElements])),\n\tUINT32(\n\t\t\t\"uint32\",\n\t\t\t(blockSize, gridPosition, numElements) -> new IntArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew int[numElements])),\n\tUINT64(\n\t\t\t\"uint64\",\n\t\t\t(blockSize, gridPosition, numElements) -> new LongArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew long[numElements])),\n\tINT8(\n\t\t\t\"int8\",\n\t\t\t(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew byte[numElements])),\n\tINT16(\n\t\t\t\"int16\",\n\t\t\t(blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew short[numElements])),\n\tINT32(\n\t\t\t\"int32\",\n\t\t\t(blockSize, gridPosition, numElements) -> new IntArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew int[numElements])),\n\tINT64(\n\t\t\t\"int64\",\n\t\t\t(blockSize, gridPosition, numElements) -> new LongArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew long[numElements])),\n\tFLOAT32(\n\t\t\t\"float32\",\n\t\t\t(blockSize, gridPosition, numElements) -> new FloatArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew float[numElements])),\n\tFLOAT64(\n\t\t\t\"float64\",\n\t\t\t(blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew double[numElements])),\n\tSTRING(\n\t\t\t\"string\",\n\t\t\t(blockSize, gridPosition, numElements) -> new StringDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew String[numElements])),\n\tOBJECT(\n\t\t\t\"object\",\n\t\t\t(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(\n\t\t\t\t\tblockSize,\n\t\t\t\t\tgridPosition,\n\t\t\t\t\tnew byte[numElements]));\n\n\tprivate final String label;\n\n\tprivate final DataBlockFactory dataBlockFactory;\n\n\tDataType(final String label, final DataBlockFactory dataBlockFactory) {\n\n\t\tthis.label = label;\n\t\tthis.dataBlockFactory = dataBlockFactory;\n\t}\n\n\t@Override\n\tpublic String toString() {\n\n\t\treturn label;\n\t}\n\n\tpublic static DataType fromString(final String string) {\n\n\t\tfor (final DataType value : values())\n\t\t\tif (value.toString().equals(string))\n\t\t\t\treturn value;\n\t\treturn null;\n\t}\n\n\t/**\n\t * Factory for {@link DataBlock DataBlocks}.\n\t *\n\t * @param blockSize\n\t * the block size\n\t * @param gridPosition\n\t * the grid position\n\t * @param numElements\n\t * the number of elements (not necessarily one element per block\n\t * element)\n\t * @return the data block\n\t */\n\tpublic DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) {\n\n\t\treturn dataBlockFactory.createDataBlock(blockSize, gridPosition, numElements);\n\t}\n\n\t/**\n\t * Factory for {@link DataBlock DataBlocks} with one data element for each\n\t * block element (e.g. pixel image).\n\t *\n\t * @param blockSize\n\t * the block size\n\t * @param gridPosition\n\t * the grid position\n\t * @return the data block\n\t */\n\tpublic DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) {\n\n\t\treturn dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize));\n\t}\n\n\tprivate interface DataBlockFactory {\n\n\t\tDataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements);\n\t}\n\n\tstatic public class JsonAdapter implements JsonDeserializer, JsonSerializer {\n\n\t\t@Override\n\t\tpublic DataType deserialize(\n\t\t\t\tfinal JsonElement json,\n\t\t\t\tfinal Type typeOfT,\n\t\t\t\tfinal JsonDeserializationContext context) throws JsonParseException {\n\n\t\t\treturn DataType.fromString(json.getAsString());\n\t\t}\n\n\t\t@Override\n\t\tpublic JsonElement serialize(\n\t\t\t\tfinal DataType src,\n\t\t\t\tfinal Type typeOfSrc,\n\t\t\t\tfinal JsonSerializationContext context) {\n\n\t\t\treturn new JsonPrimitive(src.toString());\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonSerializationContext;\nimport com.google.gson.JsonSerializer;\n\nimport org.janelia.saalfeldlab.n5.codec.BlockCodec;\nimport org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;\nimport org.janelia.saalfeldlab.n5.codec.CodecInfo;\nimport org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;\nimport org.janelia.saalfeldlab.n5.shard.DatasetAccess;\nimport org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess;\nimport org.janelia.saalfeldlab.n5.shard.ShardCodecInfo;\nimport org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;\n\nimport java.io.Serializable;\nimport java.lang.reflect.Type;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.stream.Collectors;\n\nimport org.janelia.saalfeldlab.n5.codec.DataCodecInfo;\nimport org.janelia.saalfeldlab.n5.codec.DatasetCodec;\nimport org.janelia.saalfeldlab.n5.codec.DatasetCodecInfo;\n\n\n/**\n * Mandatory dataset attributes:\n *\n * \n * - long[] : dimensions
\n * - int[] : blockSize
\n * - {@link DataType} : dataType
\n * - {@link CodecInfo}... : encode/decode routines
\n *
\n *\n * @author Stephan Saalfeld\n */\npublic class DatasetAttributes implements Serializable {\n\n\tprivate static final long serialVersionUID = -4521467080388947553L;\n\n\tpublic static final String DIMENSIONS_KEY = \"dimensions\";\n\tpublic static final String BLOCK_SIZE_KEY = \"blockSize\";\n\tpublic static final String SHARD_SIZE_KEY = \"shardSize\";\n\tpublic static final String DATA_TYPE_KEY = \"dataType\";\n\tpublic static final String COMPRESSION_KEY = \"compression\";\n\tpublic static final String CODEC_KEY = \"codecs\";\n\n\tpublic static final String[] N5_DATASET_ATTRIBUTES = new String[]{\n\t\t\tDIMENSIONS_KEY, BLOCK_SIZE_KEY, DATA_TYPE_KEY, COMPRESSION_KEY, CODEC_KEY\n\t};\n\n\t/* version 0 */\n\tprotected static final String compressionTypeKey = \"compressionType\";\n\n\tprivate final long[] dimensions;\n\n\t// number of samples per chunk per dimension\n\tprivate final int[] chunkSize;\n\n\t// number of samples per block per dimension\n\t// identical to chunkSize for non-sharded datasets\n\tprivate final int[] blockSize;\n\n\tprivate final DataType dataType;\n\n\tprivate final JsonElement defaultValue;\n\n\tprivate final BlockCodecInfo blockCodecInfo;\n\tprivate final DataCodecInfo[] dataCodecInfos;\n\tprivate final DatasetCodecInfo[] datasetCodecInfos;\n\n\tprivate transient final DatasetAccess access;\n\n\tpublic DatasetAttributes(\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] blockSize,\n\t\t\tfinal DataType dataType,\n\t\t\tfinal JsonElement defaultValue,\n\t\t\tfinal BlockCodecInfo blockCodecInfo,\n\t\t\tfinal DatasetCodecInfo[] datasetCodecInfos,\n\t\t\tfinal DataCodecInfo... dataCodecInfos) {\n\n\t\tthis.dimensions = dimensions;\n\t\tthis.dataType = dataType;\n\t\tthis.blockSize = blockSize;\n\t\tthis.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue;\n\n\t\tthis.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo;\n\t\tthis.datasetCodecInfos = datasetCodecInfos;\n\n\t\tif (dataCodecInfos == null)\n\t\t\tthis.dataCodecInfos = new DataCodecInfo[0];\n\t\telse\n\t\t\tthis.dataCodecInfos = Arrays.stream(dataCodecInfos)\n\t\t\t\t\t.filter(it -> it != null && !(it instanceof RawCompression))\n\t\t\t\t\t.toArray(DataCodecInfo[]::new);\n\n\t\taccess = createDatasetAccess();\n\t\tchunkSize = access.getGrid().getBlockSize(0);\n\t}\n\n\tpublic DatasetAttributes(\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] outerBlockSize,\n\t\t\tfinal DataType dataType,\n\t\t\tfinal BlockCodecInfo blockCodecInfo,\n\t\t\tfinal DatasetCodecInfo[] datasetCodecInfos,\n\t\t\tfinal DataCodecInfo... dataCodecInfos) {\n\n\t\tthis(dimensions, outerBlockSize, dataType, JsonNull.INSTANCE,\n\t\t\t\tblockCodecInfo, datasetCodecInfos, dataCodecInfos);\n\t}\n\n\tpublic DatasetAttributes(\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] outerBlockSize,\n\t\t\tfinal DataType dataType,\n\t\t\tfinal BlockCodecInfo blockCodecInfo,\n\t\t\tfinal DataCodecInfo... dataCodecInfos) {\n\n\t\tthis(dimensions, outerBlockSize, dataType, blockCodecInfo, null, dataCodecInfos);\n\t}\n\n\t/**\n\t * Constructs a DatasetAttributes instance with specified dimensions, block size, data type,\n\t * and single compressor with default codec.\n\t *\n\t * @param dimensions the dimensions of the dataset\n\t * @param blockSize the size of the blocks in the dataset\n\t * @param dataType the data type of the dataset\n\t * @param dataCodecInfos the codecs used encode/decode the data\n\t */\n\tpublic DatasetAttributes(\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] blockSize,\n\t\t\tfinal DataType dataType,\n\t\t\tfinal DataCodecInfo... dataCodecInfos) {\n\n\t\tthis(dimensions, blockSize, dataType, null, dataCodecInfos);\n\t}\n\n\t/**\n\t * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs\n\t *\n\t * @param dimensions the dimensions of the dataset\n\t * @param blockSize the size of the blocks in the dataset\n\t * @param dataType the data type of the dataset\n\t */\n\tpublic DatasetAttributes(\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] blockSize,\n\t\t\tfinal DataType dataType) {\n\n\t\tthis(dimensions, blockSize, dataType, new DataCodecInfo[0]);\n\t}\n\n\tprivate DatasetAccess createDatasetAccess() {\n\n\t\tfinal int m = nestingDepth(blockCodecInfo);\n\n\t\t// There are m codecs: 1 DataBlock codecs, and m-1 shard codecs.\n\t\t// The inner-most codec (the DataBlock codec) is at index 0.\n\t\tfinal int[][] blockSizes = new int[m][];\n\n\t\t// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs\n\t\t// blockCodecInfo.create below could fail unexpecedly with invalid\n\t\t// blockSizes so validate first\n\t\tblockSizes[m - 1] = blockSize;\n\t\tBlockCodecInfo tmpInfo = blockCodecInfo;\n\t\tfor (int l = m - 1; l > 0; --l) {\n\t\t\tfinal ShardCodecInfo info = (ShardCodecInfo)tmpInfo;\n\t\t\tblockSizes[l - 1] = info.getInnerBlockSize();\n\t\t\ttmpInfo = info.getInnerBlockCodecInfo();\n\t\t}\n\n\t\tBlockCodecInfo currentBlockCodecInfo = blockCodecInfo;\n\t\tDataCodecInfo[] currentDataCodecInfos = dataCodecInfos;\n\n\t\tDatasetCodecInfo[] datasetCodecInfos = this.datasetCodecInfos;\n\n\t\tfinal NestedGrid grid = new NestedGrid(blockSizes, dimensions);\n\t\tfinal BlockCodec[] blockCodecs = new BlockCodec[m];\n\t\tfor (int l = m - 1; l >= 0; --l) {\n\t\t\tblockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos);\n\t\t\tif (l > 0) {\n\t\t\t\tfinal ShardCodecInfo info = (ShardCodecInfo) currentBlockCodecInfo;\n\t\t\t\tcurrentBlockCodecInfo = info.getInnerBlockCodecInfo();\n\t\t\t\tcurrentDataCodecInfos = info.getInnerDataCodecInfos();\n\t\t\t\tif (info.getInnerDataCodecInfos() != null) {\n\t\t\t\t\tif (datasetCodecInfos != null && datasetCodecInfos.length > 0) {\n\t\t\t\t\t\tthrow new N5Exception.N5JsonParseException(\"Found DatasetCodecs both inside and outside of shards. Not handled\");\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t\tdatasetCodecInfos = info.getInnerDatasetCodecInfos();\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// add dataset codecs\n\t\tblockCodecs[0] = blockCodecWithDatasetCodecs(this, blockCodecs[0], datasetCodecInfos);\n\n\t\treturn new DefaultDatasetAccess<>(grid, blockCodecs);\n\t}\n\n\t@SuppressWarnings(\"unchecked\")\n\tprivate static BlockCodec blockCodecWithDatasetCodecs(final DatasetAttributes attributes, final BlockCodec blockCodec,\n\t\t\tfinal DatasetCodecInfo[] datasetCodecInfos) {\n\n\t\tBlockCodec result = blockCodec;\n\t\tif (datasetCodecInfos != null) {\n\t\t\tfor (final DatasetCodecInfo info : datasetCodecInfos) {\n\t\t\t\tresult = DatasetCodec.concatenate(info.create(attributes), (BlockCodec)result);\n\t\t\t}\n\t\t}\n\t\treturn result;\n\t}\n\n\tprivate static int nestingDepth(BlockCodecInfo info) {\n\n\t\tif (info instanceof ShardCodecInfo) {\n\t\t\treturn 1 + nestingDepth(((ShardCodecInfo)info).getInnerBlockCodecInfo());\n\t\t} else {\n\t\t\treturn 1;\n\t\t}\n\t}\n\n\tprotected BlockCodecInfo defaultBlockCodecInfo() {\n\n\t\treturn new N5BlockCodecInfo();\n\t}\n\n\tpublic long[] getDimensions() {\n\n\t\treturn dimensions;\n\t}\n\n\tpublic int getNumDimensions() {\n\n\t\treturn dimensions.length;\n\t}\n\n\tpublic int[] getChunkSize() {\n\n\t\treturn chunkSize;\n\t}\n\n\tpublic int[] getBlockSize() {\n\n\t\treturn blockSize;\n\t}\n\n\tpublic JsonElement getDefaultValue() {\n\n\t\treturn defaultValue;\n\t}\n\n\tpublic boolean isSharded() {\n\n\t\treturn blockCodecInfo instanceof ShardCodecInfo;\n\t}\n\n\t/**\n\t * Only used for deserialization for N5 backwards compatibility.\n\t * {@link Compression} is no longer a special case. Prefer to reference {@link #getDataCodecInfos()}\n\t * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility.\n\t * \n\t * Deprecated in favor of {@link #getDataCodecInfos()}.\n\t *\n\t * @return compression CodecInfo, if one was present, or else RawCompression\n\t */\n\t@Deprecated\n\tpublic Compression getCompression() {\n\n\t\treturn Arrays.stream(dataCodecInfos)\n\t\t\t\t.filter(it -> it instanceof Compression)\n\t\t\t\t.map(it -> (Compression)it)\n\t\t\t\t.findFirst()\n\t\t\t\t.orElse(new RawCompression());\n\t}\n\n\tpublic DataType getDataType() {\n\n\t\treturn dataType;\n\t}\n\n\t/**\n\t * Get the {@link DatasetAccess} for this dataset.\n\t *\n\t * @return the {@code DatasetAccess} for this dataset\n\t */\n\tprotected DatasetAccess getDatasetAccess() {\n\n\t\treturn (DatasetAccess) access;\n\t}\n\n\t/**\n\t * Returns the {@code NestedGrid} for this dataset, from which block and\n\t * shard sizes are accessible.\n\t *\n\t * @return the NestedGrid\n\t */\n\tpublic NestedGrid getNestedBlockGrid() {\n\n\t\treturn getDatasetAccess().getGrid();\n\t}\n\n\tpublic BlockCodecInfo getBlockCodecInfo() {\n\n\t\treturn blockCodecInfo;\n\t}\n\n\tpublic DataCodecInfo[] getDataCodecInfos() {\n\n\t\treturn dataCodecInfos;\n\t}\n\n\tpublic DatasetCodecInfo[] getDatasetCodecInfos() {\n\n\t\treturn datasetCodecInfos;\n\t}\n\n\tpublic String relativeBlockPath(long... position) {\n\n\t\treturn Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining(\"/\"));\n\t}\n\n\tpublic HashMap asMap() {\n\n\t\tfinal HashMap map = new HashMap<>();\n\t\tmap.put(DIMENSIONS_KEY, dimensions);\n\t\tmap.put(BLOCK_SIZE_KEY, chunkSize);\n\t\tmap.put(DATA_TYPE_KEY, dataType);\n\t\tmap.put(COMPRESSION_KEY, getCompression());\n\t\treturn map;\n\t}\n\n\tpublic static Builder builder(final long[] dimensions, final DataType dataType) {\n\n\t\treturn new Builder(dimensions, dataType);\n\t}\n\n\tpublic static Builder builder(final DatasetAttributes attributes) {\n\n\t\treturn new Builder(attributes);\n\t}\n\n\tprivate static final int[] DEFAULT_1D_BLOCK_SIZE = new int[]{65536};\n\tprivate static final int[] DEFAULT_2D_BLOCK_SIZE = new int[]{512,512};\n\tprivate static final int[] DEFAULT_3D_BLOCK_SIZE = new int[]{128,128,128};\n\tprivate static final int DEFAULT_ND_DIM_LEN = 64;\n\n\tprotected static int[] defaultBlockSize(final long[] dimensions) {\n\n\t\tfinal int[] blockSize;\n\t\tif (dimensions.length == 1)\n\t\t\tblockSize = DEFAULT_1D_BLOCK_SIZE.clone();\n\t\telse if (dimensions.length == 2)\n\t\t\tblockSize = DEFAULT_2D_BLOCK_SIZE.clone();\n\t\telse if (dimensions.length == 3)\n\t\t\tblockSize = DEFAULT_3D_BLOCK_SIZE.clone();\n\t\telse {\n\t\t\tblockSize = new int[dimensions.length];\n\t\t\tArrays.fill(blockSize, DEFAULT_ND_DIM_LEN);\n\t\t}\n\t\tfor (int i = 0; i < blockSize.length; i++)\n\t\t\tblockSize[i] = (int)Math.min(blockSize[i], dimensions[i]);\n\t\treturn blockSize;\n\t}\n\n\tpublic static class Builder {\n\n\t\tprivate final long[] dimensions;\n\t\tprivate final DataType dataType;\n\n\t\tprivate int[] blockSize;\n\t\tprivate DataCodecInfo[] dataCodecInfos = new DataCodecInfo[0];\n\n\t\tpublic Builder(final long[] dimensions, final DataType dataType) {\n\n\t\t\tthis.dimensions = dimensions.clone();\n\t\t\tthis.dataType = dataType;\n\t\t\tthis.blockSize = defaultBlockSize(dimensions);\n\t\t}\n\n\t\tpublic Builder(final DatasetAttributes attributes) {\n\n\t\t\tthis.dimensions = attributes.getDimensions();\n\t\t\tthis.dataType = attributes.getDataType();\n\t\t\tthis.blockSize = attributes.getBlockSize();\n\t\t\tthis.dataCodecInfos = attributes.getDataCodecInfos();\n\t\t}\n\n\t\tpublic Builder blockSize(final int[] blockSize) {\n\n\t\t\tthis.blockSize = blockSize.clone();\n\t\t\treturn this;\n\t\t}\n\n\t\t/**\n\t\t * Sets the compression codec. Has no effect if {@code compression} is\n\t\t * null or {@link RawCompression}.\n\t\t *\n\t\t * @param compression the compression to use\n\t\t * @return this builder\n\t\t */\n\t\tpublic Builder compression(final Compression compression) {\n\n\t\t\tif (compression != null && !(compression instanceof RawCompression))\n\t\t\t\tthis.dataCodecInfos = new DataCodecInfo[]{compression};\n\t\t\treturn this;\n\t\t}\n\n\t\tpublic DatasetAttributes build() {\n\t\t\tfinal int[] resolvedBlockSize = blockSize != null ? blockSize : defaultBlockSize(dimensions);\n\t\t\treturn new DatasetAttributes(dimensions, resolvedBlockSize, dataType, new N5BlockCodecInfo(), null, dataCodecInfos);\n\t\t}\n\t}\n\n\tprivate static DatasetAttributesAdapter adapter = null;\n\n\tpublic static DatasetAttributesAdapter getJsonAdapter() {\n\n\t\tif (adapter == null) {\n\t\t\tadapter = new DatasetAttributesAdapter();\n\t\t}\n\t\treturn adapter;\n\t}\n\n\tpublic static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer {\n\n\t\t@Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {\n\n\t\t\tif (json == null || !json.isJsonObject())\n\t\t\t\treturn null;\n\t\t\tfinal JsonObject obj = json.getAsJsonObject();\n\t\t\tfinal boolean validKeySet = obj.has(DIMENSIONS_KEY)\n\t\t\t\t\t&& obj.has(BLOCK_SIZE_KEY)\n\t\t\t\t\t&& obj.has(DATA_TYPE_KEY)\n\t\t\t\t\t&& (obj.has(CODEC_KEY) || obj.has(COMPRESSION_KEY) || obj.has(compressionTypeKey));\n\n\t\t\tif (!validKeySet)\n\t\t\t\treturn null;\n\n\t\t\tfinal long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class);\n\t\t\tfinal int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class);\n\n\t\t\tfinal DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class);\n\n\t\t\tfinal BlockCodecInfo blockCodecInfo;\n\t\t\tfinal DataCodecInfo[] dataCodecs;\n\t\t\tif (obj.has(CODEC_KEY)) {\n\t\t\t\tfinal CodecInfo[] codecs = context.deserialize(obj.get(CODEC_KEY), CodecInfo[].class);\n\t\t\t\tblockCodecInfo = (BlockCodecInfo)codecs[0];\n\t\t\t\tdataCodecs = new DataCodecInfo[codecs.length - 1];\n\t\t\t\tfor (int i = 1; i < codecs.length; i++) {\n\t\t\t\t\tdataCodecs[i - 1] = (DataCodecInfo)codecs[i];\n\t\t\t\t}\n\t\t\t} else if (obj.has(COMPRESSION_KEY)) {\n\t\t\t\tfinal Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context);\n\t\t\t\tdataCodecs = new DataCodecInfo[]{compression};\n\t\t\t\tblockCodecInfo = new N5BlockCodecInfo();\n\t\t\t} else if (obj.has(compressionTypeKey)) {\n\t\t\t\tfinal Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString());\n\t\t\t\tdataCodecs = new DataCodecInfo[]{compression};\n\t\t\t\tblockCodecInfo = new N5BlockCodecInfo();\n\t\t\t} else {\n\t\t\t\treturn null;\n\t\t\t}\n\n\t\t\treturn new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs);\n\t\t}\n\n\t\t//FIXME\n\t\t// this implements multi-codec serialization for N5. We probably don't want this now\n\t\t@Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) {\n\n\t\t\tfinal JsonObject obj = new JsonObject();\n\t\t\tobj.add(DIMENSIONS_KEY, context.serialize(src.dimensions));\n\t\t\tobj.add(BLOCK_SIZE_KEY, context.serialize(src.chunkSize));\n\t\t\tobj.add(DATA_TYPE_KEY, context.serialize(src.dataType));\n\n\t\t\tfinal DataCodecInfo[] codecs = src.dataCodecInfos;\n\t\t\t// length > 1 is actually invalid, but this is checked on construction\n\t\t\tif (codecs.length == 0)\n\t\t\t\tobj.add(COMPRESSION_KEY, context.serialize(new RawCompression()));\n\t\t\telse\n\t\t\t\tobj.add(COMPRESSION_KEY, context.serialize(codecs[0]));\n\n\t\t\treturn obj;\n\t\t}\n\n\t\tprivate static Compression getCompressionVersion0(final String compressionVersion0Name) {\n\n\t\t\tswitch (compressionVersion0Name) {\n\t\t\tcase \"raw\":\n\t\t\t\treturn new RawCompression();\n\t\t\tcase \"gzip\":\n\t\t\t\treturn new GzipCompression();\n\t\t\tcase \"bzip2\":\n\t\t\t\treturn new Bzip2Compression();\n\t\t\tcase \"lz4\":\n\t\t\t\treturn new Lz4Compression();\n\t\t\tcase \"xz\":\n\t\t\t\treturn new XzCompression();\n\t\t\t}\n\t\t\treturn null;\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class DoubleArrayDataBlock extends AbstractDataBlock {\n\n\tpublic DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final double[] data) {\n\n\t\tsuper(size, gridPosition, data, a -> a.length);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/FileKeyLockManager.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.IOException;\nimport java.lang.ref.ReferenceQueue;\nimport java.lang.ref.WeakReference;\nimport java.nio.channels.FileLock;\nimport java.nio.file.Path;\nimport java.util.Collections;\nimport java.util.EnumMap;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\nimport static org.janelia.saalfeldlab.n5.LockingPolicy.STRICT;\n\n/**\n * Provides thread-safe and process-safe read/write locking for filesystem paths.\n * Uses thread locks for JVM coordination and file locks for inter-process coordination.\n */\nclass FileKeyLockManager {\n\n\tprivate static final Map managers = Collections.synchronizedMap(new EnumMap<>(LockingPolicy.class));\n\n\tstatic FileKeyLockManager forPolicy(final LockingPolicy policy) {\n\t\treturn managers.computeIfAbsent(policy, FileKeyLockManager::new);\n\t}\n\n\t/**\n\t * @deprecated use {@link FileKeyLockManager#forPolicy(LockingPolicy)}\n\t */\n\t@Deprecated\n\tstatic final FileKeyLockManager FILE_LOCK_MANAGER = forPolicy(STRICT);\n\n\tprivate final LockingPolicy policy;\n\n\t/**\n\t * Create a new {@link FileKeyLockManager} with the specified locking policy.\n\t * \n\t * The given locking {@link LockingPolicy policy} applies to OS-level locking.\n\t * For both the {@code STRICT} and {@code PERMISSIVE} policy, a {@link\n\t * FileLock} is obtained. If this fails, {@code STRICT} will throw an {@code\n\t * IOException}. {@code PERMISSIVE} will proceed without locking. {@code\n\t * UNSAFE} will not attempt OS-level locking, however will still manage\n\t * mutual exclusion of readers and writers in the same JVM. Trying to lock\n\t * the same path with different locking policies will throw an {@code\n\t * IOException}.\n\t *\n\t * @param policy\n\t * \t\tthe locking policy\n\t */\n\tprivate FileKeyLockManager(final LockingPolicy policy) {\n\t\tthis.policy = policy;\n\t}\n\n\tprivate final ConcurrentHashMap locks = new ConcurrentHashMap<>();\n\n\tprivate final ReferenceQueue refQueue = new ReferenceQueue<>();\n\n\tprivate static class WeakValue extends WeakReference {\n\n\t\tfinal String key;\n\n\t\tWeakValue(\n\t\t\t\tfinal String key,\n\t\t\t\tfinal KeyLockState value,\n\t\t\t\tfinal ReferenceQueue queue) {\n\n\t\t\tsuper(value, queue);\n\t\t\tthis.key = key;\n\t\t}\n\t}\n\n\t/**\n\t * Remove entries from the cache whose references have been\n\t * garbage-collected.\n\t */\n\tprivate void cleanUp()\n\t{\n\t\twhile (true) {\n\t\t\tfinal WeakValue ref = (WeakValue) refQueue.poll();\n\t\t\tif (ref == null)\n\t\t\t\tbreak;\n\t\t\tlocks.remove(ref.key, ref);\n\t\t}\n\t}\n\n\tprivate KeyLockState keyLockState(final Path path, final LockingPolicy policy) throws IOException {\n\n\t\tfinal String key = path.toAbsolutePath().toString();\n\n\t\tcleanUp();\n\n\t\tfinal WeakValue existingRef = locks.get(key);\n\t\tKeyLockState state = existingRef == null ? null : existingRef.get();\n\t\tif (state != null) {\n\t\t\treturn state;\n\t\t}\n\n\t\tfinal KeyLockState newState = new KeyLockState(path, policy);\n\t\twhile (state == null) {\n\t\t\tfinal WeakValue ref = locks.compute(key,\n\t\t\t\t\t(k, v) -> (v != null && v.get() != null)\n\t\t\t\t\t\t\t? v\n\t\t\t\t\t\t\t: new WeakValue(k, newState, refQueue));\n\t\t\tstate = ref.get();\n\t\t}\n\t\treturn state;\n\t}\n\n\t/**\n\t * Acquires a read lock for the specified key. Multiple threads can hold\n\t * read locks for the same key simultaneously.\n\t * \n\t * The first reader will acquire a shared file lock. Subsequent readers\n\t * only acquire the thread-level lock.\n\t *\n\t * @param path\n\t * \t\tthe key (file path) to lock for reading\n\t *\n\t * @return a {@link LockedChannel} that must be closed when done\n\t *\n\t * @throws IOException\n\t * \t\tif acquiring the file lock fails\n\t */\n\tpublic LockedFileChannel lockForReading(final Path path) throws IOException {\n\n\t\treturn keyLockState(path, policy).acquireRead();\n\t}\n\n\t/**\n\t * Acquires a write lock for the specified key. Only one thread can hold a\n\t * write lock for a key at a time, and no readers can hold locks.\n\t *\n\t * @param path\n\t * \t\tthe file path to lock for writing\n\t *\n\t * @return a {@link LockedChannel} that must be closed when done\n\t *\n\t * @throws IOException\n\t * \t\tif acquiring the file lock fails\n\t */\n\tpublic LockedFileChannel lockForWriting(final Path path) throws IOException {\n\n\t\treturn keyLockState(path, policy).acquireWrite();\n\t}\n\n\t/**\n\t * Returns the number of keys currently being tracked.\n\t *\n\t * @return the number of keys with associated locks\n\t */\n\tint size() {\n\n\t\treturn locks.size();\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.UncheckedIOException;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.nio.ByteBuffer;\nimport java.nio.file.DirectoryNotEmptyException;\nimport java.nio.file.FileAlreadyExistsException;\nimport java.nio.file.FileSystemException;\nimport java.nio.file.Files;\nimport java.nio.file.NoSuchFileException;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.attribute.FileAttribute;\nimport java.util.Arrays;\nimport java.util.Comparator;\nimport java.util.Iterator;\nimport java.util.stream.Stream;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;\nimport org.janelia.saalfeldlab.n5.readdata.LazyRead;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\n\n/**\n * Filesystem {@link KeyValueAccess}.\n *\n * @author Stephan Saalfeld\n * @author Igor Pisarev\n * @author Philipp Hanslovsky\n */\npublic class FileSystemKeyValueAccess implements KeyValueAccess {\n\n\tprivate final FileKeyLockManager fileKeyLockManager;\n\n\tpublic FileSystemKeyValueAccess() {\n\t\tfinal LockingPolicy policy = LockingPolicy.fromString(System.getProperty(\"n5.ioPolicy\", \"permissive\"));\n\t\tthis.fileKeyLockManager = FileKeyLockManager.forPolicy(policy);\n\t}\n\n\tprivate LockedFileChannel lockForReading(final Path path) throws N5IOException {\n\n\t\ttry {\n\t\t\treturn fileKeyLockManager.lockForReading(path);\n\t\t} catch (final NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to lock file for reading: \" + path, e);\n\t\t}\n\t}\n\n\tprivate LockedFileChannel lockForWriting(final Path path) throws N5IOException {\n\n\t\ttry {\n\t\t\treturn fileKeyLockManager.lockForWriting(path);\n\t\t} catch (final NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to lock file for writing: \" + path, e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic VolatileReadData createReadData(final String normalPath) {\n\t\treturn VolatileReadData.from(new FileLazyRead(Paths.get(normalPath)));\n\t}\n\n\t@Override\n\tpublic void write(final String normalPath, final ReadData data) throws N5IOException {\n\t\tfinal Path path = Paths.get(normalPath);\n\t\ttry (final LockedFileChannel channel = lockForWriting(path)) {\n\t\t\tdata.writeTo(channel.asOutputStream());\n\t\t} catch (IOException e) {\n\t\t\tthrow new N5IOException(e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic boolean isDirectory(final String normalPath) {\n\n\t\tfinal Path path = Paths.get(normalPath);\n\t\treturn Files.isDirectory(path);\n\t}\n\n\t@Override\n\tpublic boolean isFile(final String normalPath) {\n\n\t\tfinal Path path = Paths.get(normalPath);\n\t\treturn Files.isRegularFile(path);\n\t}\n\n\t@Override\n\tpublic boolean exists(final String normalPath) {\n\n\t\tfinal Path path = Paths.get(normalPath);\n\t\treturn Files.exists(path);\n\t}\n\n\t@Override\n\tpublic long size(final String normalPath) {\n\n\t\treturn size(Paths.get(normalPath));\n\t}\n\n\tprivate static long size(final Path path) {\n\n\t\ttry {\n\t\t\treturn Files.size(path);\n\t\t} catch (NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic String[] listDirectories(final String normalPath) throws N5IOException {\n\n\t\tfinal Path path = Paths.get(normalPath);\n\t\ttry (final Stream pathStream = Files.list(path)) {\n\t\t\treturn pathStream\n\t\t\t\t\t.filter(Files::isDirectory)\n\t\t\t\t\t.map(a -> path.relativize(a).toString())\n\t\t\t\t\t.toArray(String[]::new);\n\t\t} catch (NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to list directories\", e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic String[] list(final String normalPath) throws N5IOException {\n\n\t\tfinal Path path = Paths.get(normalPath);\n\t\ttry (final Stream pathStream = Files.list(path)) {\n\t\t\treturn pathStream\n\t\t\t\t\t.map(a -> path.relativize(a).toString())\n\t\t\t\t\t.toArray(String[]::new);\n\t\t} catch (NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to list files\", e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic String[] components(final String path) {\n\n\t\tfinal Path fsPath = Paths.get(path);\n\t\tfinal Path root = fsPath.getRoot();\n\t\tfinal String separator = fsPath.getFileSystem().getSeparator();\n\t\tfinal String[] components;\n\t\tint o;\n\t\tif (root == null) {\n\t\t\tcomponents = new String[fsPath.getNameCount()];\n\t\t\to = 0;\n\t\t} else {\n\t\t\tcomponents = new String[fsPath.getNameCount() + 1];\n\t\t\tcomponents[0] = root.toString();\n\t\t\to = 1;\n\t\t}\n\n\t\tfor (int i = o; i < components.length; ++i) {\n\t\t\tString name = fsPath.getName(i - o).toString();\n\t\t\t/* Preserve trailing slash on final component if present*/\n\t\t\tif (i == components.length - 1) {\n\t\t\t\tfinal String trailingSeparator = path.endsWith(separator) ? separator : path.endsWith(\"/\") ? \"/\" : \"\";\n\t\t\t\tname += trailingSeparator;\n\t\t\t}\n\t\t\tcomponents[i] = name;\n\t\t}\n\t\treturn components;\n\t}\n\n\t@Override\n\tpublic String parent(final String path) {\n\n\t\tfinal Path parent = Paths.get(path).getParent();\n\t\tif (parent == null)\n\t\t\treturn null;\n\t\telse\n\t\t\treturn parent.toString();\n\t}\n\n\t@Override\n\tpublic String relativize(final String path, final String base) {\n\n\t\tfinal Path basePath = Paths.get(base);\n\t\treturn basePath.relativize(Paths.get(path)).toString();\n\t}\n\n\t/**\n\t * Returns a normalized path. It ensures correctness on both Unix and\n\t * Windows,\n\t * otherwise {@code pathName} is treated as UNC path on Windows, and\n\t * {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}.\n\t *\n\t * @param path the path\n\t * @return the normalized path, without leading slash\n\t */\n\t@Override\n\tpublic String normalize(final String path) {\n\n\t\treturn Paths.get(path).normalize().toString();\n\t}\n\n\t@Override\n\tpublic URI uri(final String normalPath) throws URISyntaxException {\n\n\t\t// normalize make absolute the scheme specific part only\n\t\ttry {\n\t\t\tfinal URI normalUri = URI.create(normalPath);\n\t\t\tif (normalUri.isAbsolute()) return normalUri.normalize();\n\t\t} catch (final IllegalArgumentException e) {\n\t\t\treturn new File(normalPath).toURI().normalize();\n\t\t}\n\t\treturn new File(normalPath).toURI().normalize();\n\n\t}\n\n\t@Override\n\tpublic String compose(final String... components) {\n\n\t\tif (components == null || components.length == 0)\n\t\t\treturn null;\n\t\tif (components.length == 1)\n\t\t\treturn Paths.get(components[0]).toString();\n\t\treturn Paths.get(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString();\n\t}\n\n\t@Override\n\tpublic String compose(URI uri, String... components) {\n\n\t\tPath composedPath;\n\t\tif (uri.isAbsolute())\n\t\t\tcomposedPath = Paths.get(uri);\n\t\telse\n\t\t\tcomposedPath = Paths.get(uri.toString());\n\t\tfor (String component : components) {\n\t\t\tif (component == null || component.isEmpty())\n\t\t\t\tcontinue;\n\t\t\tcomposedPath = composedPath.resolve(component);\n\t\t}\n\n\t\treturn composedPath.toAbsolutePath().toString();\n\t}\n\n\t@Override\n\tpublic void createDirectories(final String normalPath) throws N5IOException {\n\n\t\ttry {\n\t\t\tcreateDirectories(Paths.get(normalPath));\n\t\t} catch (NoSuchFileException e) {\n\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to create directories\", e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic void delete(final String normalPath) throws N5IOException {\n\n\t\ttry {\n\t\t\tfinal Path path = Paths.get(normalPath);\n\n\t\t\tif (Files.isRegularFile(path))\n\t\t\t\ttry (final LockedFileChannel channel = lockForWriting(path)) {\n\t\t\t\t\tFiles.delete(path);\n\t\t\t\t}\n\t\t\telse {\n\t\t\t\ttry (final Stream pathStream = Files.walk(path)) {\n\t\t\t\t\tfor (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) {\n\t\t\t\t\t\tfinal Path childPath = i.next();\n\t\t\t\t\t\tif (Files.isRegularFile(childPath))\n\t\t\t\t\t\t\ttry (final LockedFileChannel channel = lockForWriting(childPath)) {\n\t\t\t\t\t\t\t\tFiles.delete(childPath);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\telse\n\t\t\t\t\t\t\ttryDelete(childPath);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} catch (NoSuchFileException ignore) {\n\t\t\t/* It doesn't exist; that's sufficient for us to not complain on a `delete` call */\n\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\"Failed to delete file at \" + normalPath, e);\n\t\t}\n\t}\n\n\tprotected static void tryDelete(final Path path) throws IOException {\n\n\t\ttry {\n\t\t\tFiles.delete(path);\n\t\t} catch (final DirectoryNotEmptyException e) {\n\t\t\t/*\n\t\t\t * Even though path is expected to be an empty directory, sometimes\n\t\t\t * deletion fails on network filesystems when lock files are not\n\t\t\t * cleared immediately after the leaves have been removed.\n\t\t\t */\n\t\t\ttry {\n\t\t\t\t/* wait and reattempt */\n\t\t\t\tThread.sleep(100);\n\t\t\t\tFiles.delete(path);\n\t\t\t} catch (final InterruptedException ex) {\n\t\t\t\te.printStackTrace();\n\t\t\t\tThread.currentThread().interrupt();\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)}\n\t * that follows symlinks.\n\t *\n\t * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464\n\t *\n\t * Creates a directory by creating all nonexistent parent directories first.\n\t * Unlike the {@link Files#createDirectories} method, an exception\n\t * is not thrown if the directory could not be created because it already\n\t * exists.\n\t *\n\t * \n\t * The {@code attrs} parameter is optional {@link FileAttribute\n\t * file-attributes} to set atomically when creating the nonexistent\n\t * directories. Each file attribute is identified by its {@link\n\t * FileAttribute#name name}. If more than one attribute of the same name is\n\t * included in the array then all but the last occurrence is ignored.\n\t *\n\t *
\n\t * If this method fails, then it may do so after creating some, but not\n\t * all, of the parent directories.\n\t *\n\t * @param dir\n\t * the directory to create\n\t *\n\t * @param attrs\n\t * an optional list of file attributes to set atomically when\n\t * creating the directory\n\t *\n\t * @return the directory\n\t *\n\t * @throws UnsupportedOperationException\n\t * if the array contains an attribute that cannot be set\n\t * atomically\n\t * when creating the directory\n\t * @throws FileAlreadyExistsException\n\t * if {@code dir} exists but is not a directory (optional\n\t * specific\n\t * exception)\n\t * @throws IOException\n\t * if an I/O error occurs\n\t * @throws SecurityException\n\t * in the case of the default provider, and a security manager\n\t * is\n\t * installed, the {@link SecurityManager#checkWrite(String)\n\t * checkWrite}\n\t * method is invoked prior to attempting to create a directory\n\t * and\n\t * its {@link SecurityManager#checkRead(String) checkRead} is\n\t * invoked for each parent directory that is checked. If {@code\n\t * dir} is not an absolute path then its {@link Path#toAbsolutePath\n\t * toAbsolutePath} may need to be invoked to get its absolute\n\t * path.\n\t * This may invoke the security manager's {@link\n\t * SecurityManager#checkPropertyAccess(String)\n\t * checkPropertyAccess}\n\t * method to check access to the system property\n\t * {@code user.dir}\n\t */\n\tprotected static Path createDirectories(Path dir, final FileAttribute... attrs) throws IOException {\n\n\t\t// attempt to create the directory\n\t\ttry {\n\t\t\tcreateAndCheckIsDirectory(dir, attrs);\n\t\t\treturn dir;\n\t\t} catch (final FileAlreadyExistsException x) {\n\t\t\t// file exists and is not a directory\n\t\t\tthrow x;\n\t\t} catch (final IOException x) {\n\t\t\t// parent may not exist or other reason\n\t\t}\n\t\tSecurityException se = null;\n\t\ttry {\n\t\t\tdir = dir.toAbsolutePath();\n\t\t} catch (final SecurityException x) {\n\t\t\t// don't have permission to get absolute path\n\t\t\tse = x;\n\t\t}\n\t\t// find a descendant that exists\n\t\tPath parent = dir.getParent();\n\t\twhile (parent != null) {\n\t\t\ttry {\n\t\t\t\tparent.getFileSystem().provider().checkAccess(parent);\n\t\t\t\tbreak;\n\t\t\t} catch (final NoSuchFileException x) {\n\t\t\t\t// does not exist\n\t\t\t}\n\t\t\tparent = parent.getParent();\n\t\t}\n\t\tif (parent == null) {\n\t\t\t// unable to find existing parent\n\t\t\tif (se == null) {\n\t\t\t\tthrow new FileSystemException(\n\t\t\t\t\t\tdir.toString(),\n\t\t\t\t\t\tnull,\n\t\t\t\t\t\t\"Unable to determine if root directory exists\");\n\t\t\t} else {\n\t\t\t\tthrow se;\n\t\t\t}\n\t\t}\n\n\t\t// create directories\n\t\tPath child = parent;\n\t\tfor (final Path name : parent.relativize(dir)) {\n\t\t\tchild = child.resolve(name);\n\t\t\tcreateAndCheckIsDirectory(child, attrs);\n\t\t}\n\t\treturn dir;\n\t}\n\n\t/**\n\t * This is a copy of a previous Files#createAndCheckIsDirectory(Path,\n\t * FileAttribute...) method that follows symlinks.\n\t *\n\t * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464\n\t *\n\t * Used by createDirectories to attempt to create a directory. A no-op if the\n\t * directory already exists.\n\t *\n\t * @param dir directory path\n\t * @param attrs file attributes\n\t * @throws IOException the exception\n\t */\n\tprotected static void createAndCheckIsDirectory(\n\t\t\tfinal Path dir,\n\t\t\tfinal FileAttribute... attrs) throws IOException {\n\n\t\ttry {\n\t\t\tFiles.createDirectory(dir, attrs);\n\t\t} catch (final FileAlreadyExistsException x) {\n\t\t\tif (!Files.isDirectory(dir))\n\t\t\t\tthrow x;\n\t\t}\n\t}\n\n\t/**\n\t * Verify that the range {@code [offset, offset+length)} is fully contained in {@code [0, channelSize)}.\n\t *\n\t * @throws IndexOutOfBoundsException\n\t * \t\tif range is not fully contained\n\t */\n\tprivate static void validBounds(final long channelSize, final long offset, final long length) throws IndexOutOfBoundsException {\n\n\t\tif (offset < 0)\n\t\t\tthrow new IndexOutOfBoundsException(\"offset must be > 0, but was: \" + offset);\n\t\telse if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay\n\t\t\tthrow new IndexOutOfBoundsException(\"offset (\" + offset + \") must be less than channel size (\" + channelSize + \")\");\n\t\telse if (length >= 0 && offset + length > channelSize)\n\t\t\tthrow new IndexOutOfBoundsException(\"offset + length (\" + (offset + length) + \") must be less than channel size (\" + channelSize + \")\");\n\t}\n\n\tprivate class FileLazyRead implements LazyRead {\n\n\t\tprivate final Path path;\n\t\tprivate LockedFileChannel lock; // TODO rename\n\n\t\tFileLazyRead(final Path path) {\n\t\t\tthis.path = path;\n\t\t\tlock = lockForReading(path);\n\t\t}\n\n\t\t@Override\n\t\tpublic long size() throws N5IOException {\n\n\t\t\tif (lock == null) {\n\t\t\t\tthrow new N5IOException(\"FileLazyRead is already closed.\");\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\treturn Files.size(path);\n\t\t\t} catch (NoSuchFileException e) {\n\t\t\t\tthrow new N5NoSuchKeyException(\"No such file\", e);\n\t\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\t\tthrow new N5IOException(e);\n\t\t\t}\n\t\t}\n\n\t\t@Override\n\t\tpublic ReadData materialize(final long offset, final long length) {\n\n\t\t\tif (lock == null) {\n\t\t\t\tthrow new N5IOException(\"FileLazyRead is already closed.\");\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\tfinal long channelSize = lock.size();\n\t\t\t\tvalidBounds(channelSize, offset, length);\n\n\t\t\t\tfinal long size = length < 0 ? (channelSize - offset) : length;\n\t\t\t\tif (size > Integer.MAX_VALUE) {\n\t\t\t\t\tthrow new IndexOutOfBoundsException(\"Attempt to materialize too large data\");\n\t\t\t\t}\n\n\t\t\t\tfinal byte[] data = new byte[(int) size];\n\t\t\t\tlock.read(ByteBuffer.wrap(data), offset);\n\t\t\t\treturn ReadData.from(data);\n\t\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\t\tthrow new N5Exception.N5IOException(e);\n\t\t\t}\n\t\t}\n\n\t\t@Override\n\t\tpublic void close() throws IOException {\n\n\t\t\tif (lock != null) {\n\t\t\t\tlock.close();\n\t\t\t\tlock = null;\n\t\t\t}\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class FloatArrayDataBlock extends AbstractDataBlock {\n\n\tpublic FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) {\n\n\t\tsuper(size, gridPosition, data, a -> a.length);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.readdata.LazyRead;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.io.UncheckedIOException;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.Channels;\nimport java.nio.channels.FileChannel;\nimport java.nio.file.*;\n\nimport static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER;\n\npublic class FsIoPolicy {\n\n static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe());\n\n private static boolean validBounds(long channelSize, long offset, long length) {\n\n\t\tif (offset < 0)\n\t\t\tthrow new N5Exception(\"offset must be > 0, but was: \" + offset);\n\t\telse if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay\n\t\t\tthrow new N5Exception(\"offset (\" + offset + \") must be less than channel size (\" + channelSize + \")\");\n\t\telse if (length >= 0 && offset + length > channelSize)\n\t\t\tthrow new N5Exception(\"offset + length (\" + (offset + length) + \") must be less than channel size (\" + channelSize + \")\");\n\n return true;\n }\n\n /**\n * Opens a file channel. If the channel is opened {@code forWriting},\n * then this may create the file and the parent directories as needed.\n *\n * @throws IOException\n * \t\tif the channel cannot be opened\n */\n static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException {\n\n if (forWriting) {\n final Path parent = path.getParent();\n /* if not null and not directory, it will call `createDirectories` but we expect it to throw an IOException */\n if (parent != null && !parent.toFile().isDirectory()) {\n Files.createDirectories(parent);\n }\n return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE);\n } else {\n return FileChannel.open(path, StandardOpenOption.READ);\n }\n }\n\n\n /**\n * This method is necessary to handle the situtation where writing is successful, but `close` fails on the file channel.\n * This has been observed to happen fairly consistently on MacOS when writing to a file mounted over SMB.\n *\n * @param readData to write to the {@code Path}\n * @param path to write to\n * @throws IOException if writing failed.\n */\n private static void writeToPathIgnoreCloseException(ReadData readData, Path path) throws IOException {\n\n FileChannel channel = openFileChannel(path, true);\n OutputStream os = Channels.newOutputStream(channel);\n\n try {\n readData.writeTo(os);\n os.flush();\n channel.force(true);\n } catch (Throwable e) {\n os.close();\n channel.close();\n throw e;\n }\n\n /* if we get here, the write succeeded, and the os/channel may not be closed yet */\n try {\n os.close();\n channel.close();\n } catch (IOException | UncheckedIOException ignore) {\n /* Ignore; we know the data was written already. */\n }\n }\n\n public static class Unsafe implements IoPolicy {\n @Override\n public void write(String key, ReadData readData) throws IOException {\n final Path path = Paths.get(key);\n writeToPathIgnoreCloseException(readData, path);\n }\n\n @Override\n public VolatileReadData read(final String key) throws IOException {\n final Path path = Paths.get(key);\n FileLazyRead fileLazyRead = new FileLazyRead(path, false);\n return VolatileReadData.from(fileLazyRead);\n }\n\n @Override\n public void delete(final String key) throws IOException {\n final Path path = Paths.get(key);\n Files.deleteIfExists(path);\n }\n }\n\n public static class Atomic implements IoPolicy {\n @Override\n public void write(String key, ReadData readData) throws IOException {\n final Path path = Paths.get(key);\n try (LockedFileChannel channel = FILE_LOCK_MANAGER.lockForWriting(path)) {\n readData.writeTo(channel.asOutputStream());\n }\n }\n\n @Override\n public VolatileReadData read(String key) throws IOException {\n final Path path = Paths.get(key);\n FileLazyRead fileLazyRead = new FileLazyRead(path, true);\n return VolatileReadData.from(fileLazyRead);\n }\n\n @Override\n public void delete(final String key) throws IOException {\n final Path path = Paths.get(key);\n if (!Files.isRegularFile(path))\n Files.delete(path);\n try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) {\n Files.delete(path);\n }\n }\n }\n\n static class FileLazyRead implements LazyRead {\n\n private static final Closeable NO_OP = () -> { };\n\n private final Path path;\n private Closeable lock;\n\n FileLazyRead(final Path path) throws IOException {\n this(path, true);\n }\n\n FileLazyRead(final Path path, final boolean requireLock ) throws IOException {\n this.path = path;\n if (requireLock)\n lock = FILE_LOCK_MANAGER.lockForReading(path);\n else\n lock = NO_OP;\n }\n\n @Override\n public long size() throws N5Exception.N5IOException {\n\n if (lock == null) {\n throw new N5Exception.N5IOException(\"FileLazyRead is already closed.\");\n }\n\n\t\t\ttry {\n\t\t\t\treturn Files.size(path);\n\t\t\t} catch (NoSuchFileException e) {\n\t\t\t\tthrow new N5Exception.N5NoSuchKeyException(\"No such file\", e);\n\t\t\t} catch (IOException | UncheckedIOException e) {\n\t\t\t\tthrow new N5Exception.N5IOException(e);\n\t\t\t}\n }\n\n @Override\n public ReadData materialize(final long offset, final long length) {\n\n if (lock == null) {\n throw new N5Exception.N5IOException(\"FileLazyRead is already closed.\");\n }\n\n ReadData readData = null;\n try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) {\n\n channel.position(offset);\n\n final long channelSize = channel.size();\n if (!validBounds(channelSize, offset, length)) {\n throw new IndexOutOfBoundsException();\n }\n\n final long size = length < 0 ? (channelSize - offset) : length;\n if (size > Integer.MAX_VALUE) {\n throw new IndexOutOfBoundsException(\"Attempt to materialize too large data\");\n }\n\n final byte[] data = new byte[(int) size];\n final ByteBuffer buf = ByteBuffer.wrap(data);\n channel.read(buf);\n readData = ReadData.from(data);\n\n } catch (final NoSuchFileException e) {\n throw new N5Exception.N5NoSuchKeyException(\"No such file\", e);\n } catch (IOException | UncheckedIOException e) {\n /* Occasionally (frequently for some source remote mounted file systems) this can throw exceptions during\n * `channel.close()` which is called automatically in the try-with-resources block. In this case, we have\n * successfully read the data, and we can return it, and ignore the exception.\n * */\n if (readData == null)\n throw new N5Exception.N5IOException(e);\n }\n return readData;\n }\n\n @Override\n public void close() throws IOException {\n\n if (lock != null) {\n lock.close();\n lock = null;\n }\n }\n }\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.InputStreamReader;\nimport java.io.UncheckedIOException;\nimport java.util.List;\n\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\nimport org.janelia.saalfeldlab.n5.shard.PositionValueAccess;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\n\n/**\n * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON\n * attributes parsed with {@link Gson}.\n *\n */\npublic interface GsonKeyValueN5Reader extends GsonN5Reader {\n\n\tKeyValueAccess getKeyValueAccess();\n\n\tdefault boolean groupExists(final String normalPath) {\n\n\t\treturn getKeyValueAccess().isDirectory(absoluteGroupPath(normalPath));\n\t}\n\n\t@Override\n\tdefault boolean exists(final String pathName) {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\treturn groupExists(normalPath) || datasetExists(normalPath);\n\t}\n\n\t@Override\n\tdefault boolean datasetExists(final String pathName) throws N5Exception {\n\n\t\t// for n5, every dataset must be a group\n\t\treturn getDatasetAttributes(pathName) != null;\n\t}\n\n\t/**\n\t * Reads or creates the attributes map of a group or dataset.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return the attribute\n\t * @throws N5Exception if the attributes cannot be read\n\t */\n\t@Override\n\tdefault JsonElement getAttributes(final String pathName) throws N5Exception {\n\n\t\tfinal String groupPath = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String attributesPath = absoluteAttributesPath(groupPath);\n\n\t\ttry (final VolatileReadData readData = getKeyValueAccess().createReadData(attributesPath);) {\n\t\t\tif (readData == null) {\n\t\t\t\treturn null;\n\t\t\t}\n\t\t\treturn GsonUtils.readAttributes(new InputStreamReader(readData.inputStream()), getGson());\n\t\t} catch (final N5Exception.N5NoSuchKeyException e) {\n\t\t\treturn null;\n\t\t} catch (final UncheckedIOException | N5IOException e) {\n\t\t\tthrow new N5IOException(\"Failed to read attributes from dataset \" + pathName, e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault DataBlock readChunk(\n\t\t\tfinal String pathName,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),\n\t\t\t\t\tconvertedDatasetAttributes);\n\t\t\treturn convertedDatasetAttributes. getDatasetAccess().readChunk(posKva, gridPosition);\n\n\t\t} catch (N5Exception.N5NoSuchKeyException e) {\n\t\t\treturn null;\n\t\t}\n\t}\n\n\t@Override\n\tdefault List> readChunks(\n\t\t\tfinal String pathName,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal List blockPositions) throws N5Exception {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes);\n\t\treturn convertedDatasetAttributes. getDatasetAccess().readChunks(posKva, blockPositions);\n\t}\n\n\t@Override\n\tdefault DataBlock readBlock(\n\t\t\tfinal String pathName,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tfinal DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tfinal int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),\n\t\t\t\t\tconvertedDatasetAttributes);\n\t\t\treturn convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition, shardLevel);\n\n\t\t} catch (N5Exception.N5NoSuchKeyException e) {\n\t\t\treturn null;\n\t\t}\n\t}\n\n\t@Override\n\tdefault String[] list(final String pathName) throws N5Exception {\n\n\t\treturn getKeyValueAccess().listDirectories(absoluteGroupPath(pathName));\n\t}\n\n\t/**\n\t * Constructs the absolute path (in terms of this store) for the group or\n\t * dataset.\n\t *\n\t * @param normalGroupPath\n\t * normalized group path without leading slash\n\t * @return the absolute path to the group\n\t */\n\tdefault String absoluteGroupPath(final String normalGroupPath) {\n\n\t\treturn getKeyValueAccess().compose(getURI(), normalGroupPath);\n\t}\n\n\t/**\n\t * Constructs the absolute path (in terms of this store) for the attributes\n\t * file of a group or dataset.\n\t *\n\t * @param normalPath\n\t * normalized group path without leading slash\n\t * @return the absolute path to the attributes\n\t */\n\tdefault String absoluteAttributesPath(final String normalPath) {\n\n\t\treturn getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());\n\t}\n\n\t@Override\n\tdefault boolean blockExists(\n\t\t\tfinal String pathName,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String blockPath = getKeyValueAccess().compose(getURI(), normalPath,\n\t\t\t\tdatasetAttributes.relativeBlockPath(gridPosition));\n\t\treturn getKeyValueAccess().isFile(blockPath);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.OutputStreamWriter;\nimport java.io.UncheckedIOException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\n\nimport com.google.gson.JsonSyntaxException;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.shard.PositionValueAccess;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\n\n/**\n * Default implementation of {@link N5Writer} with JSON attributes parsed with\n * {@link Gson}.\n */\npublic interface GsonKeyValueN5Writer extends GsonN5Writer, GsonKeyValueN5Reader {\n\n\t/**\n\t * TODO This overrides the version even if incompatible, check\n\t * if this is the desired behavior or if it is always overridden, e.g. as by\n\t * the caching version. If this is true, delete this implementation.\n\t *\n\t * @param path to the group to write the version into\n\t */\n\tdefault void setVersion(final String path) {\n\n\t\tif (!VERSION.equals(getVersion()))\n\t\t\tsetAttribute(\"/\", VERSION_KEY, VERSION.toString());\n\t}\n\n\tstatic String initializeContainer(\n\t\t\tfinal KeyValueAccess keyValueAccess,\n\t\t\tfinal String basePath) throws N5IOException {\n\n\t\tfinal String normBasePath = keyValueAccess.normalize(basePath);\n\t\tkeyValueAccess.createDirectories(normBasePath);\n\t\treturn normBasePath;\n\t}\n\n\t@Override\n\tdefault void createGroup(final String path) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\tgetKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));\n\t}\n\n\t/**\n\t * Helper method that writes an attributes tree into the store\n\t * \n\t * TODO This method is not part of the public API and should be protected\n\t * in Java versions greater than 8\n\t *\n\t * @param normalGroupPath\n\t * to write the attributes to\n\t * @param attributes\n\t * to write\n\t * @throws N5Exception\n\t * if unable to write the attributes at {@code normalGroupPath}\n\t */\n\tdefault void writeAttributes(\n\t\t\tfinal String normalGroupPath,\n\t\t\tfinal JsonElement attributes) throws N5Exception {\n\n\t\tfinal ReadData newAttributesReadData = ReadData.from(os -> {\n\t\t\tfinal OutputStreamWriter writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);\n\t\t\tGsonUtils.writeAttributes(writer, attributes, getGson());\n\t\t});\n\n\t\ttry {\n\t\t\tgetKeyValueAccess().write(absoluteAttributesPath(normalGroupPath), newAttributesReadData);\n\t\t} catch (UncheckedIOException | N5IOException e) {\n\t\t\tthrow new N5Exception.N5IOException(\"Failed to write attributes into \" + normalGroupPath, e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void setAttributes(\n\t\t\tfinal String path,\n\t\t\tfinal JsonElement attributes) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\tif (!exists(normalPath))\n\t\t\tthrow new N5IOException(\"\" + normalPath + \" is not a group or dataset.\");\n\n\t\twriteAttributes(normalPath, attributes);\n\t}\n\n\t/**\n\t * Helper method that reads the existing map of attributes, JSON encodes,\n\t * inserts and overrides the provided attributes, and writes them back into\n\t * the attributes store.\n\t *\n\t * TODO This method is not part of the public API and should be protected\n\t * in Java greater than 8\n\t *\n\t * @param normalGroupPath\n\t * to write the attributes to\n\t * @param attributes\n\t * to write\n\t * @throws N5Exception\n\t * if unable to read or write the attributes at\n\t * {@code normalGroupPath}\n\t */\n\tdefault void writeAttributes(\n\t\t\tfinal String normalGroupPath,\n\t\t\tfinal Map attributes) throws N5Exception {\n\n\t\tif (attributes != null && !attributes.isEmpty()) {\n\t\t\tJsonElement root = getAttributes(normalGroupPath);\n\t\t\troot = root != null && root.isJsonObject()\n\t\t\t\t\t? root.getAsJsonObject()\n\t\t\t\t\t: new JsonObject();\n\t\t\troot = GsonUtils.insertAttributes(root, attributes, getGson());\n\t\t\twriteAttributes(normalGroupPath, root);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void setAttributes(\n\t\t\tfinal String path,\n\t\t\tfinal Map attributes) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\tif (!exists(normalPath))\n\t\t\tthrow new N5IOException(\"\" + normalPath + \" is not a group or dataset.\");\n\n\t\twriteAttributes(normalPath, attributes);\n\t}\n\n\t@Override\n\tdefault boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(groupPath);\n\t\tfinal String absoluteNormalPath = getKeyValueAccess().compose(getURI(), normalPath);\n\t\tfinal String normalKey = N5URI.normalizeAttributePath(attributePath);\n\n\t\tif (!getKeyValueAccess().isDirectory(absoluteNormalPath))\n\t\t\treturn false;\n\n\t\tif (attributePath.equals(\"/\")) {\n\t\t\tsetAttributes(normalPath, JsonNull.INSTANCE);\n\t\t\treturn true;\n\t\t}\n\n\t\tfinal JsonElement attributes = getAttributes(normalPath);\n\t\tif (GsonUtils.removeAttribute(attributes, normalKey) != null) {\n\t\t\tsetAttributes(normalPath, attributes);\n\t\t\treturn true;\n\t\t}\n\t\treturn false;\n\t}\n\n\t@Override\n\tdefault T removeAttribute(final String pathName, final String key, final Class cls) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String normalKey = N5URI.normalizeAttributePath(key);\n\n\t\tfinal JsonElement attributes = getAttributes(normalPath);\n\t\tfinal T obj;\n\t\ttry {\n\t\t\tobj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson());\n\t\t} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {\n\t\t\tthrow new N5Exception.N5ClassCastException(e);\n\t\t}\n\t\tif (obj != null) {\n\t\t\tsetAttributes(normalPath, attributes);\n\t\t}\n\t\treturn obj;\n\t}\n\n\t@Override\n\tdefault boolean removeAttributes(final String pathName, final List attributes) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\tboolean removed = false;\n\t\tfor (final String attribute : attributes) {\n\t\t\tfinal String normalKey = N5URI.normalizeAttributePath(attribute);\n\t\t\tremoved |= removeAttribute(normalPath, normalKey);\n\t\t}\n\t\treturn removed;\n\t}\n\n\t@Override\n\tdefault void writeRegion(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long[] min,\n\t\t\tfinal long[] size,\n\t\t\tfinal DataBlockSupplier chunkSupplier,\n\t\t\tfinal boolean writeFully) throws N5Exception {\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);\n\t\t\tconvertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully);\n\t\t} catch (final UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\n\t\t\t\t\t\"Failed to write blocks into dataset \" + datasetPath, e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void writeRegion(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long[] min,\n\t\t\tfinal long[] size,\n\t\t\tfinal DataBlockSupplier chunkSupplier,\n\t\t\tfinal boolean writeFully,\n\t\t\tfinal ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException {\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);\n\t\t\tconvertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully, exec);\n\t\t} catch (final UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\n\t\t\t\t\t\"Failed to write blocks into dataset \" + datasetPath, e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void writeChunks(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal DataBlock... chunks) throws N5Exception {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);\n\t\t\tconvertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(chunks));\n\t\t} catch (final UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\n\t\t\t\t\t\"Failed to write chunks into dataset \" + datasetPath, e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void writeChunk(\n\t\t\tfinal String path,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal DataBlock chunk) throws N5Exception {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes);\n\t\t\tconvertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, chunk);\n\t\t} catch (final UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\n\t\t\t\t\t\"Failed to write chunk \" + Arrays.toString(chunk.getGridPosition()) + \" into dataset \" + path,\n\t\t\t\t\te);\n\t\t}\n\t}\n\n\t@Override\n\tdefault void writeBlock(\n\t\t\tfinal String path,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal DataBlock dataBlock) throws N5Exception {\n\n\t\tfinal DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tfinal int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;\n\t\ttry {\n\t\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes);\n\t\t\tconvertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock, shardLevel);\n\t\t} catch (final UncheckedIOException e) {\n\t\t\tthrow new N5IOException(\n\t\t\t\t\t\"Failed to write block \" + Arrays.toString(dataBlock.getGridPosition()) + \" into dataset \" + path,\n\t\t\t\t\te);\n\t\t}\n\t}\n\n\t@Override\n\tdefault boolean remove(final String path) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(path);\n\t\tfinal String groupPath = absoluteGroupPath(normalPath);\n\t\tif (getKeyValueAccess().isDirectory(groupPath))\n\t\t\tgetKeyValueAccess().delete(groupPath);\n\n\t\t/* an IOException should have occurred if anything had failed midway */\n\t\treturn true;\n\t}\n\n\t@Override\n\tdefault boolean deleteBlock(\n\t\t\tfinal String path,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);\n\t\treturn posKva.remove(gridPosition);\n\t}\n\n\t@Override\n\tdefault boolean deleteChunk(\n\t\t\tfinal String path,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);\n\t\treturn datasetAttributes.getDatasetAccess().deleteChunk(posKva, gridPosition);\n\t}\n\n\t@Override\n\tdefault boolean deleteChunks(\n\t\t\tfinal String path,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal List gridPositions) throws N5Exception {\n\n\t\tfinal PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);\n\t\treturn datasetAttributes.getDatasetAccess().deleteChunks(posKva, gridPositions);\n\t}\n}"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.lang.reflect.Type;\nimport java.util.Map;\n\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonParseException;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonSyntaxException;\n\n/**\n * {@link N5Reader} with JSON attributes parsed with {@link Gson}.\n *\n */\npublic interface GsonN5Reader extends N5Reader {\n\n\tGson getGson();\n\n\t/**\n\t * Get the key for the {@link KeyValueAccess}, that is used for storing attributes.\n\t * The N5 format uses \"attributes.json\".\n\t *\n\t * @return the attributes key\n\t */\n\tString getAttributesKey();\n\n\t@Override\n\tdefault Map> listAttributes(final String pathName) throws N5Exception {\n\n\t\treturn GsonUtils.listAttributes(getAttributes(pathName));\n\t}\n\n\t@Override\n\tdefault DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(pathName);\n\t\tfinal JsonElement attributes = getAttributes(normalPath);\n\t\treturn createDatasetAttributes(attributes);\n\t}\n\n\tdefault DatasetAttributes createDatasetAttributes(final JsonElement attributes) {\n\n\t\tfinal JsonDeserializationContext context = new JsonDeserializationContext() {\n\n\t\t\t@Override public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException {\n\n\t\t\t\treturn getGson().fromJson(json, typeOfT);\n\t\t\t}\n\t\t};\n\n\t\treturn DatasetAttributes.getJsonAdapter().deserialize(attributes, DatasetAttributes.class, context);\n\t}\n\n\t@Override\n\tdefault T getAttribute(final String pathName, final String key, final Class clazz) throws N5Exception {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String normalizedAttributePath = N5URI.normalizeAttributePath(key);\n\n\t\tfinal JsonElement attributes = getAttributes(normalPathName);\n\t\ttry {\n\t\t\treturn GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson());\n\t\t} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {\n\t\t\tthrow new N5Exception.N5ClassCastException(e);\n\t\t}\n\t}\n\n\t@Override\n\tdefault T getAttribute(final String pathName, final String key, final Type type) throws N5Exception {\n\n\t\tfinal String normalPathName = N5URI.normalizeGroupPath(pathName);\n\t\tfinal String normalizedAttributePath = N5URI.normalizeAttributePath(key);\n\t\tfinal JsonElement attributes = getAttributes(normalPathName);\n\t\ttry {\n\t\t\treturn GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson());\n\t\t} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {\n\t\t\tthrow new N5Exception.N5ClassCastException(e);\n\t\t}\n\t}\n\n\t/**\n\t * Reads or the attributes of a group or dataset.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return the attributes identified by pathName\n\t * @throws N5Exception if the attribute cannot be returned\n\t */\n\tJsonElement getAttributes(final String pathName) throws N5Exception;\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonElement;\n\n/**\n * {@link N5Writer} with JSON attributes parsed with {@link Gson}.\n *\n */\npublic interface GsonN5Writer extends GsonN5Reader, N5Writer {\n\n\t/**\n\t * Set the attributes of a group. This result of this method is equivalent\n\t * with {@link N5Writer#setAttribute(String, String, Object) N5Writer#setAttribute(groupPath, \"/\", attributes)}.\n\t *\n\t * @param groupPath\n\t * to write the attributes to\n\t * @param attributes\n\t * to write\n\t * @throws N5Exception if the attributes cannot be set\n\t */\n\tvoid setAttributes(\n\t\t\tfinal String groupPath,\n\t\t\tfinal JsonElement attributes) throws N5Exception;\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.io.Writer;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.Type;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.regex.Matcher;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.reflect.TypeToken;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException;\n\n/**\n * Utility class for working with JSON.\n *\n * @author Stephan Saalfeld\n */\npublic interface GsonUtils {\n\n\t/**\n\t * Reads the attributes json from a given {@link Reader}.\n\t *\n\t * @param reader\n\t * the reader\n\t * @param gson\n\t * \t\t\t to parse Json from the {@code reader}\n\t * @return the root {@link JsonObject} of the attributes\n\t */\n\tstatic JsonElement readAttributes(final Reader reader, final Gson gson) {\n\n\t\treturn gson.fromJson(reader, JsonElement.class);\n\t}\n\n\tstatic T readAttribute(\n\t\t\tfinal JsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal Class cls,\n\t\t\tfinal Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {\n\n\t\treturn readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson);\n\t}\n\n\tstatic T readAttribute(\n\t\t\tfinal JsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal Type type,\n\t\t\tfinal Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {\n\n\t\tfinal JsonElement attribute = getAttribute(root, normalizedAttributePath);\n\t\treturn parseAttributeElement(attribute, gson, type);\n\t}\n\n\t/**\n\t * Deserialize the {@code attribute} as {@link Type type} {@code T}.\n\t *\n\t * @param attribute\n\t * to deserialize as {@link Type type}\n\t * @param gson\n\t * used to deserialize {@code attribute}\n\t * @param type\n\t * to deserialize {@code attribute} as\n\t * @param \n\t * return type represented by {@link Type type}\n\t * @return the deserialized attribute object, or {@code null} if\n\t * {@code attribute} cannot deserialize to {@code T}\n\t */\n\t@SuppressWarnings(\"unchecked\")\n\tstatic T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) throws JsonSyntaxException, NumberFormatException, ClassCastException {\n\n\t\tif (attribute == null)\n\t\t\treturn null;\n\n\t\tfinal Class clazz = (type instanceof Class) ? ((Class)type) : null;\n\t\tif (clazz != null && clazz.isAssignableFrom(HashMap.class)) {\n\t\t\tfinal Type mapType = new TypeToken>() {\n\n\t\t\t}.getType();\n\t\t\tfinal Map retMap = gson.fromJson(attribute, mapType);\n\t\t\t// noinspection unchecked\n\t\t\treturn (T)retMap;\n\t\t}\n\t\tif (attribute instanceof JsonArray) {\n\t\t\tfinal JsonArray array = attribute.getAsJsonArray();\n\t\t\ttry {\n\t\t\t\tfinal T retArray = GsonUtils.getJsonAsArray(gson, array, type);\n\t\t\t\tif (retArray != null)\n\t\t\t\t\treturn retArray;\n\t\t\t} catch (final JsonSyntaxException e) {\n\t\t\t\tif (type == String.class)\n\t\t\t\t\t//noinspection unchecked\n\t\t\t\t\treturn (T)gson.toJson(attribute);\n\t\t\t}\n\t\t}\n\t\ttry {\n\t\t\tfinal T parsedResult = gson.fromJson(attribute, type);\n\t\t\tif (parsedResult == null)\n\t\t\t\tthrow new ClassCastException(\"Cannot parse json as type \" + type.getTypeName());\n\t\t\treturn parsedResult;\n\t\t} catch (final JsonSyntaxException e) {\n\t\t\tif (type == String.class)\n\t\t\t\t//noinspection unchecked\n\t\t\t\treturn (T)gson.toJson(attribute);\n\t\t\tthrow e;\n\t\t}\n\t}\n\n\t/**\n\t * Return the attribute at {@code normalizedAttributePath} as a\n\t * {@link JsonElement}. Does not attempt to parse the attribute.\n\t * to search for the {@link JsonElement} at location\n\t * {@code normalizedAttributePath}\n\t *\n\t * @param root\n\t * \t\t\tcontaining an attribute at normalizedAttributePath\n\t * @param normalizedAttributePath\n\t * to the attribute\n\t * @return the attribute as a {@link JsonElement}.\n\t */\n\tstatic JsonElement getAttribute(JsonElement root, final String normalizedAttributePath) {\n\n\t\tfinal String[] pathParts = normalizedAttributePath.split(\"(?= jsonArray.size()) {\n\t\t\t\t\t\treturn null;\n\t\t\t\t\t}\n\t\t\t\t\troot = jsonArray.get(index);\n\t\t\t\t} else {\n\t\t\t\t\treturn null;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn root;\n\t}\n\n\t/**\n\t * Best effort implementation of {@link N5Reader#listAttributes(String)}\n\t * with limited type resolution. Possible return types are\n\t * \n\t * - null
\n\t * - boolean
\n\t * - double
\n\t * - String
\n\t * - Object
\n\t * - boolean[]
\n\t * - double[]
\n\t * - String[]
\n\t * - Object[]
\n\t *
\n\t *\n\t * @param root\n\t * the json element\n\t * @return the attribute map\n\t */\n\tstatic Map> listAttributes(final JsonElement root) throws N5JsonParseException {\n\n\t\tif (root == null) {\n\t\t\treturn new HashMap<>();\n\t\t}\n\t\tif (!root.isJsonObject()) {\n\t\t\tthrow new N5JsonParseException(\"JsonElement found, but was not JsonObject\");\n\t\t}\n\n\t\tfinal HashMap> attributes = new HashMap<>();\n\t\troot.getAsJsonObject().entrySet().forEach(entry -> {\n\t\t\tfinal Class clazz;\n\t\t\tfinal String key = entry.getKey();\n\t\t\tfinal JsonElement jsonElement = entry.getValue();\n\t\t\tif (jsonElement.isJsonNull())\n\t\t\t\tclazz = null;\n\t\t\telse if (jsonElement.isJsonPrimitive())\n\t\t\t\tclazz = classForJsonPrimitive((JsonPrimitive)jsonElement);\n\t\t\telse if (jsonElement.isJsonArray()) {\n\t\t\t\tfinal JsonArray jsonArray = (JsonArray)jsonElement;\n\t\t\t\tClass arrayElementClass = Object.class;\n\t\t\t\tif (jsonArray.size() > 0) {\n\t\t\t\t\tfinal JsonElement firstElement = jsonArray.get(0);\n\t\t\t\t\tif (firstElement.isJsonPrimitive()) {\n\t\t\t\t\t\tarrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive());\n\t\t\t\t\t\tfor (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) {\n\t\t\t\t\t\t\tfinal JsonElement element = jsonArray.get(i);\n\t\t\t\t\t\t\tif (element.isJsonPrimitive()) {\n\t\t\t\t\t\t\t\tfinal Class nextArrayElementClass = classForJsonPrimitive(\n\t\t\t\t\t\t\t\t\t\telement.getAsJsonPrimitive());\n\t\t\t\t\t\t\t\tif (nextArrayElementClass != arrayElementClass)\n\t\t\t\t\t\t\t\t\tif (nextArrayElementClass == double.class && arrayElementClass == long.class)\n\t\t\t\t\t\t\t\t\t\tarrayElementClass = double.class;\n\t\t\t\t\t\t\t\t\telse {\n\t\t\t\t\t\t\t\t\t\tarrayElementClass = Object.class;\n\t\t\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tarrayElementClass = Object.class;\n\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tclazz = Array.newInstance(arrayElementClass, 0).getClass();\n\t\t\t\t} else\n\t\t\t\t\tclazz = Object[].class;\n\t\t\t} else\n\t\t\t\tclazz = Object.class;\n\t\t\tattributes.put(key, clazz);\n\t\t});\n\t\treturn attributes;\n\t}\n\n\tstatic T getJsonAsArray(final Gson gson, final JsonArray array, final Class cls) {\n\n\t\treturn getJsonAsArray(gson, array, TypeToken.get(cls).getType());\n\t}\n\n\t@SuppressWarnings(\"unchecked\")\n\tstatic T getJsonAsArray(final Gson gson, final JsonArray array, final Type type) {\n\n\t\tfinal Class clazz = (type instanceof Class) ? ((Class)type) : null;\n\n\t\tif (type == boolean[].class) {\n\t\t\tfinal boolean[] retArray = new boolean[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal Boolean value = gson.fromJson(array.get(i), boolean.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == double[].class) {\n\t\t\tfinal double[] retArray = new double[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal double value = gson.fromJson(array.get(i), double.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == float[].class) {\n\t\t\tfinal float[] retArray = new float[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal float value = gson.fromJson(array.get(i), float.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == long[].class) {\n\t\t\tfinal long[] retArray = new long[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal long value = gson.fromJson(array.get(i), long.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == short[].class) {\n\t\t\tfinal short[] retArray = new short[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal short value = gson.fromJson(array.get(i), short.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == int[].class) {\n\t\t\tfinal int[] retArray = new int[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal int value = gson.fromJson(array.get(i), int.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == byte[].class) {\n\t\t\tfinal byte[] retArray = new byte[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal byte value = gson.fromJson(array.get(i), byte.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (type == char[].class) {\n\t\t\tfinal char[] retArray = new char[array.size()];\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tfinal char value = gson.fromJson(array.get(i), char.class);\n\t\t\t\tretArray[i] = value;\n\t\t\t}\n\t\t\treturn (T)retArray;\n\t\t} else if (clazz != null && clazz.isArray()) {\n\t\t\tfinal Class componentCls = clazz.getComponentType();\n\t\t\tfinal Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size());\n\t\t\tfor (int i = 0; i < array.size(); i++) {\n\t\t\t\tclsArray[i] = gson.fromJson(array.get(i), componentCls);\n\t\t\t}\n\t\t\t// noinspection unchecked\n\t\t\treturn (T)clsArray;\n\t\t}\n\t\treturn null;\n\t}\n\n\t/**\n\t * Return a reasonable class for a {@link JsonPrimitive}. Possible return\n\t * types are\n\t * \n\t * - boolean
\n\t * - double
\n\t * - String
\n\t * - Object
\n\t *
\n\t *\n\t * @param jsonPrimitive\n\t * the json primitive\n\t * @return the class\n\t */\n\tstatic Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) {\n\n\t\tif (jsonPrimitive.isBoolean())\n\t\t\treturn boolean.class;\n\t\telse if (jsonPrimitive.isNumber()) {\n\t\t\tfinal Number number = jsonPrimitive.getAsNumber();\n\t\t\tif (number.longValue() == number.doubleValue())\n\t\t\t\treturn long.class;\n\t\t\telse\n\t\t\t\treturn double.class;\n\t\t} else if (jsonPrimitive.isString())\n\t\t\treturn String.class;\n\t\telse\n\t\t\treturn Object.class;\n\t}\n\n\t/**\n\t * If there is an attribute in {@code root} such that it can be parsed and\n\t * deserialized as {@code T},\n\t * then remove it from {@code root}, write {@code root} to the\n\t * {@code writer}, and return the removed attribute.\n\t * \n\t * If there is an attribute at the location specified by\n\t * {@code normalizedAttributePath} but it cannot be deserialized to\n\t * {@code T}, then it is not removed.\n\t *
\n\t * If nothing is removed, then {@code root} is not written to the\n\t * {@code writer}.\n\t * to write the modified {@code root} to after removal of the attribute to\n\t * remove the attribute from\n\t *\n\t *\n\t * @param the type of the attribute to be removed\n\t * @param writer to write the modified JsonElement, which no longer contains the removed attribute\n\t * @param root element to remove the attribute from\n\t * @param normalizedAttributePath\n\t * to the attribute location of the attribute to remove to\n\t * deserialize the attribute with of the removed attribute\n\t * @param cls of the type of the attribute to be removed\n\t * @param gson used to deserialize the JsonElement as {@code T}\n\t * @return the removed attribute, or null if nothing removed\n\t * @throws IOException\n\t * if unable to write to the {@code writer}\n\t */\n\tstatic T removeAttribute(\n\t\t\tfinal Writer writer,\n\t\t\tfinal JsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal Class cls,\n\t\t\tfinal Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException {\n\n\t\tfinal T removed = removeAttribute(root, normalizedAttributePath, cls, gson);\n\t\tif (removed != null) {\n\t\t\twriteAttributes(writer, root, gson);\n\t\t}\n\t\treturn removed;\n\t}\n\n\t/**\n\t * If there is an attribute in {@code root} at location\n\t * {@code normalizedAttributePath} then remove it from {@code root}..\n\t * to write the modified {@code root} to after removal of the attribute to\n\t * remove the attribute from\n\t *\n\t * @param writer to write the modified JsonElement, which no longer contains the removed attribute\n\t * @param root to remove the attribute from\n\t * @param normalizedAttributePath\n\t * to the attribute location to deserialize the attribute with\n\t * @param gson to write the attribute to the {@code writer}\n\t * @return if the attribute was removed or not\n\t * @throws IOException if an error occurs while writing to the {@code writer}\n\t */\n\tstatic boolean removeAttribute(\n\t\t\tfinal Writer writer,\n\t\t\tfinal JsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException {\n\n\t\tfinal JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson);\n\t\tif (removed != null) {\n\t\t\twriteAttributes(writer, root, gson);\n\t\t\treturn true;\n\t\t}\n\t\treturn false;\n\t}\n\n\t/**\n\t * If there is an attribute in {@code root} such that it can be parsed and\n\t * desrialized as {@code T},\n\t * then remove it from {@code root} and return the removed attribute.\n\t * \n\t * If there is an attribute at the location specified by\n\t * {@code normalizedAttributePath} but it cannot be deserialized to\n\t * {@code T}, then it is not removed.\n\t * to remove the attribute from\n\t *\n\t * @param the type of the attribute to be removed\n\t * @param root element to remove the attribute from\n\t * @param normalizedAttributePath\n\t * to the attribute location of the attribute to remove to\n\t * deserialize the attribute with of the removed attribute\n\t * @param cls of the type of the attribute to be removed\n\t * @param gson used to deserialize the JsonElement as {@code T}\n\t * @return the removed attribute, or null if nothing removed\n\t * @throws JsonSyntaxException if the attribute is not valid json\n\t * @throws NumberFormatException if {@code T} is a {@link Number} but the attribute cannot be parsed as {@code T}\n\t * @throws ClassCastException if an attribute exists at this path, but is not of type {@code T}\n\t */\n\tstatic T removeAttribute(\n\t\t\tfinal JsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal Class cls,\n\t\t\tfinal Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {\n\n\t\tfinal T attribute = GsonUtils.readAttribute(root, normalizedAttributePath, cls, gson);\n\t\tif (attribute != null) {\n\t\t\tremoveAttribute(root, normalizedAttributePath);\n\t\t}\n\t\treturn attribute;\n\t}\n\n\t/**\n\t * Remove and return the attribute at {@code normalizedAttributePath} as a\n\t * {@link JsonElement}. Does not attempt to parse the attribute. to search\n\t * for the {@link JsonElement} at location {@code normalizedAttributePath}\n\t *\n\t * @param root\n\t * the root JsonElement\n\t * @param normalizedAttributePath\n\t * to the attribute\n\t * @return the attribute as a {@link JsonElement}.\n\t */\n\tstatic JsonElement removeAttribute(JsonElement root, final String normalizedAttributePath) {\n\n\t\tfinal String[] pathParts = normalizedAttributePath.split(\"(?= jsonArray.size()) {\n\t\t\t\t\t\treturn null;\n\t\t\t\t\t}\n\t\t\t\t\troot = jsonArray.get(index);\n\t\t\t\t\tif (i == pathParts.length - 1) {\n\t\t\t\t\t\tjsonArray.remove(index);\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\treturn null;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn root;\n\t}\n\n\t/**\n\t * Inserts {@code attribute} into {@code root} at location\n\t * {@code normalizedAttributePath} and write the resulting {@code root}.\n\t * \n\t * If {@code root} is not a {@link JsonObject}, then it is overwritten with\n\t * an object containing {@code \"normalizedAttributePath\": attribute }\n\t *\n\t * @param writer\n\t * the writer\n\t * @param root\n\t * the root json element\n\t * @param normalizedAttributePath\n\t * the attribute path\n\t * @param attribute\n\t * the attribute\n\t * @param gson\n\t * the gson\n\t * @param \n\t * the attribute type\n\t * @throws IOException\n\t * the exception\n\t */\n\tstatic void writeAttribute(\n\t\t\tfinal Writer writer,\n\t\t\tJsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal T attribute,\n\t\t\tfinal Gson gson) throws IOException {\n\n\t\troot = insertAttribute(root, normalizedAttributePath, attribute, gson);\n\t\twriteAttributes(writer, root, gson);\n\t}\n\n\t/**\n\t * Writes the attributes JsonElement to a given {@link Writer}.\n\t * This will overwrite any existing attributes.\n\t *\n\t * @param writer\n\t * the writer\n\t * @param root\n\t * the root json element\n\t * @param gson\n\t * the gson\n\t * @param \n\t * the attribute type\n\t * @throws IOException\n\t * the exception\n\t */\n\tstatic void writeAttributes(\n\t\t\tfinal Writer writer,\n\t\t\tfinal JsonElement root,\n\t\t\tfinal Gson gson) throws IOException {\n\n\t\tgson.toJson(root, writer);\n\t\twriter.flush();\n\t}\n\n\tstatic JsonElement insertAttributes(JsonElement root, final Map attributes, final Gson gson) {\n\n\t\tfor (final Map.Entry attribute : attributes.entrySet()) {\n\t\t\troot = insertAttribute(root, N5URI.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson);\n\t\t}\n\t\treturn root;\n\t}\n\n\tstatic JsonElement insertAttribute(\n\t\t\tJsonElement root,\n\t\t\tfinal String normalizedAttributePath,\n\t\t\tfinal T attribute,\n\t\t\tfinal Gson gson) {\n\n\t\tLinkedAttributePathToken pathToken = N5URI.getAttributePathTokens(normalizedAttributePath);\n\n\t\t/* No path to traverse or build; just return the value */\n\t\tif (pathToken == null)\n\t\t\treturn gson.toJsonTree(attribute);\n\n\t\tJsonElement json = root;\n\t\twhile (pathToken != null) {\n\n\t\t\tfinal JsonElement parent = pathToken.setAndCreateParentElement(json);\n\n\t\t\t/*\n\t\t\t * We may need to create or override the existing root if it is\n\t\t\t * non-existent or incompatible.\n\t\t\t */\n\t\t\tfinal boolean rootOverriden = json == root && parent != json;\n\t\t\tif (root == null || rootOverriden) {\n\t\t\t\troot = parent;\n\t\t\t}\n\n\t\t\tjson = pathToken.writeChild(gson, attribute);\n\n\t\t\tpathToken = pathToken.next();\n\t\t}\n\t\treturn root;\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.zip.Deflater;\nimport java.util.zip.DeflaterOutputStream;\nimport java.util.zip.InflaterInputStream;\nimport org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;\nimport org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;\nimport org.apache.commons.compress.compressors.gzip.GzipParameters;\nimport org.janelia.saalfeldlab.n5.Compression.CompressionType;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.serialization.NameConfig;\n\n@CompressionType(\"gzip\")\n@NameConfig.Name(\"gzip\")\npublic class GzipCompression implements Compression {\n\n\tprivate static final long serialVersionUID = 8630847239813334263L;\n\n\t/**\n\t * Explicit equivalent of {@link java.util.zip.Deflater#DEFAULT_COMPRESSION}: zlib defines\n\t * level 6 as \"a default compromise between speed and compression.\" An explicit value is used\n\t * instead of {@code DEFAULT_COMPRESSION} (-1) because -1 is not a valid level for Zarr codecs.\n\t *\n\t * @see zlib Manual\n\t */\n\tprivate static final int N5_DEFAULT_GZIP_LEVEL = 6;\n\n\t@CompressionParameter\n\t@NameConfig.Parameter\n\tprivate final int level;\n\n\t/**\n\t * This is not a NameConfig.Parameter because this parameter must not be\n\t * serialized for zarr\n\t */\n\t@CompressionParameter\n\tprivate final boolean useZlib;\n\n\tprivate final transient GzipParameters parameters = new GzipParameters();\n\n\tpublic GzipCompression() {\n\n\t\tthis(N5_DEFAULT_GZIP_LEVEL);\n\t}\n\n\tpublic GzipCompression(final int level) {\n\n\t\tthis(level, false);\n\t}\n\n\tpublic GzipCompression(final int level, final boolean useZlib) {\n\n\t\tthis.level = level;\n\t\tthis.useZlib = useZlib;\n\t}\n\n\t@Override\n\tpublic boolean equals(final Object other) {\n\n\t\tif (other == null || other.getClass() != GzipCompression.class)\n\t\t\treturn false;\n\t\telse {\n\t\t\tfinal GzipCompression gz = ((GzipCompression)other);\n\t\t\treturn useZlib == gz.useZlib && level == gz.level;\n\t\t}\n\t}\n\n\tprivate InputStream decode(final InputStream in) throws IOException {\n\t\tif (useZlib) {\n\t\t\treturn new InflaterInputStream(in);\n\t\t} else {\n\t\t\treturn GzipCompressorInputStream.builder()\n\t\t\t\t\t.setInputStream(in)\n\t\t\t\t\t.setDecompressConcatenated(true)\n\t\t\t\t\t.get();\n\t\t}\n\t}\n\n\t@Override\n\tpublic ReadData decode(final ReadData readData) throws N5IOException {\n\n\t\ttry {\n\t\t\treturn ReadData.from(decode(readData.inputStream()));\n\t\t} catch (IOException e) {\n\t\t\tthrow new N5IOException(e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic ReadData encode(final ReadData readData) {\n\t\tif (useZlib) {\n\t\t\treturn readData.encode(out -> new DeflaterOutputStream(out, new Deflater(level)));\n\t\t} else {\n\t\t\treturn readData.encode(out -> {\n\t\t\t\tparameters.setCompressionLevel(level);\n\t\t\t\treturn new GzipCompressorOutputStream(out, parameters);\n\t\t\t});\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.apache.commons.io.IOUtils;\n\nimport org.apache.commons.lang3.function.TriFunction;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.http.ListResponseParser;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\n\nimport java.io.Closeable;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.io.Reader;\nimport java.io.Writer;\nimport java.net.HttpURLConnection;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.URL;\nimport java.nio.channels.NonWritableChannelException;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport org.janelia.saalfeldlab.n5.readdata.LazyRead;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\n\n/**\n * A read-only {@link KeyValueAccess} implementation using HTTP. As a result, calling lockForWriting, createDirectories, or delete will throw an {@link N5Exception}.\n * \n * The behavior of list, listDirectories, and isDirectory will depend on the server configuration. See the documentation of those methods for details.\n *
\n * Methods that take a \"normalPath\" as an argument expect absolute URIs.\n */\npublic class HttpKeyValueAccess implements KeyValueAccess {\n\n\tpublic static final String HEAD = \"HEAD\";\n\tpublic static final String GET = \"GET\";\n\n\tpublic static final String RANGE = \"Range\";\n\tpublic static final String ACCEPT_RANGE = \"Accept-Range\";\n\tpublic static final String BYTES = \"bytes\";\n\n\tprivate int readTimeoutMilliseconds;\n\tprivate int connectionTimeoutMilliseconds;\n\n\tprivate ListResponseParser listResponseParser = ListResponseParser.defaultListParser();\n\tprivate ListResponseParser listDirectoryResponseParser = ListResponseParser.defaultDirectoryListParser();\n\n\t/**\n\t * Opens an {@link HttpKeyValueAccess}\n\t *\n\t * @throws N5IOException if the access could not be created\n\t */\n\tpublic HttpKeyValueAccess() {\n\n\t\treadTimeoutMilliseconds = 5000;\n\t\tconnectionTimeoutMilliseconds = 5000;\n\t}\n\n\tpublic void setReadTimeout(int readTimeoutMilliseconds) {\n\n\t\tthis.readTimeoutMilliseconds = readTimeoutMilliseconds;\n\t}\n\n\tpublic void setConnectionTimeout(int connectionTimeoutMilliseconds) {\n\n\t\tthis.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;\n\t}\n\n\tpublic void setListParser(final ListResponseParser parser) {\n\n\t\tlistResponseParser = parser;\n\t}\n\n\tpublic void setListDirectoryParser(final ListResponseParser parser) {\n\n\t\tlistDirectoryResponseParser = parser;\n\t}\n\n\t@Override\n\tpublic String normalize(final String path) {\n\n\t\treturn N5URI.normalizeGroupPath(path);\n\t}\n\n\t@Override\n\tpublic URI uri(final String normalPath) throws URISyntaxException {\n\n\t\treturn new URI(normalPath);\n\t}\n\n\t/**\n\t * Test whether the {@code normalPath} exists.\n\t *
\n\t * Removes leading slash from {@code normalPath}, and then checks whether\n\t * either {@code path} or {@code path + \"/\"} is a key.\n\t *\n\t * @param normalPath is expected to be in normalized form, no further efforts are\n\t * made to normalize it.\n\t * @return {@code true} if {@code path} exists, {@code false} otherwise\n\t */\n\t@Override\n\tpublic boolean exists(final String normalPath) {\n\n\t\ttry {\n\t\t\trequireValidHttpResponse(normalPath, \"HEAD\", \"Error checking existence: \" + normalPath, true);\n\t\t\treturn true;\n\t\t} catch (N5Exception.N5NoSuchKeyException e) {\n\t\t\treturn false;\n\t\t}\n\t}\n\n\t@Override public long size(String normalPath) {\n\n\t\tfinal HttpURLConnection head = requireValidHttpResponse(normalPath, \"HEAD\", \"Error checking existence: \" + normalPath, true);\n\t\treturn head.getContentLengthLong();\n\t}\n\n\t/**\n\t * Test whether the path is a directory.\n\t *
\n\t * Appends trailing \"/\" to {@code normalPath} if there is none, removes\n\t * leading \"/\", and then checks whether resulting {@code path} is a key.\n\t *\n\t * @param normalPath is expected to be in normalized form, no further efforts are\n\t * made to normalize it.\n\t * @return {@code true} if {@code path} (with trailing \"/\") exists as a key,\n\t * {@code false} otherwise\n\t */\n\t@Override\n\tpublic boolean isDirectory(final String normalPath) {\n\n\t\ttry {\n\t\t\trequireValidHttpResponse(getDirectoryPath(normalPath), HEAD, false, (code, msg,http) -> {\n\t\t\t\tfinal N5Exception cause = validExistsResponse(code, \"Error checking directory: \" + normalPath, msg, true);\n\t\t\t\tif (code >= 300 && code < 400) {\n\t\t\t\t\tfinal String redirectLocation = http.getHeaderField(\"Location\");\n\t\t\t\t\tif (!(redirectLocation.endsWith(\"/\") || redirectLocation.endsWith(\"index.html\")))\n\t\t\t\t\t\treturn new N5Exception.N5NoSuchKeyException(\"Found File at \" + normalPath + \" but was not directory\");\n\t\t\t\t\treturn null;\n\t\t\t\t}\n\t\t\t\treturn cause;\n\t\t\t});\n\t\t\treturn true;\n\t\t} catch (N5Exception e) {\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate static String getDirectoryPath(String normalPath) {\n\n\t\tfinal String directoryNormalPath;\n\t\tif (normalPath.endsWith(\"/\"))\n\t\t\tdirectoryNormalPath = normalPath;\n\t\telse\n\t\t\tdirectoryNormalPath = normalPath + \"/\";\n\t\treturn directoryNormalPath;\n\t}\n\n\t/**\n\t * Test whether the path is a file.\n\t *
\n\t * Checks whether {@code normalPath} has no trailing \"/\", then removes\n\t * leading \"/\" and checks whether the resulting {@code path} is a key.\n\t *\n\t * @param normalPath is expected to be in normalized form, no further efforts are\n\t * made to normalize it.\n\t * @return {@code true} if {@code path} exists as a key and has no trailing\n\t * slash, {@code false} otherwise\n\t */\n\t@Override\n\tpublic boolean isFile(final String normalPath) {\n\n\t\t/* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */\n\t\ttry {\n\t\t\trequireValidHttpResponse(getFilePath(normalPath), HEAD, false, (code, msg, http) -> {\n\t\t\t\tfinal N5Exception cause = validExistsResponse(code, \"Error accessing file: \" + normalPath, msg, true);\n\t\t\t\tif (code >= 300 && code < 400) {\n\t\t\t\t\tfinal String redirectLocation = http.getHeaderField(\"Location\");\n\t\t\t\t\tif (redirectLocation.endsWith(\"/\") || redirectLocation.endsWith(\"index.html\"))\n\t\t\t\t\t\treturn new N5Exception.N5NoSuchKeyException(\"Found key at \" + normalPath + \" but was directory\");\n\t\t\t\t}\n\t\t\t\treturn cause;\n\t\t\t});\n\t\t\treturn true;\n\t\t} catch (N5Exception e) {\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate static String getFilePath(String normalPath) {\n\n\t\tfinal String fileNormalPath = normalPath.replaceAll(\"/+$\", \"\");\n\t\treturn fileNormalPath;\n\t}\n\n\tprivate HttpURLConnection httpRequest(String normalPath, String method) throws IOException {\n\n\t\tfinal URL url = URI.create(normalPath).toURL();\n\t\tfinal HttpURLConnection connection = (HttpURLConnection)url.openConnection();\n\t\tconnection.setReadTimeout(readTimeoutMilliseconds);\n\t\tconnection.setConnectTimeout(connectionTimeoutMilliseconds);\n\t\tconnection.setRequestMethod(method);\n\t\treturn connection;\n\t}\n\n\t@Override\n\tpublic VolatileReadData createReadData(final String normalPath) {\n\t\treturn VolatileReadData.from(new HttpLazyRead(normalPath));\n\t}\n\n\tpublic LockedChannel lockForReading(final String normalPath) throws N5IOException {\n\t\t//TODO Caleb: Maybe check exists lazily when attempting to read\n\t\ttry {\n\t\t\tif (!exists(normalPath))\n\t\t\t\tthrow new N5Exception.N5NoSuchKeyException(\"Key does not exist: \" + normalPath);\n\t\t\treturn new HttpObjectChannel(uri(normalPath), 0, -1);\n\t\t} catch (URISyntaxException e) {\n\t\t\tthrow new N5Exception(\"Invalid URI Syntax\", e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic LockedChannel lockForWriting(final String normalPath) throws N5IOException {\n\n\t\tthrow new N5Exception(\"HttpKeyValueAccess is read-only\");\n\t}\n\n\t@Override\n\tpublic void write(final String normalPath, final ReadData data) throws N5IOException {\n\n\t\tthrow new N5Exception(\"HttpKeyValueAccess is read-only\");\n\t}\n\n\t/**\n\t * List all 'directory'-like children of a path.\n\t *
\n\t * Will throw an N5IOException both if a connection to the server can not be established, or the server does not allow listing.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the directories\n\t * @throws N5IOException\n\t * if an error occurs during listing\n\t */\n\t@Override\n\tpublic String[] listDirectories(final String normalPath) throws N5IOException {\n\n\t\treturn queryListEntries(normalPath, listDirectoryResponseParser, true);\n\t}\n\n\t/**\n\t * List all children of a path.\n\t *
\n\t * Will throw an N5IOException both if a connection to the server can not be\n\t * established, or the server does not allow listing.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further efforts are\n\t * made to normalize it.\n\t * @return the the child paths\n\t * @throws N5IOException\n\t * if an error occurs during listing\n\t */\n\t@Override\n\tpublic String[] list(final String normalPath) throws N5IOException {\n\n\t\treturn queryListEntries(normalPath, listResponseParser, true);\n\t}\n\n\tprivate String[] queryListEntries(String normalPath, ListResponseParser parser, boolean allowRedirect) throws N5IOException{\n\n\t\tfinal HttpURLConnection http = requireValidHttpResponse(normalPath, GET, \"Error listing directory at \" + normalPath, allowRedirect);\n\t\ttry {\n\t\t\tfinal String listResponse = responseToString(http.getInputStream());\n\t\t\treturn parser.parseListResponse(listResponse);\n\t\t} catch (IOException e) {\n\t\t\tthrow new N5IOException(\"Error listing directory at \" + normalPath, e);\n\t\t}\n\t}\n\n\tprivate static N5Exception validExistsResponse(int code, String responseMsg, String message, boolean allowRedirect) {\n\t\tif (code >= 200 && code < (allowRedirect ? 400 : 300)) return null;\n\n\t\tfinal RuntimeException cause = new RuntimeException(message + \"( \"+ responseMsg + \")(\" + code + \")\");\n\t\tif (code == 404 | code == 410)\n\t\t\treturn new N5Exception.N5NoSuchKeyException(message, cause);\n\n\t\treturn new N5Exception(message, cause);\n\t}\n\n\tprivate HttpURLConnection requireValidHttpResponse(String uri, String method, String message, boolean allowRedirect) throws N5Exception {\n\t\treturn requireValidHttpResponse(uri, method, (code, msg, http) -> validExistsResponse(code, msg, message, allowRedirect));\n\t}\n\n\tprivate HttpURLConnection requireValidHttpResponse(String uri, String method, TriFunction filterCode) throws N5Exception {\n\t\treturn requireValidHttpResponse(uri, method, true, filterCode);\n\t}\n\n\tprivate HttpURLConnection requireValidHttpResponse(String uri, String method, boolean followRedirects, TriFunction filterCode) throws N5Exception {\n\n\t\tfinal int code;\n\t\tfinal HttpURLConnection http;\n\t\tfinal String responseMsg;\n\t\ttry {\n\t\t\thttp = httpRequest(uri, method);\n\t\t\thttp.setInstanceFollowRedirects(followRedirects);\n\t\t\tcode = http.getResponseCode();\n\t\t\tresponseMsg = http.getResponseMessage();\n\t\t} catch (IOException e) {\n\t\t\tthrow new N5IOException(\"Could not validate HTTP Response\", e);\n\t\t}\n\n\t\tfinal N5Exception cause = filterCode.apply(code, responseMsg, http);\n\t\tif (cause != null) throw cause;\n\t\treturn http;\n\t}\n\n\tprivate String responseToString(InputStream inputStream) throws IOException {\n\n\t\treturn IOUtils.toString(inputStream, StandardCharsets.UTF_8.name());\n\t}\n\n\t@Override\n\tpublic void createDirectories(final String normalPath) {\n\n\t\tthrow new N5Exception(\"HttpKeyValueAccess is read-only\");\n\t}\n\n\t@Override\n\tpublic void delete(final String normalPath) {\n\n\t\tthrow new N5Exception(\"HttpKeyValueAccess is read-only\");\n\t}\n\n\tprivate class HttpObjectChannel implements LockedChannel {\n\n\t\tprotected final URI uri;\n\t\tprivate final long startByte;\n\t\tprivate final long size;\n\t\tprivate final ArrayList resources = new ArrayList<>();\n\n\t\tprotected HttpObjectChannel(final URI uri, long startByte, long size) {\n\n\t\t\tthis.uri = uri;\n\t\t\tthis.startByte = startByte;\n\t\t\tthis.size = size;\n\t\t}\n\n\t\tprivate boolean isPartialRead() {\n\t\t\treturn startByte > 0 || (size >= 0 && size != Long.MAX_VALUE);\n\t\t}\n\n\t\t@Override\n\t\tpublic InputStream newInputStream() throws N5IOException {\n\n\t\t\ttry {\n\t\t\t\tHttpURLConnection conn = (HttpURLConnection)uri.toURL().openConnection();\n\t\t\t\tif (isPartialRead()) {\n\t\t\t\t\tconn.setRequestProperty(RANGE, rangeString());\n\t\t\t\t\tfinal String acceptRanges = conn.getHeaderField(ACCEPT_RANGE);\n\t\t\t\t\tif (acceptRanges == null || !acceptRanges.equals(BYTES)) {\n\t\t\t\t\t\tconn.disconnect();\n\t\t\t\t\t\tconn = (HttpURLConnection)uri.toURL().openConnection();\n\t\t\t\t\t\treturn ReadData.from(conn.getInputStream()).materialize().slice(startByte, size).inputStream();\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn conn.getInputStream();\n\t\t\t} catch (FileNotFoundException e) {\n\t\t\t\t/*default HttpURLConnection throws FileNotFoundException on 404 or 410 */\n\t\t\t\tthrow new N5Exception.N5NoSuchKeyException(\"Could not open stream for \" + uri, e);\n\t\t\t} catch (IOException e) {\n\t\t\t\tthrow new N5IOException(\"Could not open stream for \" + uri, e);\n\t\t\t}\n\t\t}\n\n\t\tprivate String rangeString() {\n\n\t\t\tfinal String lastByte = (size > 0) ? Long.toString(startByte + size - 1) : \"\";\n\t\t\treturn String.format(\"%s=%d-%s\", BYTES, startByte, lastByte);\n\t\t}\n\n\t\t@Override\n\t\tpublic Reader newReader() {\n\n\t\t\tfinal InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8);\n\t\t\tsynchronized (resources) {\n\t\t\t\tresources.add(reader);\n\t\t\t}\n\t\t\treturn reader;\n\t\t}\n\n\t\t@Override\n\t\tpublic OutputStream newOutputStream() {\n\n\t\t\tthrow new NonWritableChannelException();\n\t\t}\n\n\t\t@Override\n\t\tpublic Writer newWriter() {\n\n\t\t\tthrow new NonWritableChannelException();\n\t\t}\n\n\t\t@Override\n\t\tpublic void close() throws IOException {\n\n\t\t\tsynchronized (resources) {\n\t\t\t\tfor (final Closeable resource : resources) {\n\t\t\t\t\tresource.close();\n\t\t\t\t}\n\t\t\t\tresources.clear();\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate class HttpLazyRead implements LazyRead {\n\n\t\tprivate final String normalKey;\n\n\t\tHttpLazyRead(String normalKey) {\n\t\t\tthis.normalKey = normalKey;\n\t\t}\n\n\t\t@Override\n\t\tpublic long size() {\n\t\t\treturn HttpKeyValueAccess.this.size(normalKey);\n\t\t}\n\n\t\t@Override\n\t\tpublic ReadData materialize(long offset, long length) {\n\t\t\ttry (final HttpObjectChannel ch = new HttpObjectChannel(uri(normalKey), offset, length)) {\n\t\t\t\treturn ReadData.from(ch.newInputStream()).materialize();\n\t\t\t} catch (IOException e) {\n\t\t\t\tthrow new N5IOException(e);\n\t\t\t} catch (URISyntaxException e) {\n\t\t\t\tthrow new N5Exception(e);\n\t\t\t}\n\t\t}\n\n\t\t@Override\n\t\tpublic void close() {\n\t\t}\n\t}\n\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class IntArrayDataBlock extends AbstractDataBlock {\n\n\tpublic IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) {\n\n\t\tsuper(size, gridPosition, data, a -> a.length);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\n\nimport java.io.IOException;\n\npublic interface IoPolicy {\n\n void write(String key, ReadData readData) throws IOException;\n\n VolatileReadData read(String key) throws IOException;\n\n void delete(String key) throws IOException;\n\n static IoPolicy withFallback(IoPolicy primary, IoPolicy fallback) {\n return new IoPolicy() {\n\n @Override\n public void write(String key, ReadData readData) throws IOException {\n try {\n primary.write(key, readData);\n } catch (IOException e) {\n fallback.write(key, readData);\n }\n }\n\n @Override\n public VolatileReadData read(String key) throws IOException {\n try {\n return primary.read(key);\n } catch (IOException e) {\n return fallback.read(key);\n }\n }\n\n @Override\n public void delete(String key) throws IOException {\n try {\n primary.delete(key);\n } catch (IOException e) {\n fallback.delete(key);\n }\n }\n };\n }\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/KeyLockState.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.IOException;\nimport java.nio.channels.FileChannel;\nimport java.nio.file.Path;\nimport java.nio.file.StandardOpenOption;\nimport java.util.concurrent.Semaphore;\n\n/**\n * Per-key state that tracks both thread locks and file locks.\n */\nclass KeyLockState {\n\n\tprivate final Path path;\n\n\tprivate final LockingPolicy policy;\n\n\tpublic KeyLockState(final Path path, LockingPolicy policy) {\n\t\tthis.path = path;\n\t\tthis.policy = policy;\n\t}\n\n\t/**\n\t * The current system-level file lock (shared for reading, exclusive for\n\t * writing). Is {@link ChannelLock#close closed} when the last {@link\n\t * #releaseRead() Reader is released}, or the (one and only) {@link\n\t * #releaseWrite() Writer is released}.\n\t */\n\tprivate volatile ChannelLock channelLock;\n\n\t/**\n\t * Multiple Readers coordinate via this mutex. {@code numReaders} may only\n\t * be modified when {@code readerMutex} is held.\n\t */\n\tprivate final Semaphore readerMutex = new Semaphore(1);\n\tprivate int numReaders = 0;\n\n\t/**\n\t * This coordinates mutual exclusion between one writer and (the first of)\n\t * multiple readers. {@code channelLock} may only be created or closed when\n\t * {@code channelLockMutex} is held.\n\t */\n\tprivate final Semaphore channelLockMutex = new Semaphore(1);\n\n\tLockedFileChannel acquireRead() throws IOException {\n\n\t\ttry {\n\t\t\treaderMutex.acquire();\n\t\t\ttry {\n\t\t\t\tif (numReaders == 0) {\n\t\t\t\t\t// We are the first Reader, and are responsible for creating the channelLock\n\t\t\t\t\t// (Other concurrent Readers will still be blocked in readerMutex.)\n\n\t\t\t\t\t// If a Writer is still open, this will block us until the Writer is closed.\n\t\t\t\t\tchannelLockMutex.acquire();\n\n\t\t\t\t\ttry {\n\t\t\t\t\t\tchannelLock = ChannelLock.lock(path, false, policy);\n\t\t\t\t\t} catch (IOException e) {\n\t\t\t\t\t\t// Something went wrong. Back off.\n\t\t\t\t\t\tchannelLockMutex.release();\n\t\t\t\t\t\tthrow e;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// We have a FileChannel.\n\t\t\t\t// Create a LockedFileChannel that will releaseRead() when it is closed.\n\t\t\t\t++numReaders;\n\t\t\t\treturn new LockedFileChannel(channelLock.getChannel(), this::releaseRead);\n\t\t\t} finally {\n\t\t\t\treaderMutex.release();\n\t\t\t}\n\t\t} catch (InterruptedException e) {\n\t\t\tthrow new IOException(e);\n\t\t}\n\t}\n\n\tvoid releaseRead() throws IOException {\n\n\t\ttry {\n\t\t\treaderMutex.acquire();\n\t\t\ttry {\n\t\t\t\t--numReaders;\n\t\t\t\tif (numReaders == 0) {\n\t\t\t\t\t// We were the last Reader, and are responsible for releasing the channelLock\n\t\t\t\t\treleaseChannelLock();\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\treaderMutex.release();\n\t\t\t}\n\t\t} catch (InterruptedException e) {\n\t\t\tthrow new IOException(e);\n\t\t}\n\t}\n\n\tprivate void releaseChannelLock() throws IOException {\n\n\t\ttry {\n\t\t\tchannelLock.close();\n\t\t} finally {\n\t\t\tchannelLockMutex.release();\n\t\t}\n\t}\n\n\tLockedFileChannel acquireWrite() throws IOException {\n\n\t\ttry {\n\t\t\t// If another Writer or Reader is still open, this will block until it is closed.\n\t\t\tchannelLockMutex.acquire();\n\n\t\t\ttry {\n\t\t\t\tchannelLock = ChannelLock.lock(path, true, policy);\n\t\t\t} catch (IOException e) {\n\t\t\t\t// Something went wrong. Back off.\n\t\t\t\tchannelLockMutex.release();\n\t\t\t\tthrow e;\n\t\t\t}\n\n\t\t\t// We have a WRITE ChannelLock.\n\t\t\t// Create a LockedFileChannel that will releaseWrite() when it is closed.\n\t\t\treturn new LockedFileChannel(channelLock.getChannel(), this::releaseWrite);\n\t\t} catch (InterruptedException e) {\n\t\t\tthrow new IOException(e);\n\t\t}\n\t}\n\n\tvoid releaseWrite() throws IOException {\n\t\treleaseChannelLock();\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\n\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.nio.file.FileSystem;\nimport java.util.Arrays;\nimport java.util.stream.Collectors;\n\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.readdata.VolatileReadData;\n\n/**\n * Key value read primitives used by {@link N5KeyValueReader}\n * implementations. This interface implements a subset of access primitives\n * provided by {@link FileSystem} to reduce the implementation burden for\n * backends lacking a {@link FileSystem} implementation (such as AWS-S3).\n *\n * @author Stephan Saalfeld\n */\npublic interface KeyValueAccess {\n\n\t/**\n\t * Split a path string into its components.\n\t *\n\t * @param path\n\t * the path\n\t * @return the path components\n\t */\n\tdefault String[] components( final String path ) {\n\n\t\tString[] components = Arrays.stream(path.split(\"/\"))\n\t\t\t\t.filter(x -> !x.isEmpty())\n\t\t\t\t.toArray(String[]::new);\n\t\tif (components.length == 0)\n\t\t\treturn path.startsWith(\"/\") ? new String[]{\"/\"} : new String[]{\"\"};\n\n\t\tif (path.startsWith(\"/\") && !components[0].equals(\"/\")) {\n\t\t\tfinal String[] prependRoot = new String[components.length + 1];\n\t\t\tprependRoot[0] = \"/\";\n\t\t\tSystem.arraycopy(components, 0, prependRoot, 1, components.length);\n\t\t\tcomponents = prependRoot;\n\t\t}\n\n\t\tif (path.endsWith(\"/\") && !components[components.length - 1].endsWith(\"/\")) {\n\t\t\tcomponents[components.length - 1] = components[components.length - 1] + \"/\";\n\t\t}\n\t\treturn components;\n\t}\n\n\t/**\n\t * Compose a path from a base uri and subsequent components.\n\t *\n\t * @param uri the base path uri\n\t * @param components the path components\n\t * @return the path\n\t */\n\tdefault String compose( final URI uri, final String... components ) {\n\n\t\tint firstNonEmptyIdx = 0;\n\t\twhile (firstNonEmptyIdx < components.length && (components[firstNonEmptyIdx] == null || components[firstNonEmptyIdx].isEmpty())) {\n\t\t\tfirstNonEmptyIdx++;\n\t\t}\n\n\t\t/*If there are no non-empty components, there is nothing to compose against; return the uri. */\n\t\tif (components.length == firstNonEmptyIdx)\n\t\t\treturn uri.toString();\n\n\t\t/* allocate space for the initial path and the new components, skipping empty strings */\n\t\tfinal int nonEmptysize = components.length - firstNonEmptyIdx;\n\t\tfinal String[] allComponents = new String[1 + nonEmptysize];\n\t\tif (uri.getPath().isEmpty())\n\t\t\t//TODO Caleb: This `isEmpty()` check is only necessary for Java 8. In newer versions\n\t\t\t//\tURI resolution is updated so that resolving and empty path with a new path adds\n\t\t\t//\ta leading `/` between the rest of the URI and the path part. In Java 8 it doesn't\n\t\t\t// add the `/` so it ends up directly concatenating the path part with URI\n\t\t\tallComponents[0] = \"/\";\n\t\telse\n\t\t\tallComponents[0] = uri.getPath();\n\n\t\tSystem.arraycopy(components, firstNonEmptyIdx, allComponents, 1, nonEmptysize);\n\n\t\tURI composedUri = uri;\n\t\tfor (int i = 0; i < allComponents.length; i++) {\n\t\t\tfinal String component = allComponents[i];\n\t\t\tif (component == null || component.isEmpty())\n\t\t\t\tcontinue;\n\t\t\telse if (component.endsWith(\"/\") || i == allComponents.length - 1)\n\t\t\t\tcomposedUri = composedUri.resolve(N5URI.encodeAsUriPath(component));\n\t\t\telse\n\t\t\t\tcomposedUri = composedUri.resolve(N5URI.encodeAsUriPath(component + \"/\"));\n\t\t}\n\t\treturn composedUri.toString();\n\t}\n\n\t@Deprecated\n\tdefault String compose( final String... components ) {\n\n\t\treturn normalize(\n\t\t\t\tArrays.stream(components)\n\t\t\t\t\t\t.filter(x -> !x.isEmpty())\n\t\t\t\t\t\t.collect(Collectors.joining(\"/\"))\n\t\t);\n\n\t}\n\n\t/**\n\t * Get the parent of a path string.\n\t *\n\t * @param path\n\t * the path\n\t * @return the parent path or null if the path has no parent\n\t */\n\tdefault String parent( final String path ) {\n\t\tfinal String removeTrailingSlash = path.replaceAll(\"/+$\", \"\");\n\t\treturn normalize(N5URI.getAsUri(removeTrailingSlash).resolve(\"\").toString());\n\t}\n\n\t/**\n\t * Relativize path relative to base.\n\t *\n\t * @param path\n\t * the path\n\t * @param base\n\t * the base path\n\t * @return the result or null if the path has no parent\n\t */\n\tdefault String relativize( final String path, final String base ) {\n\n\t\ttry {\n\t\t\t/*\n\t\t\t * Must pass absolute path to `uri`. if it already is, this is\n\t\t\t * redundant, and has no impact on the result. It's not true that\n\t\t\t * the inputs are always referencing absolute paths, but it doesn't\n\t\t\t * matter in this case, since we only care about the relative\n\t\t\t * portion of `path` to `base`, so the result always ignores the\n\t\t\t * absolute prefix anyway.\n\t\t\t */\n\t\t\treturn normalize(uri(\"/\" + base).relativize(uri(\"/\" + path)).toString());\n\t\t} catch (final URISyntaxException e) {\n\t\t\tthrow new N5Exception(\"Cannot relativize path (\" + path + \") with base (\" + base + \")\", e);\n\t\t}\n\t}\n\n\t/**\n\t * Normalize a path to canonical form. All paths pointing to the same\n\t * location return the same output. This is most important for cached\n\t * data pointing at the same location getting the same key.\n\t *\n\t * @param path\n\t * the path\n\t * @return the normalized path\n\t */\n\tString normalize( final String path );\n\n\t/**\n\t * Get the absolute (including scheme) {@link URI} of the given path\n\t *\n\t * @param uriString\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return absolute URI\n\t * @throws URISyntaxException if the given path is not a proper URI\n\t */\n\tdefault URI uri(final String uriString) throws URISyntaxException {\n\t\ttry {\n\t\t\treturn URI.create(uriString);\n\t\t} catch (Exception ignore) {\n\t\t\treturn N5URI.encodeAsUri(uriString);\n\t\t}\n\t}\n\t/**\n\t * Test whether the path exists.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return true if the path exists\n\t */\n\tboolean exists( final String normalPath );\n\n\t/**\n\t * Returns the size in bytes of the object at the given normalPath if it exists.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the size of the object in bytes.\n\t * @throws N5Exception.N5NoSuchKeyException if the given key does not exist\n\t */\n\tlong size( final String normalPath ) throws N5Exception.N5NoSuchKeyException;\n\n\t/**\n\t * Test whether the path is a directory.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return true if the path is a directory\n\t */\n\tboolean isDirectory( String normalPath );\n\n\t/**\n\t * Test whether the path is a file.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return true if the path is a file\n\t */\n\tboolean isFile( String normalPath ); // TODO: Looks un-used. Remove?\n\n\t/**\n\t * Create a {@link VolatileReadData} through which data at the normal key\n\t * can be read.\n\t * \n\t * Implementations should read lazily if possible. Consumers may call {@link\n\t * ReadData#materialize()} to force a read operation if needed.\n\t *
\n\t * If supported by this KeyValueAccess implementation, partial reads are\n\t * possible by {@link ReadData#slice slicing} the returned {@code ReadData}.\n\t *
\n\t * The resulting {@code VolatileReadData} is potentially lazy. If the requested\n\t * key does not exist, it will throw {@code N5NoSuchKeyException}. Whether\n\t * the exception is thrown when {@link KeyValueAccess#createReadData(String)}] is called,\n\t * or when trying to materialize the {@code VolatileReadData} is implementation dependent.\n\t *\n\t * @param normalPath\n\t * \t\tis expected to be in normalized form, no further efforts are made to normalize it\n\t *\n\t * @return a ReadData\n\t *\n\t * @throws N5IOException\n\t * \t\tif an error occurs\n\t */\n\tVolatileReadData createReadData(final String normalPath) throws N5IOException;\n\n\t/**\n\t * Write {@code data} to the given {@code normalPath}.\n\t *
\n\t * Existing data at {@code normalPath} will be overridden.\n\t *\n\t * @param normalPath\n\t * \t\tis expected to be in normalized form, no further efforts are made to normalize it\n\t * @param data\n\t * \t\tthe data to write\n\t *\n\t * @throws N5IOException\n\t * \t\tif an error occurs\n\t */\n\tvoid write(String normalPath, ReadData data) throws N5IOException;\n\n\t/**\n\t * Create a lock on a path for reading. This isn't meant to be kept\n\t * around. Create, use, [auto]close, e.g.\n\t * \n\t * try (final lock = store.lockForReading()) {\n\t * ...\n\t * }\n\t * \n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the locked channel\n\t * @throws N5IOException\n\t * if a locked channel could not be created\n\t * @deprecated migrate to {@link KeyValueAccess#createReadData(String)}\n\t */\n\t@Deprecated\n\tdefault LockedChannel lockForReading( final String normalPath ) throws N5IOException {\n\t\treturn new BufferedKvaLockedChannel(this, normalPath);\n\t}\n\n\t/**\n\t * Create an exclusive lock on a path for writing. If the file doesn't exist\n\t * yet, it will be created, including all directories leading up to it. This\n\t * lock isn't meant to be kept around. Create, use, [auto]close, e.g.\n\t * \n\t * try (final lock = store.lockForWriting()) {\n\t * ...\n\t * }\n\t * \n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the locked channel\n\t * @throws N5IOException\n\t * if a locked channel could not be created\n\t * @deprecated migrate to {@link KeyValueAccess#write(String, ReadData)}\n\t */\n\t@Deprecated\n\tdefault LockedChannel lockForWriting( final String normalPath ) throws N5IOException {\n\t\treturn new BufferedKvaLockedChannel(this, normalPath);\n\t}\n\n\t/**\n\t * List all 'directory'-like children of a path.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the directories\n\t * @throws N5IOException\n\t * if an error occurs during listing\n\t */\n\tString[] listDirectories( final String normalPath ) throws N5IOException;\n\n\t/**\n\t * List all children of a path.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @return the the child paths\n\t * @throws N5IOException if an error occurs during listing\n\t */\n\tString[] list( final String normalPath ) throws N5IOException;\n\n\t/**\n\t * Create a directory and all parent paths along the way. The directory\n\t * and parent paths are discoverable. On a filesystem, this usually means\n\t * that the directories exist, on a key value store that is unaware of\n\t * directories, this may be implemented as creating an object for each path.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @throws N5IOException\n\t * if an error occurs during creation\n\t */\n\tvoid createDirectories( final String normalPath ) throws N5IOException;\n\n\t/**\n\t * Delete a path. If the path is a directory, delete it recursively.\n\t *\n\t * @param normalPath\n\t * is expected to be in normalized form, no further\n\t * efforts are made to normalize it.\n\t * @throws N5IOException\n\t * if an error occurs during deletion\n\t */\n\tvoid delete( final String normalPath ) throws N5IOException;\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.util.Iterator;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\n\npublic abstract class LinkedAttributePathToken implements Iterator> {\n\n\t/**\n\t * The JsonElement which contains the mapping that this token represents.\n\t */\n\tprotected T parentJson;\n\n\t/**\n\t * The token representing the subsequent path element.\n\t */\n\tprotected LinkedAttributePathToken childToken;\n\n\t/**\n\t * @return a reference object of type {@code T}\n\t */\n\tpublic abstract T getJsonType();\n\n\t/**\n\t * This method will return the child element, if the {@link #parentJson}\n\t * contains a child that\n\t * is valid for this token. If the {@link #parentJson} does not have a\n\t * child, this method will\n\t * create it. If the {@link #parentJson} does have a child, but it is not\n\t * {@link #jsonCompatible(JsonElement)}\n\t * with our {@link #childToken}, then this method will also create a new\n\t * (compatible) child, and replace\n\t * the existing incompatible child.\n\t *\n\t * @return the resulting child element\n\t */\n\tpublic abstract JsonElement getOrCreateChildElement();\n\n\t/**\n\t * This method will write into {@link #parentJson} the subsequent\n\t * {@link JsonElement}.\n\t *
\n\t * The written JsonElement will EITHER be:\n\t * \n\t * - The result of serializing {@code value} ( if {@link #childToken} is\n\t * {@code null} )
\n\t * - The {@link JsonElement} which represents our {@link #childToken} (See\n\t * {@link #getOrCreateChildElement()} )
\n\t *
\n\t *\n\t * @param gson\n\t * instance used to serialize {@code value }\n\t * @param value\n\t * to write\n\t * @return the object that was written.\n\t */\n\tpublic JsonElement writeChild(final Gson gson, final Object value) {\n\n\t\tif (childToken != null)\n\t\t\t/*\n\t\t\t * If we have a child, get/create it and set current json element to\n\t\t\t * it\n\t\t\t */\n\t\t\treturn getOrCreateChildElement();\n\t\telse {\n\t\t\t/* We are done, no token remaining */\n\t\t\twriteValue(gson, value);\n\t\t\treturn getChildElement();\n\t\t}\n\t}\n\n\t/**\n\t * Check if the provided {@code json} is compatible with this token.\n\t *
\n\t * Compatibility means that the provided {@code JsonElement} does not\n\t * explicitly conflict with this token. That means that {@code null} is\n\t * always compatible. The only real incompatibility is when the\n\t * {@code JsonElement}\n\t * that we receive is different that we expect.\n\t *\n\t * @param json\n\t * element that we are testing for compatibility.\n\t * @return false if {@code json} is not null and is not of type {@code T}\n\t */\n\t@SuppressWarnings(\"BooleanMethodIsAlwaysInverted\")\n\tpublic boolean jsonCompatible(final JsonElement json) {\n\n\t\treturn json == null || json.getClass() == getJsonType().getClass();\n\t}\n\n\t/**\n\t * Write {@code value} into {@link #parentJson}.\n\t *
\n\t * Should only be called when {@link #childToken} is null (i.e. this is the\n\t * last token in the attribute path).\n\t *\n\t * @param gson\n\t * instance used to serialize {@code value}\n\t * @param value\n\t * to serialize\n\t */\n\tprotected abstract void writeValue(Gson gson, Object value);\n\n\t/**\n\t * Write the {@link JsonElement} the corresponds to {@link #childToken} into\n\t * {@link #parentJson}.\n\t */\n\tprotected abstract void writeChildElement();\n\n\t/**\n\t * @return the element that is represented by {@link #childToken} if\n\t * present.\n\t */\n\tprotected abstract JsonElement getChildElement();\n\n\t/**\n\t * If {@code json} is compatible with the type or token that we are, then\n\t * set {@link #parentJson} to {@code json}.\n\t *
\n\t * However, if {@code json} is either {@code null} or not\n\t * {@link #jsonCompatible(JsonElement)} then\n\t * {@link #parentJson} will be set to a new instance of {@code T}.\n\t *\n\t * @param json\n\t * to attempt to set to {@link #parentJson}\n\t * @return the value set to {@link #parentJson}.\n\t */\n\t@SuppressWarnings(\"unchecked\")\n\tprotected JsonElement setAndCreateParentElement(final JsonElement json) {\n\n\t\tif (json == null || !jsonCompatible(json)) {\n\t\t\t// noinspection unchecked\n\t\t\tparentJson = (T)getJsonType().deepCopy();\n\t\t} else {\n\t\t\t// noinspection unchecked\n\t\t\tparentJson = (T)json;\n\t\t}\n\t\treturn parentJson;\n\t}\n\n\t/**\n\t * @return if we have a {@link #childToken}.\n\t */\n\t@Override\n\tpublic boolean hasNext() {\n\n\t\treturn childToken != null;\n\t}\n\n\t/**\n\t * @return {@link #childToken}\n\t */\n\t@Override\n\tpublic LinkedAttributePathToken next() {\n\n\t\treturn childToken;\n\t}\n\n\tpublic static class ObjectAttributeToken extends LinkedAttributePathToken {\n\n\t\tprivate static final JsonObject JSON_OBJECT = new JsonObject();\n\n\t\tprivate final String key;\n\n\t\tpublic ObjectAttributeToken(final String key) {\n\n\t\t\tthis.key = key;\n\t\t}\n\n\t\t/**\n\t\t * @return the {@link #key} this token maps it's\n\t\t * {@link #getChildElement()} to.\n\t\t */\n\t\tpublic String getKey() {\n\n\t\t\treturn key;\n\t\t}\n\n\t\t@Override\n\t\tpublic String toString() {\n\n\t\t\treturn getKey();\n\t\t}\n\n\t\t@Override\n\t\tpublic JsonObject getJsonType() {\n\n\t\t\treturn JSON_OBJECT;\n\t\t}\n\n\t\t@Override\n\t\tpublic JsonElement getOrCreateChildElement() {\n\n\t\t\tfinal JsonElement childElement = parentJson.getAsJsonObject().get(key);\n\t\t\tif (!parentJson.has(key) || childToken != null && !childToken.jsonCompatible(childElement)) {\n\t\t\t\twriteChildElement();\n\t\t\t}\n\t\t\treturn getChildElement();\n\t\t}\n\n\t\t@Override\n\t\tprotected JsonElement getChildElement() {\n\n\t\t\treturn parentJson.get(key);\n\t\t}\n\n\t\t@Override\n\t\tprotected void writeValue(final Gson gson, final Object value) {\n\n\t\t\tparentJson.add(key, gson.toJsonTree(value));\n\t\t}\n\n\t\t@Override\n\t\tprotected void writeChildElement() {\n\n\t\t\tif (childToken != null)\n\t\t\t\tparentJson.add(key, childToken.getJsonType().deepCopy());\n\n\t\t}\n\n\t}\n\n\tpublic static class ArrayAttributeToken extends LinkedAttributePathToken {\n\n\t\tprivate static final JsonArray JSON_ARRAY = new JsonArray();\n\n\t\tprivate final int index;\n\n\t\tpublic ArrayAttributeToken(final int index) {\n\n\t\t\tthis.index = index;\n\t\t}\n\n\t\t/**\n\t\t * @return the {@link #index} this token maps it's\n\t\t * {@link #getChildElement()} to.\n\t\t */\n\t\tpublic int getIndex() {\n\n\t\t\treturn index;\n\t\t}\n\n\t\t@Override\n\t\tpublic String toString() {\n\n\t\t\treturn \"[\" + getIndex() + \"]\";\n\t\t}\n\n\t\t@Override\n\t\tpublic JsonArray getJsonType() {\n\n\t\t\treturn JSON_ARRAY;\n\t\t}\n\n\t\t@Override\n\t\tpublic JsonElement getOrCreateChildElement() {\n\n\t\t\tif (index >= parentJson.size() || childToken != null && !childToken.jsonCompatible(parentJson.get(index)))\n\t\t\t\twriteChildElement();\n\n\t\t\treturn parentJson.get(index);\n\t\t}\n\n\t\t@Override\n\t\tprotected void writeChildElement() {\n\n\t\t\tif (childToken != null) {\n\t\t\t\tfillArrayToIndex(parentJson, index, null);\n\t\t\t\tparentJson.set(index, childToken.getJsonType().deepCopy());\n\t\t\t}\n\t\t}\n\n\t\t@Override\n\t\tprotected JsonElement getChildElement() {\n\n\t\t\treturn parentJson.get(index);\n\t\t}\n\n\t\t@Override\n\t\tprotected void writeValue(final Gson gson, final Object value) {\n\n\t\t\tfillArrayToIndex(parentJson, index, value);\n\t\t\tparentJson.set(index, gson.toJsonTree(value));\n\t\t}\n\n\t\t/**\n\t\t * Fill {@code array} up to, and including, {@code index} with a default\n\t\t * value, determined by the type of {@code value}.\n\t\t * Importantly, this does NOT set {@code array[index]=value}.\n\t\t *\n\t\t * @param array\n\t\t * to fill\n\t\t * @param index\n\t\t * to fill the array to (inclusive)\n\t\t * @param value\n\t\t * used to determine the default array fill value\n\t\t */\n\t\tprivate static void fillArrayToIndex(final JsonArray array, final int index, final Object value) {\n\n\t\t\tfinal JsonElement fillValue;\n\t\t\tif (valueRepresentsANumber(value)) {\n\t\t\t\tfillValue = new JsonPrimitive(0);\n\t\t\t} else {\n\t\t\t\tfillValue = JsonNull.INSTANCE;\n\t\t\t}\n\n\t\t\tfor (int i = array.size(); i <= index; i++) {\n\t\t\t\tarray.add(fillValue);\n\t\t\t}\n\t\t}\n\n\t\t/**\n\t\t * Check if {@value} represents a {@link Number}.\n\t\t * True if {@code value} either:\n\t\t * \n\t\t * - is a {@code Number}, or;
\n\t\t * - is a {@link JsonPrimitive} where\n\t\t * {@link JsonPrimitive#isNumber()}
\n\t\t *
\n\t\t *\n\t\t * @param value\n\t\t * we wish to check if represents a {@link Number} or not.\n\t\t * @return true if {@code value} represents a {@link Number}\n\t\t */\n\t\tprivate static boolean valueRepresentsANumber(final Object value) {\n\n\t\t\treturn value instanceof Number || (value instanceof JsonPrimitive && ((JsonPrimitive)value).isNumber());\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\n\nimport java.io.Closeable;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.io.Reader;\nimport java.io.Writer;\n\n/**\n * A lock on a path that can create a {@link Reader}, {@link Writer},\n * {@link InputStream}, or {@link OutputStream}.\n *\n * @author Stephan Saalfeld\n */\npublic interface LockedChannel extends Closeable {\n\n\t/**\n\t * Create a UTF-8 {@link Reader}.\n\t *\n\t * @return the reader\n\t * @throws N5IOException\n\t * if the reader could not be created\n\t */\n\tReader newReader() throws N5IOException;\n\n\t/**\n\t * Create a new {@link InputStream}.\n\t *\n\t * @return the input stream\n\t * @throws N5IOException\n\t * if an input stream could not be created\n\t */\n\tInputStream newInputStream() throws N5IOException;\n\n\t/**\n\t * Create a new UTF-8 {@link Writer}.\n\t *\n\t * @return the writer\n\t * @throws N5IOException\n\t * if a writer could not be created\n\t */\n\tWriter newWriter() throws N5IOException;\n\n\t/**\n\t * Create a new {@link OutputStream}.\n\t *\n\t * @return the output stream\n\t * @throws N5IOException\n\t * if an output stream could not be created\n\t */\n\tOutputStream newOutputStream() throws N5IOException;\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.Channels;\nimport java.nio.channels.FileChannel;\nimport java.nio.channels.WritableByteChannel;\n\n/**\n * LockedFileChannel implementation for both read and write operations.\n * \n * When closing this {@code LockedFileChannel}, {@code releaseLock} is called,\n * but the {@code channel} is not closed. The channel may be shared among\n * multiple {@code LockedFileChannel} for concurrent readers. {@code\n * releaseLock} should take care of closing the channel when the last reader (or\n * the only writer) is closed.\n */\nclass LockedFileChannel implements Closeable {\n\n\t// TODO: Consider splitting LockedFileChannel into read-only and write-only part.\n\t// TODO: Consider removing LockedFileChannel and having\n\t// FileKeyLockManager.acquireRead() and FileKeyLockManager.acquireWrite()\n\t// return appropriately wrapped SeekableByteChannel.\n\n\tprivate final FileChannel channel;\n\tprivate ReleaseLock releaseLock;\n\n\t@FunctionalInterface\n\tpublic interface ReleaseLock {\n\n\t\tvoid release() throws IOException;\n\t}\n\n\tLockedFileChannel(final FileChannel channel, final ReleaseLock releaseLock) {\n\n\t\tthis.channel = channel;\n\t\tthis.releaseLock = releaseLock;\n\t}\n\n\t/**\n\t * Returns the size of this channel's file.\n\t *
\n\t * See {@link FileChannel#size()}.\n\t */\n\tpublic long size() throws IOException {\n\t\treturn channel.size();\n\t}\n\n\t/**\n\t * Reads a sequence of bytes from this channel into the given buffer,\n\t * starting at the given file position.\n\t *
\n\t * See {@link FileChannel#read(ByteBuffer, long)}.\n\t */\n\tpublic int read(final ByteBuffer dst, final long position) throws IOException {\n\t\treturn channel.read(dst, position);\n\t}\n\n\t/**\n\t * Return an {@link OutputStream} that writes into this channel.\n\t * Closing the OutputStream will close this channel.\n\t */\n\tpublic OutputStream asOutputStream() {\n\t\treturn Channels.newOutputStream(new ClosingChannelWrapper());\n\t}\n\n\tvolatile boolean firstWrite = true;\n\n\tprivate class ClosingChannelWrapper implements WritableByteChannel {\n\n\t\t@Override\n\t\tpublic int write(final ByteBuffer src) throws IOException {\n\t\t\tif (!firstWrite)\n\t\t\t\treturn channel.write(src);\n\n\t\t\ttry {\n\t\t\t\tchannel.truncate(0);\n\t\t\t\treturn channel.write(src);\n\t\t\t} finally {\n\t\t\t\tfirstWrite = false;\n\t\t\t}\n\t\t}\n\n\t\t@Override\n\t\tpublic boolean isOpen() {\n\t\t\treturn channel.isOpen();\n\t\t}\n\n\t\t@Override\n\t\tpublic void close() throws IOException {\n\t\t\tchannel.close();\n\t\t\tLockedFileChannel.this.close();\n\t\t}\n\t}\n\n\t@Override\n\tpublic void close() throws IOException {\n\n\t\tif (releaseLock != null) {\n\t\t\treleaseLock.release();\n\t\t\treleaseLock = null;\n\t\t\t// Mote that setting releaseLock=null here drops the (method)\n\t\t\t// reference to LockKeyState, which potentially allows clearing the\n\t\t\t// WeakReference that FileLockManager holds.\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/LockingPolicy.java",
"content": "package org.janelia.saalfeldlab.n5;\n\n/**\n * File locking policy.\n * \n * Usually, we want to coordinate reads and writs to a container such that\n *
\n * - multiple readers can access a key simultaneously (blocking all writers).
\n * - A writer should have exclusive access to a key (blocking all other readers and writers).
\n *
\n * However, this cannot always be enforced for all backends:\n * For SMB on macOS OS-level file locking is broken.\n * For AWS S3 and Google Cloud we can detect (but not prevent) concurrent modifications.\n * \n * Sometimes we know that we can disregard locking, for example in read-only settings.\n *
\n * {@code IoPolicy} can be used to configure locking policy for backends ({@link\n * KeyValueAccess}) that support various locking strategies (potentially coming\n * with performance trade-offs).\n *
\n * The policy values ({@link #STRICT}, {@link #UNSAFE}, {@link #PERMISSIVE})\n * specify intent. Detailed interpretation is up to the backend implementation.\n */\npublic enum LockingPolicy {\n\n\t/**\n\t * Protect all reads and writes by locks.\n\t * Fail if locking is not possible.\n\t */\n\tSTRICT,\n\n\t/**\n\t * Reads and writes are unprotected.\n\t */\n\tUNSAFE,\n\n\t/**\n\t * Try to lock for all reads and writes.\n\t * Fall back to unprotected reads and writes if locking is not possible.\n\t * This is the default.\n\t */\n\tPERMISSIVE;\n\n\tstatic LockingPolicy fromString(final String s) {\n\t\tif (\"strict\".equalsIgnoreCase(s))\n\t\t\treturn STRICT;\n\t\telse if (\"unsafe\".equalsIgnoreCase(s))\n\t\t\treturn UNSAFE;\n//\t\telse if (\"permissive\".equalsIgnoreCase(s))\n//\t\t\treturn PERMISSIVE;\n\t\telse\n\t\t\treturn PERMISSIVE;\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class LongArrayDataBlock extends AbstractDataBlock {\n\n\tpublic LongArrayDataBlock(final int[] size, final long[] gridPosition, final long[] data) {\n\n\t\tsuper(size, gridPosition, data, a -> a.length);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport net.jpountz.lz4.LZ4BlockInputStream;\nimport net.jpountz.lz4.LZ4BlockOutputStream;\nimport org.janelia.saalfeldlab.n5.Compression.CompressionType;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.serialization.NameConfig;\n\n@CompressionType(\"lz4\")\n@NameConfig.Name(\"lz4\")\npublic class Lz4Compression implements Compression {\n\n\tprivate static final long serialVersionUID = -9071316415067427256L;\n\n\t@CompressionParameter\n\t@NameConfig.Parameter\n\tprivate final int blockSize;\n\n\tpublic Lz4Compression(final int blockSize) {\n\n\t\tthis.blockSize = blockSize;\n\t}\n\n\tpublic Lz4Compression() {\n\n\t\tthis(1 << 16);\n\t}\n\n\t@Override\n\tpublic boolean equals(final Object other) {\n\n\t\tif (other == null || other.getClass() != Lz4Compression.class)\n\t\t\treturn false;\n\t\telse\n\t\t\treturn blockSize == ((Lz4Compression)other).blockSize;\n\t}\n\n\t@Override\n\tpublic ReadData decode(final ReadData readData) {\n\n\t\treturn ReadData.from(new LZ4BlockInputStream(readData.inputStream()));\n\t}\n\n\t@Override\n\tpublic ReadData encode(final ReadData readData) {\n\t\treturn readData.encode(out -> new LZ4BlockOutputStream(out, blockSize));\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class N5Exception extends RuntimeException {\n\n\tpublic N5Exception() {\n\n\t\tsuper();\n\t}\n\n\tpublic N5Exception(final String message) {\n\n\t\tsuper(message);\n\t}\n\n\tpublic N5Exception(final String message, final Throwable cause) {\n\n\t\tsuper(message, cause);\n\t}\n\n\tpublic N5Exception(final Throwable cause) {\n\n\t\tsuper(cause);\n\t}\n\n\tprotected N5Exception(\n\t\t\tfinal String message,\n\t\t\tfinal Throwable cause,\n\t\t\tfinal boolean enableSuppression,\n\t\t\tfinal boolean writableStackTrace) {\n\n\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t}\n\n\tpublic static class N5IOException extends N5Exception{\n\n\t\tpublic N5IOException(final String message) {\n\n\t\t\tsuper(message);\n\t\t}\n\n\t\tpublic N5IOException(final String message, final Throwable cause) {\n\n\t\t\tsuper(message, cause);\n\t\t}\n\n\t\tpublic N5IOException(final Throwable cause) {\n\n\t\t\tsuper(cause);\n\t\t}\n\n\t\tprotected N5IOException(\n\t\t\t\tfinal String message,\n\t\t\t\tfinal Throwable cause,\n\t\t\t\tfinal boolean enableSuppression,\n\t\t\t\tfinal boolean writableStackTrace) {\n\n\t\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t\t}\n\t}\n\n\t/**\n\t * This excpetion represents the situation when an attribute is requested by key as a specific Class,\n\t * \tthat attribute key does exist, but is not parseable as the desired Class\n\t */\n\tpublic static class N5ClassCastException extends N5Exception {\n\n\n\t\tpublic N5ClassCastException(final Class cls) {\n\n\t\t\tsuper(\"Cannot cast as class \" + cls.getName());\n\t\t}\n\n\t\tpublic N5ClassCastException(final String message) {\n\n\t\t\tsuper(message);\n\t\t}\n\n\t\tpublic N5ClassCastException(final String message, final Throwable cause) {\n\n\t\t\tsuper(message, cause);\n\t\t}\n\n\t\tpublic N5ClassCastException(final Throwable cause) {\n\n\t\t\tsuper(cause);\n\t\t}\n\n\t\tprotected N5ClassCastException(\n\t\t\t\tfinal String message,\n\t\t\t\tfinal Throwable cause,\n\t\t\t\tfinal boolean enableSuppression,\n\t\t\t\tfinal boolean writableStackTrace) {\n\n\t\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t\t}\n\t}\n\n\tpublic static class N5NoSuchKeyException extends N5IOException {\n\n\t\tpublic N5NoSuchKeyException(final String message) {\n\n\t\t\tsuper(message);\n\t\t}\n\n\t\tpublic N5NoSuchKeyException(final String message, final Throwable cause) {\n\n\t\t\tsuper(message, cause);\n\t\t}\n\n\t\tpublic N5NoSuchKeyException(final Throwable cause) {\n\n\t\t\tsuper(cause);\n\t\t}\n\n\t\tprotected N5NoSuchKeyException(\n\t\t\t\tfinal String message,\n\t\t\t\tfinal Throwable cause,\n\t\t\t\tfinal boolean enableSuppression,\n\t\t\t\tfinal boolean writableStackTrace) {\n\n\t\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t\t}\n\t}\n\n\t/**\n\t * Exception to represent an error when attempting to parse json attributes\n\t */\n\tpublic static class N5JsonParseException extends N5Exception {\n\t\tpublic N5JsonParseException(final String message) {\n\n\t\t\tsuper(message);\n\t\t}\n\n\t\tpublic N5JsonParseException(final String message, final Throwable cause) {\n\n\t\t\tsuper(message, cause);\n\t\t}\n\n\t\tpublic N5JsonParseException(final Throwable cause) {\n\n\t\t\tsuper(cause);\n\t\t}\n\n\t\tprotected N5JsonParseException(\n\t\t\t\tfinal String message,\n\t\t\t\tfinal Throwable cause,\n\t\t\t\tfinal boolean enableSuppression,\n\t\t\t\tfinal boolean writableStackTrace) {\n\n\t\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t\t}\n\t}\n\n\n\tpublic static class N5ConcurrentModificationException extends N5IOException {\n\n\t\tpublic N5ConcurrentModificationException(final String message) {\n\n\t\t\tsuper(message);\n\t\t}\n\n\t\tpublic N5ConcurrentModificationException(final String message, final Throwable cause) {\n\n\t\t\tsuper(message, cause);\n\t\t}\n\n\t\tpublic N5ConcurrentModificationException(final Throwable cause) {\n\n\t\t\tsuper(cause);\n\t\t}\n\n\t\tprotected N5ConcurrentModificationException(\n\t\t\t\tfinal String message,\n\t\t\t\tfinal Throwable cause,\n\t\t\t\tfinal boolean enableSuppression,\n\t\t\t\tfinal boolean writableStackTrace) {\n\n\t\t\tsuper(message, cause, enableSuppression, writableStackTrace);\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.nio.file.FileSystems;\n\nimport com.google.gson.Gson;\nimport com.google.gson.GsonBuilder;\n\n/**\n * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON\n * attributes parsed with {@link Gson}.\n *\n * @author Stephan Saalfeld\n * @author Igor Pisarev\n * @author Philipp Hanslovsky\n */\n//public class N5FSReader extends N5KeyValueReader {\npublic class N5FSReader extends N5KeyValueReader {\n\n\t/**\n\t * Opens an {@link N5FSReader} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * @param basePath\n\t * N5 base path\n\t * @param gsonBuilder\n\t * the gson builder\n\t * @param cacheMeta\n\t * cache attributes and meta data\n\t * Setting this to true avoids frequent reading and parsing of\n\t * JSON encoded attributes and other meta data that requires\n\t * accessing the store. This is most interesting for high latency\n\t * backends. Changes of cached attributes and meta data by an\n\t * independent writer on the same container will not be tracked.\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tpublic N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta)\n\t\t\tthrows N5Exception {\n\n\t\tsuper(\n\t\t\t\tnew FileSystemKeyValueAccess(),\n\t\t\t\tbasePath,\n\t\t\t\tgsonBuilder,\n\t\t\t\tcacheMeta);\n\n\t\tif (!exists(\"/\"))\n\t\t\tthrow new N5Exception.N5IOException(\"No container exists at \" + basePath);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSReader} at a given base path.\n\t *\n\t * @param basePath\n\t * N5 base path\n\t * @param cacheMeta\n\t * cache attributes and meta data\n\t * Setting this to true avoids frequent reading and parsing of\n\t * JSON encoded attributes and other meta data that requires\n\t * accessing the store. This is most interesting for high latency\n\t * backends. Changes of cached attributes and meta data by an\n\t * independent writer on the same container will not be tracked.\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tpublic N5FSReader(final String basePath, final boolean cacheMeta) throws N5Exception {\n\n\t\tthis(basePath, new GsonBuilder(), cacheMeta);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSReader} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * @param basePath\n\t * N5 base path\n\t * @param gsonBuilder\n\t * the gson builder\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tpublic N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception {\n\n\t\tthis(basePath, gsonBuilder, false);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSReader} at a given base path.\n\t *\n\t * @param basePath\n\t * N5 base path\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tpublic N5FSReader(final String basePath) throws N5Exception {\n\n\t\tthis(basePath, new GsonBuilder(), false);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.nio.file.FileSystems;\n\nimport com.google.gson.GsonBuilder;\n\n/**\n * Filesystem {@link N5Writer} implementation with version compatibility check.\n *\n * @author Stephan Saalfeld\n */\npublic class N5FSWriter extends N5KeyValueWriter {\n\n\t/**\n\t * Opens an {@link N5FSWriter} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * If the base path does not exist, it will be created.\n\t *\n\t * If the base path exists and if the N5 version of the container is\n\t * compatible with this implementation, the N5 version of this container\n\t * will be set to the current N5 version of this implementation.\n\t *\n\t * @param basePath\n\t * n5 base path\n\t * @param gsonBuilder\n\t * the gson builder\n\t * @param cacheAttributes\n\t * cache attributes and meta data\n\t * Setting this to true avoidsfrequent reading and parsing of\n\t * JSON encoded attributes andother meta data that requires\n\t * accessing the store. This ismost interesting for high latency\n\t * backends. Changes of cachedattributes and meta data by an\n\t * independent writer on the samecontainer will not be tracked.\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be written to or cannot be created,\n\t * if the N5 version of the container is not compatible with\n\t * this implementation.\n\t */\n\tpublic N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes)\n\t\t\tthrows N5Exception {\n\n\t\tsuper(\n\t\t\t\tnew FileSystemKeyValueAccess(),\n\t\t\t\tbasePath,\n\t\t\t\tgsonBuilder,\n\t\t\t\tcacheAttributes);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSWriter} at a given base path.\n\t *\n\t * If the base path does not exist, it will be created.\n\t *\n\t * If the base path exists and if the N5 version of the container is\n\t * compatible with this implementation, the N5 version of this container\n\t * will be set to the current N5 version of this implementation.\n\t *\n\t * @param basePath\n\t * base path\n\t * @param cacheAttributes\n\t * attributes and meta data\n\t * Setting this to true avoidsfrequent reading and parsing of\n\t * JSON encoded attributes andother meta data that requires\n\t * accessing the store. This ismost interesting for high latency\n\t * backends. Changes of cachedattributes and meta data by an\n\t * independent writer on the samecontainer will not be tracked.\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be written to or cannot be created,\n\t * if the N5 version of the container is not compatible with\n\t * this implementation.\n\t */\n\tpublic N5FSWriter(final String basePath, final boolean cacheAttributes) throws N5Exception {\n\n\t\tthis(basePath, new GsonBuilder(), cacheAttributes);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSWriter} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t * \n\t * If the base path does not exist, it will be created.\n\t *
\n\t * \n\t * If the base path exists and if the N5 version of the container is\n\t * compatible with this implementation, the N5 version of this container\n\t * will be set to the current N5 version of this implementation.\n\t *
\n\t *\n\t * @param basePath\n\t * base path\n\t * @param gsonBuilder\n\t * gson builder\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be written to or cannot be created,\n\t * if the N5 version of the container is not compatible with\n\t * this implementation.\n\t */\n\tpublic N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception {\n\n\t\tthis(basePath, gsonBuilder, false);\n\t}\n\n\t/**\n\t * Opens an {@link N5FSWriter} at a given base path.\n\t * \n\t * If the base path does not exist, it will be created.\n\t *
\n\t * \n\t * If the base path exists and if the N5 version of the container is\n\t * compatible with this implementation, the N5 version of this container\n\t * will be set to the current N5 version of this implementation.\n\t *
\n\t *\n\t * @param basePath\n\t * n5 base path\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be written to or cannot be created,\n\t * if the N5 version of the container is not compatible with\n\t * this implementation.\n\t */\n\tpublic N5FSWriter(final String basePath) throws N5Exception {\n\n\t\tthis(basePath, new GsonBuilder());\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.net.URI;\nimport java.net.URISyntaxException;\n\nimport com.google.gson.JsonElement;\nimport org.janelia.saalfeldlab.n5.cache.N5JsonCache;\n\nimport com.google.gson.Gson;\nimport com.google.gson.GsonBuilder;\n\n/**\n * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON\n * attributes parsed with {@link Gson}.\n *\n * @author Stephan Saalfeld\n * @author Igor Pisarev\n * @author Philipp Hanslovsky\n */\npublic class N5KeyValueReader implements CachedGsonKeyValueN5Reader {\n\n\tpublic static final String ATTRIBUTES_JSON = \"attributes.json\";\n\n\tprotected final KeyValueAccess keyValueAccess;\n\n\tprotected final Gson gson;\n\tprotected final boolean cacheMeta;\n\tprotected URI uri;\n\n\tprivate final N5JsonCache cache;\n\n\t/**\n\t * Opens an {@link N5KeyValueReader} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * @param keyValueAccess\n\t * \t\t\t the KeyValueAccess backend used\n\t * @param basePath\n\t * N5 base path\n\t * @param gsonBuilder\n\t * \t\t\t the GsonBuilder\n\t * @param cacheMeta\n\t * cache attributes and meta data\n\t * Setting this to true avoidsfrequent reading and parsing of\n\t * JSON encoded attributes andother meta data that requires\n\t * accessing the store. This ismost interesting for high latency\n\t * backends. Changes of cachedattributes and meta data by an\n\t * independent writer will not betracked.\n\t *\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tpublic N5KeyValueReader(\n\t\t\tfinal KeyValueAccess keyValueAccess,\n\t\t\tfinal String basePath,\n\t\t\tfinal GsonBuilder gsonBuilder,\n\t\t\tfinal boolean cacheMeta)\n\t\t\tthrows N5Exception {\n\n\t\tthis(true, keyValueAccess, basePath, gsonBuilder, cacheMeta, true);\n\t}\n\n\t/**\n\t * Opens an {@link N5KeyValueReader} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * @param checkVersion\n\t * the version check\n\t * @param keyValueAccess\n\t * the backend KeyValueAccess used\n\t * @param basePath\n\t * base path\n\t * @param gsonBuilder\n\t * the GsonBuilder\n\t * @param cacheMeta\n\t * cache attributes and meta data Setting this to true avoids\n\t * frequent reading and parsing of JSON encoded attributes and\n\t * other meta data that requires accessing the store. This is\n\t * most interesting for high latency backends. Changes of cached\n\t * attributes and meta data by an independent writer will not be\n\t * tracked.\n\t * @param checkExists\n\t * if true, an N5IOException will be thrown if a container does\n\t * not exist at the specified location\n\t * @throws N5Exception\n\t * if the base path cannot be read or does not exist, if the N5\n\t * version of the container is not compatible with this\n\t * implementation.\n\t */\n\tprotected N5KeyValueReader(\n\t\t\tfinal boolean checkVersion,\n\t\t\tfinal KeyValueAccess keyValueAccess,\n\t\t\tfinal String basePath,\n\t\t\tfinal GsonBuilder gsonBuilder,\n\t\t\tfinal boolean cacheMeta,\n\t\t\tfinal boolean checkExists)\n\t\t\tthrows N5Exception {\n\n\t\tthis.keyValueAccess = keyValueAccess;\n\t\tthis.gson = registerGson(gsonBuilder).create();\n\t\tthis.cacheMeta = cacheMeta;\n\n\t\tif (this.cacheMeta)\n\t\t\tthis.cache = newCache();\n\t\telse\n\t\t\tthis.cache = null;\n\n\t\ttry {\n\t\t\turi = keyValueAccess.uri(basePath);\n\t\t} catch (final URISyntaxException e) {\n\t\t\tthrow new N5Exception(e);\n\t\t}\n\n\t\tboolean versionFound = false;\n\t\tif (checkVersion) {\n\t\t\t/* Existence checks, if any, go in subclasses */\n\t\t\t/* Check that version (if there is one) is compatible. */\n\t\t\tfinal Version version = getVersion();\n\t\t\tversionFound = !version.equals(NO_VERSION);\n\t\t\tif (!VERSION.isCompatible(version))\n\t\t\t\tthrow new N5Exception.N5IOException(\n\t\t\t\t\t\"Incompatible version \" + version + \" (this is \" + VERSION + \").\");\n\t\t}\n\n\t\t// if a version was found, the container exists - don't need to check again\n\t\tif (checkExists && (!versionFound && !inferExistence(\"/\")))\n\t\t\tthrow new N5Exception.N5IOException(\"No container exists at \" + basePath);\n\t}\n\n\tprivate boolean inferExistence(String path) {\n\n\t\tfinal JsonElement attributes = getAttributes(path);\n\t\treturn attributes != null || exists(path);\n\t}\n\n\tprotected GsonBuilder registerGson(final GsonBuilder gsonBuilder) {\n\n\t\tgsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());\n\t\tgsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());\n\t\tgsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter());\n\t\tgsonBuilder.disableHtmlEscaping();\n\t\treturn gsonBuilder;\n\t}\n\n\t@Override\n\tpublic String getAttributesKey() {\n\n\t\treturn ATTRIBUTES_JSON;\n\t}\n\n\t@Override\n\tpublic Gson getGson() {\n\n\t\treturn gson;\n\t}\n\n\t@Override\n\tpublic KeyValueAccess getKeyValueAccess() {\n\n\t\treturn keyValueAccess;\n\t}\n\n\t@Override\n\tpublic URI getURI() {\n\n\t\treturn uri;\n\t}\n\n\t@Override\n\tpublic boolean cacheMeta() {\n\n\t\treturn cacheMeta;\n\t}\n\n\t@Override\n\tpublic N5JsonCache getCache() {\n\n\t\treturn this.cache;\n\t}\n\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport com.google.gson.GsonBuilder;\n\n/**\n * Filesystem {@link N5Writer} implementation with version compatibility check.\n *\n * @author Stephan Saalfeld\n */\npublic class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyValueN5Writer {\n\n\t/**\n\t * Opens an {@link N5KeyValueWriter} at a given base path with a custom\n\t * {@link GsonBuilder} to support custom attributes.\n\t *\n\t * \n\t * If the base path does not exist, it will be created.\n\t *
\n\t * If the base path exists and if the N5 version of the container is\n\t * compatible with this implementation, the N5 version of this container\n\t * will be set to the current N5 version of this implementation.\n\t *\n\t * @param keyValueAccess\n\t * \t\t\t the backend key value access to use\n\t * @param basePath\n\t * n5 base path\n\t * @param gsonBuilder\n\t * the gson builder\n\t * @param cacheAttributes\n\t * Setting this to true avoids frequent reading and parsing of\n\t * JSON encoded attributes, this is most interesting for high\n\t * latency file systems. Changes of attributes by an independent\n\t * writer will not be tracked.\n\t * @throws N5Exception\n\t * if the base path cannot be written to or cannot be created,\n\t * if the N5 version of the container is not compatible with\n\t * this implementation.\n\t */\n\tpublic N5KeyValueWriter(\n\t\t\tfinal KeyValueAccess keyValueAccess,\n\t\t\tfinal String basePath,\n\t\t\tfinal GsonBuilder gsonBuilder,\n\t\t\tfinal boolean cacheAttributes)\n\t\t\tthrows N5Exception {\n\n\t\tsuper(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes, false);\n\n\t\tVersion version = null;\n\t\ttry {\n\t\t\tversion = getVersion();\n\t\t\tif (!VERSION.isCompatible(version))\n\t\t\t\tthrow new N5Exception.N5IOException(\"Incompatible version \" + version + \" (this is \" + VERSION + \").\");\n\t\t} catch (final NullPointerException e) {}\n\n\t\tif (version == null || version.equals(new Version(0, 0, 0, \"\"))) {\n\t\t\tcreateGroup(\"/\");\n\t\t\tsetVersion(\"/\");\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.io.ObjectInputStream;\nimport java.io.Serializable;\nimport java.io.UncheckedIOException;\nimport java.lang.reflect.Type;\nimport java.net.URI;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.function.Predicate;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;\n\n/**\n * A simple structured container for hierarchies of chunked\n * n-dimensional datasets and attributes.\n *\n * @author Stephan Saalfeld\n * @see \"https://github.com/axtimwalde/n5\"\n */\npublic interface N5Reader extends AutoCloseable {\n\n\tclass Version {\n\n\t\tprivate final int major;\n\t\tprivate final int minor;\n\t\tprivate final int patch;\n\t\tprivate final String suffix;\n\n\t\tpublic Version(\n\t\t\t\tfinal int major,\n\t\t\t\tfinal int minor,\n\t\t\t\tfinal int patch,\n\t\t\t\tfinal String rest) {\n\n\t\t\tthis.major = major;\n\t\t\tthis.minor = minor;\n\t\t\tthis.patch = patch;\n\t\t\tthis.suffix = rest;\n\t\t}\n\n\t\tpublic Version(\n\t\t\t\tfinal int major,\n\t\t\t\tfinal int minor,\n\t\t\t\tfinal int patch) {\n\n\t\t\tthis(major, minor, patch, \"\");\n\t\t}\n\n\t\t/**\n\t\t * Creates a version from a SemVer compatible version string.\n\t\t *
\n\t\t * If the version string is null or not a SemVer version, this\n\t\t * version will be \"0.0.0\"\n\t\t *
\n\t\t *\n\t\t * @param versionString\n\t\t * the string representation of the version\n\t\t */\n\t\tpublic Version(final String versionString) {\n\n\t\t\tboolean isSemVer = false;\n\t\t\tif (versionString != null) {\n\t\t\t\tfinal Matcher matcher = Pattern.compile(\"(\\\\d+)(\\\\.(\\\\d+))?(\\\\.(\\\\d+))?(.*)\").matcher(versionString);\n\t\t\t\tisSemVer = matcher.find();\n\t\t\t\tif (isSemVer) {\n\t\t\t\t\tmajor = Integer.parseInt(matcher.group(1));\n\t\t\t\t\tfinal String minorString = matcher.group(3);\n\t\t\t\t\tif (!minorString.equals(\"\"))\n\t\t\t\t\t\tminor = Integer.parseInt(minorString);\n\t\t\t\t\telse\n\t\t\t\t\t\tminor = 0;\n\t\t\t\t\tfinal String patchString = matcher.group(5);\n\t\t\t\t\tif (!patchString.equals(\"\"))\n\t\t\t\t\t\tpatch = Integer.parseInt(patchString);\n\t\t\t\t\telse\n\t\t\t\t\t\tpatch = 0;\n\t\t\t\t\tsuffix = matcher.group(6);\n\t\t\t\t} else {\n\t\t\t\t\tmajor = 0;\n\t\t\t\t\tminor = 0;\n\t\t\t\t\tpatch = 0;\n\t\t\t\t\tsuffix = \"\";\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tmajor = 0;\n\t\t\t\tminor = 0;\n\t\t\t\tpatch = 0;\n\t\t\t\tsuffix = \"\";\n\t\t\t}\n\t\t}\n\n\t\tpublic final int getMajor() {\n\n\t\t\treturn major;\n\t\t}\n\n\t\tpublic final int getMinor() {\n\n\t\t\treturn minor;\n\t\t}\n\n\t\tpublic final int getPatch() {\n\n\t\t\treturn patch;\n\t\t}\n\n\t\tpublic final String getSuffix() {\n\n\t\t\treturn suffix;\n\t\t}\n\n\t\t@Override\n\t\tpublic String toString() {\n\n\t\t\tfinal StringBuilder s = new StringBuilder();\n\t\t\ts.append(major);\n\t\t\ts.append(\".\");\n\t\t\ts.append(minor);\n\t\t\ts.append(\".\");\n\t\t\ts.append(patch);\n\t\t\ts.append(suffix);\n\n\t\t\treturn s.toString();\n\t\t}\n\n\t\t@Override\n\t\tpublic boolean equals(final Object other) {\n\n\t\t\tif (other instanceof Version) {\n\t\t\t\tfinal Version otherVersion = (Version)other;\n\t\t\t\treturn (major == otherVersion.major) &\n\t\t\t\t\t\t(minor == otherVersion.minor) &\n\t\t\t\t\t\t(patch == otherVersion.patch) &\n\t\t\t\t\t\t(suffix.equals(otherVersion.suffix));\n\t\t\t} else\n\t\t\t\treturn false;\n\t\t}\n\n\t\t/**\n\t\t * Returns true if this implementation is compatible with a given\n\t\t * version.\n\t\t *\n\t\t * Currently, this means that the version is less than or equal to\n\t\t * 1.X.X.\n\t\t *\n\t\t * @param version\n\t\t * the version\n\t\t * @return true if this version is compatible\n\t\t */\n\t\tpublic boolean isCompatible(final Version version) {\n\n\t\t\treturn version.getMajor() <= major;\n\t\t}\n\t}\n\n\t/**\n\t * SemVer version of this N5 spec.\n\t */\n\tVersion NO_VERSION = new Version(0, 0, 0);\n\n\t/**\n\t * SemVer version of this N5 spec.\n\t */\n\tVersion VERSION = new Version(4, 0, 0);\n\n\t/**\n\t * Version attribute key.\n\t */\n\tString VERSION_KEY = \"n5\";\n\n\t/**\n\t * Get the SemVer version of this container as specified in the 'version'\n\t * attribute of the root group.\n\t *\n\t * If no version is specified or the version string does not conform to\n\t * the SemVer format, 0.0.0 will be returned. For incomplete versions,\n\t * such as 1.2, the missing elements are filled with 0, i.e. 1.2.0 in this\n\t * case.\n\t *\n\t * @return the version\n\t * @throws N5Exception\n\t * the exception\n\t */\n\tdefault Version getVersion() throws N5Exception {\n\n\t\treturn new Version(getAttribute(\"/\", VERSION_KEY, String.class));\n\t}\n\n\t/**\n\t * Returns the URI of the container's root.\n\t *\n\t * @return the base path URI\n\t */\n\tURI getURI();\n\n\t/**\n\t * Reads an attribute.\n\t *\n\t * @param pathName\n\t * group path\n\t * @param key\n\t * the key\n\t * @param clazz\n\t * attribute class\n\t * @param \n\t * the attribute type\n\t * @return the attribute\n\t * @throws N5Exception\n\t * the exception\n\t */\n\t T getAttribute(\n\t\t\tString pathName,\n\t\t\tString key,\n\t\t\tClass clazz) throws N5Exception;\n\n\t/**\n\t * Reads an attribute.\n\t *\n\t * @param pathName\n\t * group path\n\t * @param key\n\t * the key\n\t * @param type\n\t * attribute Type (use this for specifying generic types)\n\t * @param \n\t * the attribute type\n\t * @return the attribute\n\t * @throws N5Exception\n\t * the exception\n\t */\n\t T getAttribute(\n\t\t\tString pathName,\n\t\t\tString key,\n\t\t\tType type) throws N5Exception;\n\n\t/**\n\t * Get mandatory dataset attributes.\n\t *\n\t * @param pathName\n\t * dataset path\n\t * @return dataset attributes or null if either dimensions or dataType are\n\t * not set\n\t * @throws N5Exception\n\t * the exception\n\t */\n\tDatasetAttributes getDatasetAttributes(String pathName) throws N5Exception;\n\n\t/**\n\t * Some implementations may need to convert arbitrary DatasetAttributes to their specific equivalent variant.\n\t * Ideally, this method would be `protected`, but that's not valid for the interface. The default implementation\n\t * is the identity (returns the input DatasetAttributes unchanged).\n\t * \n\t * The returned DatasetAttributes is not guaranteed to be a unique instance.\n\t *\n\t * @param attributes\n\t * \t\t\t\tto convert\n\t * @return the converted attributes\n\t */\n\tdefault DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes attributes) {\n\t\treturn attributes;\n\t}\n\n\n\t/**\n\t * Reads a chunk as a {@link DataBlock}.\n\t *\n\t * @param \n\t * the DataBlock data type\n\t * @param pathName\n\t * dataset path\n\t * @param datasetAttributes\n\t * the dataset attributes\n\t * @param gridPosition\n\t * the grid position\n\t * @return the data block\n\t * @throws N5Exception\n\t * the exception\n\t */\n\t DataBlock readChunk(\n\t\t\tString pathName,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong... gridPosition) throws N5Exception;\n\n\t/**\n\t * Reads multiple chunks as {@link DataBlock}s.\n\t * \n\t * Implementations may optimize / batch read operations when possible, e.g.\n\t * in the case that the datasets are sharded.\n\t *\n\t * @param \n\t * the DataBlock data type\n\t * @param pathName\n\t * dataset path\n\t * @param datasetAttributes\n\t * the dataset attributes\n\t * @param gridPositions\n\t * a list of grid positions\n\t * @return a list of data blocks\n\t * @throws N5Exception\n\t * the exception\n\t */\n\tdefault List> readChunks(\n\t\t\tfinal String pathName,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal List gridPositions) throws N5Exception {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tfinal ArrayList> blocks = new ArrayList<>();\n\t\tfor( final long[] p : gridPositions )\n\t\t\tblocks.add(readChunk(pathName, convertedDatasetAttributes, p));\n\n\t\treturn blocks;\n\t}\n\n\t/**\n\t * Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded).\n\t * \n\t * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}\n\t * This method's behavior is identical to {@link #readChunk} for un-sharded datasets.\n\t *\n\t * @param \n\t * the DataBlock data type\n\t * @param pathName\n\t * dataset path\n\t * @param datasetAttributes\n\t * the dataset attributes\n\t * @param gridPosition\n\t * the position in the block grid\n\t * @return the data block\n\t * @throws N5Exception\n\t * the exception\n\t *\n\t * @see DatasetAttributes#getNestedBlockGrid()\n\t */\n\t DataBlock readBlock(\n\t\t\tString pathName,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong... gridPosition) throws N5Exception;\n\n\t/**\n\t * Checks if a block exists at the given grid position without reading the data.\n\t * \n\t * A block is the highest (coarsest) level of the dataset's {@link\n\t * NestedGrid}, that is, a shard if the dataset is sharded, or a chunk\n\t * otherwise.\n\t *
\n\t * This method only checks for the presence of the key value for the gridPosition, it does not\n\t * read or validate the contents. As a result, this method refers to chunks in un-sharded datasets\n\t * or shards in sharded datasets.\n\t *\n\t * @param pathName\n\t * dataset path\n\t * @param datasetAttributes\n\t * the dataset attributes\n\t * @param gridPosition\n\t * the block grid position\n\t * @return true if the block file exists\n\t * @throws N5Exception\n\t * the exception\n\t */\n\tboolean blockExists(\n\t\t\tString pathName,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong... gridPosition) throws N5Exception;\n\n\t/**\n\t * Load a {@link DataBlock} as a {@link Serializable}. The offset is given\n\t * in\n\t * {@link DataBlock} grid coordinates.\n\t *\n\t * @param dataset\n\t * the dataset path\n\t * @param attributes\n\t * the dataset attributes\n\t * @param \n\t * the data block type\n\t * @param gridPosition\n\t * the grid position\n\t * @return the data block\n\t * @throws N5Exception\n\t * the exception\n\t * @throws ClassNotFoundException\n\t * the class not found exception\n\t */\n\tdefault T readSerializedBlock(\n\t\t\tfinal String dataset,\n\t\t\tfinal DatasetAttributes attributes,\n\t\t\tfinal long... gridPosition) throws N5Exception, ClassNotFoundException {\n\n\t\tDatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(attributes);\n\t\tfinal DataBlock block = readChunk(dataset, convertedDatasetAttributes, gridPosition);\n\t\tif (block == null)\n\t\t\treturn null;\n\n\t\tfinal ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.getData());\n\t\ttry (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) {\n\t\t\treturn (T) in.readObject();\n\t\t} catch (final IOException | UncheckedIOException e) {\n\t\t\tthrow new N5Exception.N5IOException(e);\n\t\t}\n\t}\n\n\t/**\n\t * Test whether a group or dataset exists at a given path.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return true if the path exists\n\t */\n\tboolean exists(String pathName);\n\n\t/**\n\t * Test whether a dataset exists at a given path.\n\t *\n\t * @param pathName\n\t * dataset path\n\t * @return true if a dataset exists\n\t * @throws N5Exception\n\t * an exception is thrown if the existence of a dataset at the given path cannot be determined.\n\t */\n\tdefault boolean datasetExists(final String pathName) throws N5Exception {\n\n\t\treturn exists(pathName) && getDatasetAttributes(pathName) != null;\n\t}\n\n\t/**\n\t * List all groups (including datasets) in a group.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return list of children\n\t * @throws N5Exception\n\t * an exception is thrown if pathName is not a valid group\n\t */\n\tString[] list(String pathName) throws N5Exception;\n\n\t/**\n\t * Recursively list all groups and datasets in the given path.\n\t * Only paths that satisfy the provided filter will be included, but the\n\t * children of paths that were excluded may be included (filter does not\n\t * apply to the subtree).\n\t *\n\t * @param pathName\n\t * base group path\n\t * @param filter\n\t * filter for children to be included\n\t * @return list of child groups and datasets\n\t * @throws N5Exception\n\t * \t\tan exception is thrown if pathName is not a valid group\n\t */\n\tdefault String[] deepList(\n\t\t\tfinal String pathName,\n\t\t\tfinal Predicate filter) throws N5Exception {\n\n\t\tfinal String groupSeparator = getGroupSeparator();\n\t\tfinal String normalPathName = pathName\n\t\t\t\t.replaceAll(\n\t\t\t\t\t\t\"(^\" + groupSeparator + \"*)|(\" + groupSeparator + \"*$)\",\n\t\t\t\t\t\t\"\");\n\n\t\tfinal List absolutePaths = deepList(this, normalPathName, false, filter);\n\t\treturn absolutePaths\n\t\t\t\t.stream()\n\t\t\t\t.map(a -> a.replaceFirst(normalPathName + \"(\" + groupSeparator + \"?)\", \"\"))\n\t\t\t\t.filter(a -> !a.isEmpty())\n\t\t\t\t.toArray(String[]::new);\n\t}\n\n\t/**\n\t * Recursively list all groups and datasets in the given path.\n\t *\n\t * @param pathName\n\t * base group path\n\t * @return list of groups and datasets\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t */\n\tdefault String[] deepList(final String pathName) throws N5Exception {\n\n\t\treturn deepList(pathName, a -> true);\n\t}\n\n\t/**\n\t * Recursively list all datasets in the given path. Only paths that satisfy the\n\t * provided filter will be included, but the children of paths that were\n\t * excluded may be included (filter does not apply to the subtree).\n\t *\n\t * \n\t * This method delivers the same results as\n\t *
\n\t *\n\t * \n\t * {@code\n\t * n5.deepList(prefix, a -> {\n\t * \ttry {\n\t * \t\treturn n5.datasetExists(a) && filter.test(a);\n\t * \t} catch (final N5Exception e) {\n\t * \t\treturn false;\n\t * \t}\n\t * });\n\t * }\n\t * \n\t * \n\t * but will execute {@link #datasetExists(String)} only once per node. This can\n\t * be relevant for performance on high latency backends such as cloud stores.\n\t *
\n\t *\n\t * @param pathName base group path\n\t * @param filter filter for datasets to be included\n\t * @return list of groups\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t */\n\tdefault String[] deepListDatasets(\n\t\t\tfinal String pathName,\n\t\t\tfinal Predicate filter) throws N5Exception {\n\n\t\tfinal String groupSeparator = getGroupSeparator();\n\t\tfinal String normalPathName = pathName\n\t\t\t\t.replaceAll(\n\t\t\t\t\t\t\"(^\" + groupSeparator + \"*)|(\" + groupSeparator + \"*$)\",\n\t\t\t\t\t\t\"\");\n\n\t\tfinal List absolutePaths = deepList(this, normalPathName, true, filter);\n\t\treturn absolutePaths\n\t\t\t\t.stream()\n\t\t\t\t.map(a -> a.replaceFirst(normalPathName + \"(\" + groupSeparator + \"?)\", \"\"))\n\t\t\t\t.filter(a -> !a.isEmpty())\n\t\t\t\t.toArray(String[]::new);\n\t}\n\n\t/**\n\t * Recursively list all including datasets in the given path.\n\t *\n\t * \n\t * This method delivers the same results as\n\t *
\n\t *\n\t * \n\t * {@code\n\t * n5.deepList(prefix, a -> {\n\t * \ttry {\n\t * \t\treturn n5.datasetExists(a);\n\t * \t} catch (final N5Exception e) {\n\t * \t\treturn false;\n\t * \t}\n\t * });\n\t * }\n\t * \n\t * \n\t * but will execute {@link #datasetExists(String)} only once per node. This\n\t * can\n\t * be relevant for performance on high latency backends such as cloud\n\t * stores.\n\t *
\n\t *\n\t * @param pathName\n\t * base group path\n\t * @return list of groups\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t */\n\tdefault String[] deepListDatasets(final String pathName) throws N5Exception {\n\n\t\treturn deepListDatasets(pathName, a -> true);\n\t}\n\n\t/**\n\t * Helper method to recursively list all groups and datasets. This method is not part of the\n\t * public API and is accessible only because Java 8 does not support private\n\t * interface methods yet.\n\t * \n\t * TODO make private when committing to Java versions newer than 8\n\t *\n\t * @param n5 the n5 reader\n\t * @param pathName the base group path\n\t * @param datasetsOnly true if only dataset paths should be returned\n\t * @param filter a dataset filter\n\t * @return the list of all children\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t */\n\tstatic ArrayList deepList(\n\t\t\tfinal N5Reader n5,\n\t\t\tfinal String pathName,\n\t\t\tfinal boolean datasetsOnly,\n\t\t\tfinal Predicate filter) throws N5Exception {\n\n\t\tfinal ArrayList children = new ArrayList<>();\n\t\tfinal boolean isDataset = n5.datasetExists(pathName);\n\n\t\tfinal boolean passDatasetTest = datasetsOnly && !isDataset;\n\t\tif (!passDatasetTest && filter.test(pathName))\n\t\t\tchildren.add(pathName);\n\n\t\tif (!isDataset) {\n\t\t\tfinal String groupSeparator = n5.getGroupSeparator();\n\t\t\tfinal String[] baseChildren = n5.list(pathName);\n\t\t\tfor (final String child : baseChildren)\n\t\t\t\tchildren.addAll(deepList(n5, pathName + groupSeparator + child, datasetsOnly, filter));\n\t\t}\n\n\t\treturn children;\n\t}\n\n\t/**\n\t * Recursively list all groups (including datasets) in the given group, in\n\t * parallel, using the given {@link ExecutorService}. Only paths that\n\t * satisfy\n\t * the provided filter will be included, but the children of paths that were\n\t * excluded may be included (filter does not apply to the subtree).\n\t *\n\t * @param pathName\n\t * base group path\n\t * @param filter\n\t * filter for children to be included\n\t * @param executor\n\t * executor service\n\t * @return list of datasets\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t * @throws ExecutionException\n\t * the execution exception\n\t * @throws InterruptedException\n\t * the interrupted exception\n\t */\n\tdefault String[] deepList(\n\t\t\tfinal String pathName,\n\t\t\tfinal Predicate filter,\n\t\t\tfinal ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {\n\n\t\tfinal String groupSeparator = getGroupSeparator();\n\t\tfinal String normalPathName = pathName.replaceAll(\"(^\" + groupSeparator + \"*)|(\" + groupSeparator + \"*$)\", \"\");\n\t\tfinal ArrayList results = new ArrayList();\n\t\tfinal LinkedBlockingQueue> datasetFutures = new LinkedBlockingQueue<>();\n\t\tdeepListHelper(this, normalPathName, false, filter, executor, datasetFutures);\n\n\t\twhile (!datasetFutures.isEmpty()) {\n\t\t\tfinal String result = datasetFutures.poll().get();\n\t\t\tif (result != null && !result.equals(normalPathName))\n\t\t\t\tresults.add(result.substring(normalPathName.length() + groupSeparator.length()));\n\t\t}\n\n\t\treturn results.toArray(new String[0]);\n\t}\n\n\t/**\n\t * Recursively list all groups (including datasets) in the given group, in\n\t * parallel, using the given {@link ExecutorService}.\n\t *\n\t * @param pathName\n\t * base group path\n\t * @param executor\n\t * executor service\n\t * @return list of groups\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t * @throws ExecutionException\n\t * the execution exception\n\t * @throws InterruptedException\n\t * this exception is thrown if execution is interrupted\n\t */\n\tdefault String[] deepList(\n\t\t\tfinal String pathName,\n\t\t\tfinal ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {\n\n\t\treturn deepList(pathName, a -> true, executor);\n\t}\n\n\t/**\n\t * Recursively list all datasets in the given group, in parallel, using the\n\t * given {@link ExecutorService}. Only paths that satisfy the provided\n\t * filter\n\t * will be included, but the children of paths that were excluded may be\n\t * included (filter does not apply to the subtree).\n\t *\n\t * \n\t * This method delivers the same results as\n\t *
\n\t *\n\t * \n\t * {@code\n\t * n5.deepList(prefix, a -> {\n\t * \ttry {\n\t * \t\treturn n5.datasetExists(a) && filter.test(a);\n\t * \t} catch (final N5Exception e) {\n\t * \t\treturn false;\n\t * \t}\n\t * }, exec);\n\t * }\n\t * \n\t * \n\t * but will execute {@link #datasetExists(String)} only once per node. This\n\t * can\n\t * be relevant for performance on high latency backends such as cloud\n\t * stores.\n\t *
\n\t *\n\t * @param pathName\n\t * base group path\n\t * @param filter\n\t * filter for datasets to be included\n\t * @param executor\n\t * executor service\n\t * @return list of datasets\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t * @throws ExecutionException\n\t * the execution exception\n\t * @throws InterruptedException\n\t * this exception is thrown if execution is interrupted\n\t */\n\tdefault String[] deepListDatasets(\n\t\t\tfinal String pathName,\n\t\t\tfinal Predicate filter,\n\t\t\tfinal ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {\n\n\t\tfinal String groupSeparator = getGroupSeparator();\n\t\tfinal String normalPathName = pathName.replaceAll(\"(^\" + groupSeparator + \"*)|(\" + groupSeparator + \"*$)\", \"\");\n\t\tfinal ArrayList results = new ArrayList();\n\t\tfinal LinkedBlockingQueue> datasetFutures = new LinkedBlockingQueue<>();\n\t\tdeepListHelper(this, normalPathName, true, filter, executor, datasetFutures);\n\n\t\tdatasetFutures.poll().get(); // skip self\n\t\twhile (!datasetFutures.isEmpty()) {\n\t\t\tfinal String result = datasetFutures.poll().get();\n\t\t\tif (result != null)\n\t\t\t\tresults.add(result.substring(normalPathName.length() + groupSeparator.length()));\n\t\t}\n\n\t\treturn results.toArray(new String[0]);\n\t}\n\n\t/**\n\t * Recursively list all datasets in the given group, in parallel, using the\n\t * given {@link ExecutorService}.\n\t *\n\t * \n\t * This method delivers the same results as\n\t *
\n\t *\n\t * \n\t * {@code\n\t * n5.deepList(prefix, a -> {\n\t * \ttry {\n\t * \t\treturn n5.datasetExists(a);\n\t * \t} catch (final N5Exception e) {\n\t * \t\treturn false;\n\t * \t}\n\t * }, exec);\n\t * }\n\t * \n\t * \n\t * but will execute {@link #datasetExists(String)} only once per node. This\n\t * can\n\t * be relevant for performance on high latency backends such as cloud\n\t * stores.\n\t *
\n\t *\n\t * @param pathName\n\t * base group path\n\t * @param executor\n\t * executor service\n\t * @return list of groups\n\t * @throws N5Exception\n\t * \t\t\t\tan exception is thrown if pathName is not a valid group\n\t * @throws ExecutionException\n\t * the execution exception\n\t * @throws InterruptedException\n\t * this exception is thrown if execution is interrupted\n\t */\n\tdefault String[] deepListDatasets(\n\t\t\tfinal String pathName,\n\t\t\tfinal ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {\n\n\t\treturn deepListDatasets(pathName, a -> true, executor);\n\t}\n\n\t/**\n\t * Helper method for parallel deep listing. This method is not part of the\n\t * public API and is accessible only because Java 8 does not support private\n\t * interface methods yet.\n\t *\n\t * TODO make private when committing to Java versions newer than 8\n\t *\n\t * @param n5\n\t * the n5 reader\n\t * @param path\n\t * the base path\n\t * @param datasetsOnly\n\t * true if only dataset paths should be returned\n\t * @param filter\n\t * filter for datasets to be included\n\t * @param executor\n\t * the executor service\n\t * @param datasetFutures\n\t * result futures\n\t */\n\tstatic void deepListHelper(\n\t\t\tfinal N5Reader n5,\n\t\t\tfinal String path,\n\t\t\tfinal boolean datasetsOnly,\n\t\t\tfinal Predicate filter,\n\t\t\tfinal ExecutorService executor,\n\t\t\tfinal LinkedBlockingQueue> datasetFutures) {\n\n\t\tfinal String groupSeparator = n5.getGroupSeparator();\n\n\t\tdatasetFutures.add(executor.submit(() -> {\n\n\t\t\tboolean isDataset = false;\n\t\t\ttry {\n\t\t\t\tisDataset = n5.datasetExists(path);\n\t\t\t} catch (final N5Exception e) {}\n\n\t\t\tif (!isDataset) {\n\t\t\t\tString[] children = null;\n\t\t\t\ttry {\n\t\t\t\t\tchildren = n5.list(path);\n\t\t\t\t\tfor (final String child : children) {\n\t\t\t\t\t\tfinal String fullChildPath = path + groupSeparator + child;\n\t\t\t\t\t\tdeepListHelper(n5, fullChildPath, datasetsOnly, filter, executor, datasetFutures);\n\t\t\t\t\t}\n\t\t\t\t} catch (final N5Exception e) {}\n\t\t\t}\n\t\t\tfinal boolean passDatasetTest = datasetsOnly && !isDataset;\n\t\t\treturn !passDatasetTest && filter.test(path) ? path : null;\n\t\t}));\n\t}\n\n\t/**\n\t * List all attributes and their class of a group.\n\t *\n\t * @param pathName\n\t * group path\n\t * @return a map of attribute keys to their inferred class\n\t * @throws N5Exception if an error occurred during listing\n\t */\n\tMap> listAttributes(String pathName) throws N5Exception;\n\n\t/**\n\t * Returns the symbol that is used to separate nodes in a group path.\n\t *\n\t * @return the group separator\n\t */\n\tdefault String getGroupSeparator() {\n\n\t\treturn \"/\";\n\t}\n\n\t/**\n\t * Creates a group path by concatenating all nodes with the node separator\n\t * defined by {@link #getGroupSeparator()}. The string will not have a\n\t * leading or trailing node separator symbol.\n\t *\n\t * @param nodes a collection of child node names\n\t * @return the full group path\n\t */\n\tdefault String groupPath(final String... nodes) {\n\n\t\tif (nodes == null || nodes.length == 0)\n\t\t\treturn \"\";\n\n\t\tfinal String groupSeparator = getGroupSeparator();\n\t\tfinal StringBuilder builder = new StringBuilder(nodes[0]);\n\n\t\tfor (int i = 1; i < nodes.length; ++i) {\n\n\t\t\tbuilder.append(groupSeparator);\n\t\t\tbuilder.append(nodes[i]);\n\t\t}\n\n\t\treturn builder.toString();\n\t}\n\n\t/**\n\t * Default implementation of {@link AutoCloseable#close()} for all\n\t * implementations that do not hold any closable resources.\n\t */\n\t@Override\n\tdefault void close() {}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5URI.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.nio.ByteBuffer;\nimport java.nio.CharBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.CharsetDecoder;\nimport java.nio.charset.CoderResult;\nimport java.nio.charset.CodingErrorAction;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.function.Consumer;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n * A {@link URI} for N5 containers, groups, datasets, and attributes.\n * \n * Container paths are stored in the URI path. Group / dataset paths are stored in the URI query,\n * and attribute paths are stored in the URI fragment.\n */\npublic class N5URI {\n\n\tprivate static final Charset UTF8 = StandardCharsets.UTF_8;\n\tpublic static final Pattern ARRAY_INDEX = Pattern.compile(\"\\\\[([0-9]+)]\");\n\tfinal URI uri;\n\tprivate final String scheme;\n\tprivate final String container;\n\tprivate final String group;\n\tprivate final String attribute;\n\n\tpublic N5URI(final String uri) throws URISyntaxException {\n\n\t\tthis(encodeAsUri(uri));\n\t}\n\n\tpublic N5URI(final URI uri) {\n\n\t\tthis.uri = uri;\n\t\tscheme = uri.getScheme() == null ? null : uri.getScheme();\n\t\tfinal String schemeSpecificPartWithoutQuery = getSchemeSpecificPartWithoutQuery();\n\t\tif (uri.getScheme() == null) {\n\t\t\tcontainer = schemeSpecificPartWithoutQuery.replaceFirst(\"//\", \"\");\n\t\t} else {\n\t\t\tcontainer = uri.getScheme() + \":\" + schemeSpecificPartWithoutQuery;\n\t\t}\n\t\tgroup = uri.getQuery();\n\t\tattribute = decodeFragment(uri.getRawFragment());\n\t}\n\n\t/**\n\t * @return the container path\n\t */\n\tpublic String getContainerPath() {\n\n\t\treturn container;\n\t}\n\n\tpublic URI getURI() {\n\t\treturn uri;\n\t}\n\n\t/**\n\t * @return the group path, or root (\"/\") if none was provided\n\t */\n\tpublic String getGroupPath() {\n\n\t\treturn group != null ? group : \"/\";\n\t}\n\n\t/**\n\t * @return the normalized group path\n\t */\n\tpublic String normalizeGroupPath() {\n\n\t\treturn normalizeGroupPath(getGroupPath());\n\t}\n\n\t/**\n\t * @return the attribute path, or root (\"/\") if none was provided\n\t */\n\tpublic String getAttributePath() {\n\n\t\treturn attribute != null ? attribute : \"/\";\n\t}\n\n\t/**\n\t * @return the normalized attribute path\n\t */\n\tpublic String normalizeAttributePath() {\n\n\t\treturn normalizeAttributePath(getAttributePath());\n\t}\n\n\t/**\n\t * Parse this {@link N5URI} as a {@link LinkedAttributePathToken}.\n\t *\n\t * @see N5URI#getAttributePathTokens(String)\n\t * @return the linked attribute path token\n\t */\n\tpublic LinkedAttributePathToken getAttributePathTokens() {\n\n\t\treturn getAttributePathTokens(normalizeAttributePath());\n\t}\n\n\t/**\n\t * Parses the {@link String normalizedAttributePath} to a list of\n\t * {@link LinkedAttributePathToken}.\n\t * This is useful for traversing or constructing a json representation of\n\t * the provided {@link String normalizedAttributePath}.\n\t * Note that {@link String normalizedAttributePath} should be normalized\n\t * prior to generating this list\n\t *\n\t * @param normalizedAttributePath\n\t * to parse into {@link LinkedAttributePathToken}s\n\t * @return the head of the {@link LinkedAttributePathToken}s\n\t */\n\tpublic static LinkedAttributePathToken getAttributePathTokens(final String normalizedAttributePath) {\n\n\t\tfinal String[] attributePathParts = normalizedAttributePath.replaceAll(\"^/\", \"\").split(\"(?> firstTokenRef = new AtomicReference<>();\n\t\tfinal AtomicReference> currentTokenRef = new AtomicReference<>();\n\t\tfinal Consumer> updateCurrentToken = (newToken) -> {\n\t\t\tif (firstTokenRef.get() == null) {\n\t\t\t\tfirstTokenRef.set(newToken);\n\t\t\t\tcurrentTokenRef.set(firstTokenRef.get());\n\t\t\t} else {\n\t\t\t\tfinal LinkedAttributePathToken currentToken = currentTokenRef.get();\n\t\t\t\tcurrentToken.childToken = newToken;\n\t\t\t\tcurrentTokenRef.set(newToken);\n\t\t\t}\n\t\t};\n\n\t\tfor (final String pathPart : attributePathParts) {\n\t\t\tfinal Matcher matcher = ARRAY_INDEX.matcher(pathPart);\n\t\t\tfinal LinkedAttributePathToken newToken;\n\t\t\tif (matcher.matches()) {\n\t\t\t\tfinal int index = Integer.parseInt(matcher.group().replace(\"[\", \"\").replace(\"]\", \"\"));\n\t\t\t\tnewToken = new LinkedAttributePathToken.ArrayAttributeToken(index);\n\t\t\t} else {\n\t\t\t\tfinal String pathPartUnEscaped = pathPart.replaceAll(\"\\\\\\\\/\", \"/\").replaceAll(\"\\\\\\\\\\\\[\", \"[\");\n\t\t\t\tnewToken = new LinkedAttributePathToken.ObjectAttributeToken(pathPartUnEscaped);\n\t\t\t}\n\t\t\tupdateCurrentToken.accept(newToken);\n\t\t}\n\n\t\treturn firstTokenRef.get();\n\t}\n\n\tprivate String getSchemePart() {\n\n\t\treturn scheme == null ? \"\" : scheme + \"://\";\n\t}\n\n\tprivate String getContainerPart() {\n\n\t\treturn container;\n\t}\n\n\tprivate String getGroupPart() {\n\n\t\treturn group == null ? \"\" : \"?\" + group;\n\t}\n\n\tprivate String getAttributePart() {\n\n\t\treturn attribute == null ? \"\" : \"#\" + attribute;\n\t}\n\n\t@Override\n\tpublic String toString() {\n\n\t\treturn getContainerPart() + getGroupPart() + getAttributePart();\n\t}\n\n\tprivate String getSchemeSpecificPartWithoutQuery() {\n\n\t\t/* Why not substring \"?\"? */\n\t\treturn uri.getSchemeSpecificPart().replace(\"?\" + uri.getQuery(), \"\");\n\t}\n\n\t/**\n\t * N5URI is always considered absolute if a scheme is provided.\n\t * If no scheme is provided, the N5URI is absolute if it starts with either\n\t * \"/\" or \"[A-Z]:\"\n\t *\n\t * @return if the path for this N5URI is absolute\n\t */\n\tpublic boolean isAbsolute() {\n\n\t\tif (scheme != null)\n\t\t\treturn true;\n\t\tfinal String path = uri.getPath();\n\t\tif (!path.isEmpty()) {\n\t\t\tfinal char char0 = path.charAt(0);\n\t\t\treturn char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z');\n\t\t}\n\t\treturn false;\n\t}\n\n\t/**\n\t * Generate a new N5URI which is the result of resolving {@link N5URI\n\t * relativeN5Url} to this {@link N5URI}.\n\t * If relativeN5Url is not relative to this N5URI, then the resulting N5URI\n\t * is equivalent to relativeN5Url.\n\t *\n\t * @param relativeN5Url\n\t * N5URI to resolve against ourselves\n\t * @return the result of the resolution.\n\t * @throws URISyntaxException\n\t * if the uri is malformed\n\t */\n\tpublic N5URI resolve(final N5URI relativeN5Url) throws URISyntaxException {\n\n\t\tfinal URI thisUri = uri;\n\t\tfinal URI relativeUri = relativeN5Url.uri;\n\n\t\tfinal StringBuilder newUri = new StringBuilder();\n\n\t\tif (relativeUri.getScheme() != null) {\n\t\t\treturn relativeN5Url;\n\t\t}\n\t\tfinal String thisScheme = thisUri.getScheme();\n\t\tif (thisScheme != null) {\n\t\t\tnewUri.append(thisScheme).append(\":\");\n\t\t}\n\n\t\tif (relativeUri.getAuthority() != null) {\n\t\t\tnewUri\n\t\t\t\t\t.append(relativeUri.getAuthority())\n\t\t\t\t\t.append(relativeUri.getPath())\n\t\t\t\t\t.append(relativeN5Url.getGroupPart())\n\t\t\t\t\t.append(relativeN5Url.getAttributePart());\n\t\t\treturn new N5URI(newUri.toString());\n\t\t}\n\t\tfinal String thisAuthority = thisUri.getAuthority();\n\t\tif (thisAuthority != null) {\n\t\t\tnewUri.append(\"//\").append(thisAuthority);\n\t\t}\n\n\t\tfinal String path = relativeUri.getPath();\n\t\tif (!path.isEmpty()) {\n\t\t\tif (!relativeN5Url.isAbsolute()) {\n\t\t\t\tnewUri.append(thisUri.getPath()).append('/');\n\t\t\t}\n\t\t\tnewUri\n\t\t\t\t\t.append(path)\n\t\t\t\t\t.append(relativeN5Url.getGroupPart())\n\t\t\t\t\t.append(relativeN5Url.getAttributePart());\n\t\t\treturn new N5URI(newUri.toString());\n\t\t}\n\t\tnewUri.append(thisUri.getPath());\n\n\t\tfinal String query = relativeUri.getQuery();\n\t\tif (query != null) {\n\t\t\tif (query.charAt(0) != '/' && thisUri.getQuery() != null) {\n\t\t\t\tnewUri.append(this.getGroupPart()).append('/');\n\t\t\t\tnewUri.append(relativeUri.getQuery());\n\t\t\t} else {\n\t\t\t\tnewUri.append(relativeN5Url.getGroupPart());\n\t\t\t}\n\t\t\tnewUri.append(relativeN5Url.getAttributePart());\n\t\t\treturn new N5URI(newUri.toString());\n\t\t}\n\t\tnewUri.append(this.getGroupPart());\n\n\t\tfinal String fragment = relativeUri.getFragment();\n\t\tif (fragment != null) {\n\t\t\tif (fragment.charAt(0) != '/' && thisUri.getFragment() != null) {\n\t\t\t\tnewUri.append(this.getAttributePart()).append('/');\n\t\t\t} else {\n\t\t\t\tnewUri.append(relativeN5Url.getAttributePart());\n\t\t\t}\n\n\t\t\treturn new N5URI(newUri.toString());\n\t\t}\n\t\tnewUri.append(this.getAttributePart());\n\n\t\treturn new N5URI(newUri.toString());\n\t}\n\n\t/**\n\t * Generate a new N5URI which is the result of resolving {@link URI\n\t * relativeUri} to this {@link N5URI}.\n\t * If relativeUri is not relative to this N5URI, then the resulting N5URI is\n\t * equivalent to relativeUri.\n\t *\n\t * @param relativeUri\n\t * URI to resolve against ourselves\n\t * @return the result of the resolution.\n\t * @throws URISyntaxException\n\t * if the uri is malformed\n\t */\n\tpublic N5URI resolve(final URI relativeUri) throws URISyntaxException {\n\n\t\treturn resolve(new N5URI(relativeUri));\n\t}\n\n\t/**\n\t * Generate a new N5URI which is the result of resolving {@link String\n\t * relativeString} to this {@link N5URI}\n\t * If relativeString is not relative to this N5URI, then the resulting N5URI\n\t * is equivalent to relativeString.\n\t *\n\t * @param relativeString\n\t * String to resolve against ourselves\n\t * @return the result of the resolution.\n\t * @throws URISyntaxException\n\t * if the uri is malformed\n\t */\n\tpublic N5URI resolve(final String relativeString) throws URISyntaxException {\n\n\t\treturn resolve(new N5URI(relativeString));\n\t}\n\n\t/**\n\t * Normalize a POSIX path, resulting in removal of redundant \"/\", \"./\", and\n\t * resolution of relative \"../\".\n\t * \n\t * NOTE: currently a private helper method only used by {@link N5URI#normalizeGroupPath(String)}.\n\t * \tIt's safe to do in that case since relative group paths should always be POSIX compliant.\n\t * \tA new helper method to understand other path types (e.g. Windows) may be necessary eventually.\n\t *\n\t * @param path\n\t * to normalize\n\t * @return the normalized path\n\t */\n\tprivate static String normalizePath(String path) {\n\n\t\tpath = path == null ? \"\" : path;\n\t\tfinal char[] pathChars = path.toCharArray();\n\n\t\tfinal List tokens = new ArrayList<>();\n\t\tfinal StringBuilder curToken = new StringBuilder();\n\t\tboolean escape = false;\n\t\tfor (final char character : pathChars) {\n\t\t\t/* Skip if we last saw escape */\n\t\t\tif (escape) {\n\t\t\t\tescape = false;\n\t\t\t\tcurToken.append(character);\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\t/* Check if we are escape character */\n\t\t\tif (character == '\\\\') {\n\t\t\t\tescape = true;\n\t\t\t} else if (character == '/') {\n\t\t\t\tif (tokens.isEmpty() && curToken.length() == 0) {\n\t\t\t\t\t/* If we are root, and the first token, then add the '/' */\n\t\t\t\t\tcurToken.append(character);\n\t\t\t\t}\n\n\t\t\t\t/*\n\t\t\t\t * The current token is complete, add it to the list, if it\n\t\t\t\t * isn't empty\n\t\t\t\t */\n\t\t\t\tfinal String newToken = curToken.toString();\n\t\t\t\tif (!newToken.isEmpty()) {\n\t\t\t\t\t/*\n\t\t\t\t\t * If our token is '..' then remove the last token instead\n\t\t\t\t\t * of adding a new one\n\t\t\t\t\t */\n\t\t\t\t\tif (newToken.equals(\"..\")) {\n\t\t\t\t\t\ttokens.remove(tokens.size() - 1);\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttokens.add(newToken);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t/* reset for the next token */\n\t\t\t\tcurToken.setLength(0);\n\t\t\t} else {\n\t\t\t\tcurToken.append(character);\n\t\t\t}\n\t\t}\n\t\tfinal String lastToken = curToken.toString();\n\t\tif (!lastToken.isEmpty()) {\n\t\t\tif (lastToken.equals(\"..\")) {\n\t\t\t\ttokens.remove(tokens.size() - 1);\n\t\t\t} else {\n\t\t\t\ttokens.add(lastToken);\n\t\t\t}\n\t\t}\n\t\tif (tokens.isEmpty())\n\t\t\treturn \"\";\n\t\tString root = \"\";\n\t\tif (tokens.get(0).equals(\"/\")) {\n\t\t\ttokens.remove(0);\n\t\t\troot = \"/\";\n\t\t}\n\t\treturn root + tokens\n\t\t\t\t.stream()\n\t\t\t\t.filter(it -> !it.equals(\".\"))\n\t\t\t\t.filter(it -> !it.isEmpty())\n\t\t\t\t.reduce((l, r) -> l + \"/\" + r)\n\t\t\t\t.orElse(\"\");\n\t}\n\n\t/**\n\t * Normalize a group path relative to a container's root, resulting in\n\t * removal of redundant \"/\", \"./\", resolution of relative \"../\",\n\t * and removal of leading slashes.\n\t *\n\t * @param path\n\t * to normalize\n\t * @return the normalized path\n\t */\n\tpublic static String normalizeGroupPath(final String path) {\n\n\t\t/*\n\t\t * Alternatively, could do something like the below in every\n\t\t * KeyValueReader implementation\n\t\t *\n\t\t * return keyValueAccess.relativize( N5URI.normalizeGroupPath(path),\n\t\t * basePath);\n\t\t *\n\t\t * has to be in the implementations, since KeyValueAccess doesn't have a\n\t\t * basePath.\n\t\t */\n\t\treturn normalizePath(path.startsWith(\"/\") || path.startsWith(\"\\\\\") ? path.substring(1) : path);\n\t}\n\n\tprivate enum N5UriPattern {\n\n\t\t/**\n\t\t * matches any `/` or `[N]` where `N` is non-negative\n\t\t */\n\t\tMULTI_PART_ATTRIBUTE(Pattern.compile(\".*((?\\\\[[0-9]+])\")),\n\t\t/**\n\t\t * matches `A[N]` where A is some non-empty preceding path\n\t\t */\n\t\tATTRIBUTE_ARRAY_EXCEPT_START(Pattern.compile(\"((?\\\\[[0-9]+]))\")),\n\t\t/**\n\t\t * The following Pattern has 4 possible matches.\n\t\t * It is intended to be used to remove matching portions iteratively until no further matches are found:\n\t\t * \n\t\t * The first 3 matches can remove redundant separators of the form:\n\t\t *
\n\t\t * - (?<=/)/+ : `a///b` -> `a/b`
\n\t\t * - (?<=(/|^))(\\./)+ : `a/./b` -> `a/b`
\n\t\t * - ((/|(?<=/))\\.)$ : `a/b/` -> `a/b`
\n\t\t *
\n\t\t * The next match avoids removing `/` when it is NOT redundant (e.g. only character, or escaped):\n\t\t * \n\t\t * - (? `/ , `/a/b/\\\\/` -> `/a/b/\\\\/`
\n\t\t *
\n\t\t * The last match resolves relative paths:\n\t\t * \n\t\t * - 5. ((?<=^/)|^|(?<=(/|^))[^/]+(?\n\t\t *
\n\t\t *\t\t\t- `a/../b` -> `b`
\n\t\t *\t\t\t- `/a/../b` -> `/b`
\n\t\t *\t\t\t- `../a/../b` -> `b`
\n\t\t *\t\t\t- `/../a/../b` -> `/b`
\n\t\t *\t\t\t- `/../a/../../b` -> `/b`
\n\t\t *
\n\t\t *
\n\t\t *\n\t\t */\n\t\tRELATIVE_ATTRIBUTE_PARTS(Pattern.compile( \"((?<=/)/+|(?<=(/|^))(\\\\./)+|((/|(?<=/))\\\\.)$|(?\n\t * Attribute paths have a few special characters:\n\t *
\n\t * \n\t * When normalizing:\n\t *
\n\t * - \"/\" are added before and after any indexing brackets [N]
\n\t * - any redundant \"/\" are removed
\n\t * - any relative \"..\" and \".\" are resolved
\n\t *
\n\t * \n\t * Examples of valid attribute paths, and their normalizations\n\t *
\n\t * - /a/b/c becomes /a/b/c
\n\t * - /a/b/c/ becomes /a/b/c
\n\t * - ///a///b///c becomes /a/b/c
\n\t * - /a/././b/c becomes /a/b/c
\n\t * - /a/b[1]c becomes /a/b/[1]/c
\n\t * - /a/b/[1]c becomes /a/b/[1]/c
\n\t * - /a/b[1]/c becomes /a/b/[1]/c
\n\t * - /a/b[1]/c/.. becomes /a/b/[1]
\n\t *
\n\t *\n\t * @param attributePath\n\t * to normalize\n\t * @return the normalized attribute path\n\t */\n\tpublic static String normalizeAttributePath(final String attributePath) {\n\n\t\t/*\n\t\t * Short circuit if there are no non-escaped `/` or array indices (e.g.\n\t\t * [N] where N is a non-negative integer)\n\t\t */\n\t\tif (!N5UriPattern.MULTI_PART_ATTRIBUTE.matches(attributePath))\n\t\t\treturn attributePath;\n\n\t\t/* Add separator after arrays at the beginning `[10]b` -> `[10]/b` */\n\t\tfinal String attrPathPlusFirstIndexSeparator = N5UriPattern.appendSlashAfterArrayStart(attributePath);\n\n\t\t/*\n\t\t * Add separator before and after arrays not at the beginning `a[10]b`\n\t\t * -> `a/[10]/b`\n\t\t */\n\t\tfinal String attrPathPlusIndexSeparators = N5UriPattern.addSlashAroundArrayExceptStart(attrPathPlusFirstIndexSeparator);\n\n\t\t/* remove relative path parts like `./`, `../`, `a//b, `a/b/` */\n\t\treturn N5UriPattern.removeRelativePathParts(attrPathPlusIndexSeparators);\n\t}\n\n\t/**\n\t * If uri is a valid URI, just return it as a URI. Else, encode if possible.\n\t *\n\t * @param uri as String to get as URI\n\t * @return URI from input. Encoded if necessary\n\t */\n\tpublic static URI getAsUri(final String uri) throws N5Exception {\n\n\t\ttry {\n\t\t\treturn URI.create(uri);\n\t\t} catch (Exception ignore) {\n\t\t\ttry {\n\t\t\t\treturn N5URI.encodeAsUri(uri);\n\t\t\t} catch (URISyntaxException e) {\n\t\t\t\tthrow new N5Exception(\"Could not encode as URI: \" + uri, e);\n\t\t\t}\n\t\t}\n\t}\n\n\tpublic static URI encodeAsUriPath(final String path) {\n\t\ttry {\n\t\t\treturn new URI(null, null, path, null);\n\t\t} catch (Exception e) {\n\t\t\tthrow new IllegalArgumentException(\"Could not encode as URI path component: \" + path, e);\n\t\t}\n\t}\n\n\t/**\n\t * Encode the inpurt {@link String uri} so that illegal characters are\n\t * properly escaped prior to generating the resulting {@link URI}.\n\t *\n\t * @param uri\n\t * to encode\n\t * @return the {@link URI} created from encoding the {@link String uri}\n\t * @throws URISyntaxException\n\t * \t\tif the provided String is not a valid URI\n\t */\n\tpublic static URI encodeAsUri(final String uri) throws URISyntaxException {\n\n\t\tif (uri.trim().length() == 0) {\n\t\t\t//TODO Caleb: ???\n\t\t\treturn new URI(uri);\n\t\t}\n\t\t/*\n\t\t * find last # symbol to split fragment on. If we don't remove it first,\n\t\t * then it will encode it, and not parse it separately\n\t\t * after we remove the temporary _N5 scheme\n\t\t */\n\t\tfinal int fragmentIdx = uri.lastIndexOf('#');\n\t\tfinal String uriWithoutFragment;\n\t\tfinal String fragment;\n\t\tif (fragmentIdx >= 0) {\n\t\t\turiWithoutFragment = uri.substring(0, fragmentIdx);\n\t\t\tfragment = uri.substring(fragmentIdx + 1);\n\t\t} else {\n\t\t\turiWithoutFragment = uri;\n\t\t\tfragment = null;\n\t\t}\n\t\t/* Edge case to handle when uriWithoutFragment is empty */\n\t\tfinal URI _n5Uri;\n\t\tif (uriWithoutFragment.length() == 0 && fragment != null && fragment.length() > 0) {\n\t\t\t_n5Uri = new URI(\"N5Internal\", \"//STAND_IN\", fragment);\n\t\t} else {\n\t\t\t_n5Uri = new URI(\"N5Internal\", uriWithoutFragment, fragment);\n\t\t}\n\n\t\tfinal URI n5Uri;\n\t\tif (fragment == null) {\n\t\t\tn5Uri = new URI(_n5Uri.getRawSchemeSpecificPart());\n\t\t} else {\n\t\t\tif (Objects.equals(_n5Uri.getPath(), \"\") && Objects.equals(_n5Uri.getAuthority(), \"STAND_IN\")) {\n\t\t\t\tn5Uri = new URI(\"#\" + _n5Uri.getRawFragment());\n\t\t\t} else {\n\t\t\t\tn5Uri = new URI(_n5Uri.getRawSchemeSpecificPart() + \"#\" + _n5Uri.getRawFragment());\n\t\t\t}\n\t\t}\n\t\treturn n5Uri;\n\t}\n\n\t/**\n\t * Generate an {@link N5URI} from a container, group, and attribute\n\t *\n\t * @param container\n\t * of the N5Url\n\t * @param group\n\t * of the N5Url\n\t * @param attribute\n\t * of the N5Url\n\t * @return the {@link N5URI}\n\t * @throws URISyntaxException\n\t * if the uri is malformed\n\t */\n\tpublic static N5URI from(\n\t\t\tfinal String container,\n\t\t\tfinal String group,\n\t\t\tfinal String attribute) throws URISyntaxException {\n\n\t\tfinal String containerPart = container != null ? container : \"\";\n\t\tfinal String groupPart = group != null ? \"?\" + group : \"\";\n\t\tfinal String attributePart = attribute != null ? \"#\" + attribute : \"\";\n\t\treturn new N5URI(containerPart + groupPart + attributePart);\n\t}\n\n\t/**\n\t * Intentionally copied from {@link URI} for internal use\n\t *\n\t * @see URI#decode(char)\n\t */\n\t@SuppressWarnings(\"JavadocReference\")\n\tprivate static int decode(final char c) {\n\n\t\tif ((c >= '0') && (c <= '9'))\n\t\t\treturn c - '0';\n\t\tif ((c >= 'a') && (c <= 'f'))\n\t\t\treturn c - 'a' + 10;\n\t\tif ((c >= 'A') && (c <= 'F'))\n\t\t\treturn c - 'A' + 10;\n\t\tassert false;\n\t\treturn -1;\n\t}\n\n\t/**\n\t * Intentionally copied from {@link URI} for internal use\n\t *\n\t * @see URI#decode(char, char)\n\t */\n\t@SuppressWarnings(\"JavadocReference\")\n\tprivate static byte decode(final char c1, final char c2) {\n\n\t\treturn (byte)(((decode(c1) & 0xf) << 4)\n\t\t\t\t| ((decode(c2) & 0xf) << 0));\n\t}\n\n\t/**\n\t * Modified from {@link URI#decode(String)} to ignore the listed exception,\n\t * where it doesn't decode escape values inside square braces.\n\t * \n\t * As an example of the original implementation, a backslash inside a square\n\t * brace would be encoded to \"[%5C]\", and\n\t * when calling {@code decode(\"[%5C]\")} it would not decode to \"[\\]\" since\n\t * the encode escape sequence is inside square braces.\n\t *
\n\t * We keep all the decoding logic in this modified version, EXCEPT, that we\n\t * don't check for and ignore encoded sequences inside square braces.\n\t *
\n\t * Thus, {@code decode(\"[%5C]\")} -> \"[\\]\".\n\t *\n\t * @see URI#decode(char, char)\n\t */\n\t@SuppressWarnings(\"JavadocReference\")\n\tprivate static String decodeFragment(final String rawFragment) {\n\n\t\tif (rawFragment == null)\n\t\t\treturn rawFragment;\n\t\tfinal int n = rawFragment.length();\n\t\tif (n == 0)\n\t\t\treturn rawFragment;\n\t\tif (rawFragment.indexOf('%') < 0)\n\t\t\treturn rawFragment;\n\n\t\tfinal StringBuffer sb = new StringBuffer(n);\n\t\tfinal ByteBuffer bb = ByteBuffer.allocate(n);\n\t\tfinal CharBuffer cb = CharBuffer.allocate(n);\n\n\t\tfinal CharsetDecoder dec = UTF8\n\t\t\t\t.newDecoder()\n\t\t\t\t.onMalformedInput(CodingErrorAction.REPLACE)\n\t\t\t\t.onUnmappableCharacter(CodingErrorAction.REPLACE);\n\n\t\t// This is not horribly efficient, but it will do for now\n\t\tchar c = rawFragment.charAt(0);\n\n\t\tfor (int i = 0; i < n;) {\n\t\t\tassert c == rawFragment.charAt(i); // Loop invariant\n\t\t\tif (c != '%') {\n\t\t\t\tsb.append(c);\n\t\t\t\tif (++i >= n)\n\t\t\t\t\tbreak;\n\t\t\t\tc = rawFragment.charAt(i);\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tbb.clear();\n\t\t\tfor (;;) {\n\t\t\t\tassert (n - i >= 2);\n\t\t\t\tbb.put(decode(rawFragment.charAt(++i), rawFragment.charAt(++i)));\n\t\t\t\tif (++i >= n)\n\t\t\t\t\tbreak;\n\t\t\t\tc = rawFragment.charAt(i);\n\t\t\t\tif (c != '%')\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t\tbb.flip();\n\t\t\tcb.clear();\n\t\t\tdec.reset();\n\t\t\tCoderResult cr = dec.decode(bb, cb, true);\n\t\t\tassert cr.isUnderflow();\n\t\t\tcr = dec.flush(cb);\n\t\t\tassert cr.isUnderflow();\n\t\t\tsb.append(cb.flip());\n\t\t}\n\n\t\treturn sb.toString();\n\t}\n\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java",
"content": "package org.janelia.saalfeldlab.n5;\n\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.ObjectOutputStream;\nimport java.io.Serializable;\nimport java.io.UncheckedIOException;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;\n\n/**\n * A simple structured container API for hierarchies of chunked\n * n-dimensional datasets and attributes.\n *\n * @author Stephan Saalfeld\n * @see \"https://github.com/axtimwalde/n5\"\n */\npublic interface N5Writer extends N5Reader {\n\n\t/**\n\t * Sets an attribute.\n\t *\n\t * @param groupPath group path\n\t * @param attributePath the key\n\t * @param attribute the attribute\n\t * @param the attribute type\n\t * @throws N5Exception the exception\n\t */\n\tdefault void setAttribute(\n\t\t\tfinal String groupPath,\n\t\t\tfinal String attributePath,\n\t\t\tfinal T attribute) throws N5Exception {\n\n\t\tsetAttributes(groupPath, Collections.singletonMap(attributePath, attribute));\n\t}\n\n\t/**\n\t * Sets a map of attributes. The passed attributes are inserted into the\n\t * existing attribute tree. New attributes, including their parent\n\t * objects will be added, existing attributes whose paths are not included\n\t * will remain unchanged, those whose paths are included will be overridden.\n\t *\n\t * @param groupPath group path\n\t * @param attributes the attribute map of attribute paths and values\n\t * @throws N5Exception the exception\n\t */\n\tvoid setAttributes(\n\t\t\tString groupPath,\n\t\t\tMap attributes) throws N5Exception;\n\n\t/**\n\t * Remove the attribute from group {@code pathName} with key {@code key}.\n\t *\n\t * @param groupPath group path\n\t * @param attributePath of attribute to remove\n\t * @return true if attribute removed, else false\n\t * @throws N5Exception the exception\n\t */\n\tboolean removeAttribute(String groupPath, String attributePath) throws N5Exception;\n\n\t/**\n\t * Remove the attribute from group {@code pathName} with key {@code key} and\n\t * type {@code T}.\n\t * \n\t * If an attribute at {@code pathName} and {@code key} exists, but is not of\n\t * type {@code T}, it is not removed.\n\t *\n\t * @param groupPath group path\n\t * @param attributePath of attribute to remove\n\t * @param clazz of the attribute to remove\n\t * @param of the attribute\n\t * @return the removed attribute, as {@code T}, or {@code null} if no\n\t * matching attribute\n\t * @throws N5Exception if removing he attribute failed, parsing the attribute failed, or the attribute cannot be interpreted as T\n\t */\n\t T removeAttribute(String groupPath, String attributePath, Class clazz) throws N5Exception;\n\n\t/**\n\t * Remove attributes as provided by {@code attributes}.\n\t * \n\t * If any element of {@code attributes} does not exist, it will be ignored.\n\t * If at least one attribute from {@code attributes} is removed, this will\n\t * return {@code true}.\n\t *\n\t *\n\t * @param groupPath group path\n\t * @param attributePaths to remove\n\t * @return true if any of the listed attributes were removed\n\t * @throws N5Exception the exception\n\t */\n\tdefault boolean removeAttributes(final String groupPath, final List attributePaths) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(groupPath);\n\t\tboolean removed = false;\n\t\tfor (final String attribute : attributePaths) {\n\t\t\tremoved |= removeAttribute(normalPath, N5URI.normalizeAttributePath(attribute));\n\t\t}\n\t\treturn removed;\n\t}\n\n\t/**\n\t * Sets mandatory dataset attributes.\n\t *\n\t * @param datasetPath dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @throws N5Exception the exception\n\t */\n\tdefault void setDatasetAttributes(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes) throws N5Exception {\n\n\t\tsetAttributes(datasetPath, getConvertedDatasetAttributes(datasetAttributes).asMap());\n\t}\n\n\t/**\n\t * Set the SemVer version of this container as specified in the\n\t * {@link N5Reader#VERSION_KEY} attribute of the root group. This default\n\t * implementation writes the version only if the current version is not\n\t * equal {@link N5Reader#VERSION}.\n\t *\n\t * @throws N5Exception the exception\n\t */\n\tdefault void setVersion() throws N5Exception {\n\n\t\tif (!VERSION.equals(getVersion()))\n\t\t\tsetAttribute(\"/\", VERSION_KEY, VERSION.toString());\n\t}\n\n\t/**\n\t * Creates a group (directory)\n\t *\n\t * @param groupPath the path\n\t * @throws N5Exception the exception\n\t */\n\tvoid createGroup(String groupPath) throws N5Exception;\n\n\t/**\n\t * Removes a group or dataset (directory and all contained files).\n\t *\n\t * \n\t * {@link #remove(String) remove(\"\")} or\n\t * {@link #remove(String) remove(\"\")} will delete this N5\n\t * container. Please note that no checks for safety will be performed,\n\t * e.g. {@link #remove(String) remove(\"..\")} will try to\n\t * recursively delete the parent directory of this N5 container which\n\t * only fails because it attempts to delete the parent directory before it\n\t * is empty.\n\t *\n\t * @param groupPath group path\n\t * @return true if removal was successful, false otherwise\n\t * @throws N5Exception the exception\n\t */\n\tboolean remove(String groupPath) throws N5Exception;\n\n\t/**\n\t * Removes the N5 container.\n\t *\n\t * @return true if removal was successful, false otherwise\n\t * @throws N5Exception the exception\n\t */\n\tdefault boolean remove() throws N5Exception {\n\n\t\treturn remove(\"/\");\n\t}\n\n\t/**\n\t * Creates a dataset. This does not create any data but the path and\n\t * mandatory attributes only. The returned DatasetAttributes should be used\n\t * for future read/write operations on this dataset. It may not be the same\n\t * DatasetAttributes object that was provided, depending on the implementation.\n\t *\n\t * @param datasetPath dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @return DatasetAttributes optimal attributes object to be used for read/write operations\n\t * @throws N5Exception\n\t */\n\tdefault DatasetAttributes createDataset(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes) throws N5Exception {\n\n\t\tfinal String normalPath = N5URI.normalizeGroupPath(datasetPath);\n\t\tcreateGroup(normalPath);\n\t\tfinal DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tsetDatasetAttributes(normalPath, convertedDatasetAttributes);\n\t\treturn convertedDatasetAttributes;\n\t}\n\n\t/**\n\t * Creates a dataset. This does not create any data but the path and\n\t * mandatory attributes only. Returns the DatasetAttributes object to be\n\t * used for future read/write operations on this dataset.\n\t *\n\t * @param datasetPath dataset path\n\t * @param dimensions the dataset dimensions\n\t * @param blockSize the block size\n\t * @param dataType the data type\n\t * @param compression the compression\n\t * @return DatasetAttributes optimal attributes object to be used for read/write operations\n\t * @throws N5Exception\n\t */\n\tdefault DatasetAttributes createDataset(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal long[] dimensions,\n\t\t\tfinal int[] blockSize,\n\t\t\tfinal DataType dataType,\n\t\t\tfinal Compression compression) throws N5Exception {\n\n\t\treturn createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));\n\t}\n\n\t/**\n\t * Writes a chunk represented by a {@link DataBlock}.\n\t *\n\t * @param datasetPath dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param chunk the chunk as a DataBlock\n\t * @param the data block data type\n\t * @throws N5Exception the exception\n\t */\n\t void writeChunk(\n\t\t\tString datasetPath,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tDataBlock chunk) throws N5Exception;\n\n\t/**\n\t * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation.\n\t *\n\t * @param datasetPath dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param chunks the chunks\n\t * @param the data block data type\n\t * @throws N5Exception the exception\n\t */\n\tdefault void writeChunks(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal DataBlock... chunks) throws N5Exception {\n\n\t\t// default method is naive\n\t\tDatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes);\n\t\tfor (DataBlock block : chunks) {\n\t\t\twriteChunk(datasetPath, convertedAttributes, block);\n\t\t}\n\t}\n\n\t/**\n\t * Writes a block stored as a {@link DataBlock}.\n\t * \n\t * A block is the highest (coarsest) level of the dataset's {@link NestedGrid},\n\t * that is, a shard if the dataset is sharded, or a chunk otherwise.\n\t *
\n\t * This method's behavior is identical to {@link #writeChunk} for un-sharded datasets.\n\t *\n\t * @param pathName dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param dataBlock the data block\n\t * @param the data block data type\n\t * @throws N5Exception the exception\n\t *\n\t * @see DatasetAttributes#getNestedBlockGrid()\n\t */\n\t void writeBlock(\n\t\t\tString pathName,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tDataBlock dataBlock) throws N5Exception;\n\n\t@FunctionalInterface\n\tinterface DataBlockSupplier {\n\n\t\t/**\n\t\t *\n\t\t * @param gridPos\n\t\t * @param existingDataBlock\n\t\t * \t\texisting data to be merged into the new data block (may be {@code null})\n\t\t *\n\t\t * @return data block at the given gridPos\n\t\t */\n\t\tDataBlock get(long[] gridPos, DataBlock existingDataBlock);\n\t}\n\n\t/**\n\t * @param datasetPath the dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param min min pixel coordinate of region to write\n\t * @param size size in pixels of region to write\n\t * @param chunkSupplier is asked to create chunks within the given region\n\t * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything.\n\t * @throws N5Exception the exception\n\t */\n\t void writeRegion(\n\t\t\tString datasetPath,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong[] min,\n\t\t\tlong[] size,\n\t\t\tDataBlockSupplier chunkSupplier,\n\t\t\tboolean writeFully) throws N5Exception;\n\n\t/**\n\t * @param datasetPath the dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param min min pixel coordinate of region to write\n\t * @param size size in pixels of region to write\n\t * @param chunkSupplier is asked to create chunks within the given region\n\t * @param writeFully if false, merge existing data in shards/chunks that overlap the region boundary. if true, override everything.\n\t * @param exec used to parallelize over blocks (chunks and shards)\n\t * @throws N5Exception the exception\n\t */\n\t void writeRegion(\n\t\t\tString datasetPath,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong[] min,\n\t\t\tlong[] size,\n\t\t\tDataBlockSupplier chunkSupplier,\n\t\t\tboolean writeFully,\n\t\t\tExecutorService exec) throws N5Exception, InterruptedException, ExecutionException;\n\n\t/**\n\t * Deletes the block at {@code gridPosition}.\n\t * \n\t * A block is the highest (coarsest) level of the dataset's {@link NestedGrid},\n\t * that is, a shard if the dataset is sharded, or a chunk otherwise.\n\t * Note that {@code gridPosition} is in units of blocks at the highest\n\t * (coarsest) level, too.\n\t *
\n\t * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets.\n\t *\n\t * @param datasetPath dataset path\n\t * @param gridPosition position of block to be deleted\n\t * @throws N5Exception if the block exists but could not be deleted\n\t *\n\t * @return {@code true} if the block at {@code gridPosition} existed and was deleted.\n\t */\n\tdefault boolean deleteBlock(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\t\tfinal DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath);\n\t\treturn deleteBlock(datasetPath, datasetAttributes, gridPosition);\n\t}\n\n\t/**\n\t * Deletes the block at {@code gridPosition}.\n\t *
\n\t * A block is the highest (coarsest) level of the dataset's {@link NestedGrid},\n\t * that is, a shard if the dataset is sharded, or a chunk otherwise.\n\t * Note that {@code gridPosition} is in units of blocks at the highest\n\t * (coarsest) level, too.\n\t *
\n\t * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets.\n\t *\n\t * @param datasetPath the dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param gridPosition position of block to be deleted\n\t * @throws N5Exception if the block exists but could not be deleted\n\t *\n\t * @return {@code true} if the block at {@code gridPosition} existed and was deleted.\n\t */\n\tboolean deleteBlock(\n\t\t\tString datasetPath,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong... gridPosition) throws N5Exception;\n\n\t/**\n\t * Deletes the chunk at {@code gridPosition}.\n\t *
\n\t * Note that {@code gridPosition} is in units of chunks.\n\t *\n\t * @param datasetPath dataset path\n\t * @param gridPosition position of chunk to be deleted\n\t * @throws N5Exception if the chunk exists but could not be deleted\n\t *\n\t * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted.\n\t */\n\tdefault boolean deleteChunk(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\t\tfinal DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath);\n\t\treturn deleteChunk(datasetPath, datasetAttributes, gridPosition);\n\t}\n\n\t/**\n\t * Deletes the chunk at {@code gridPosition}.\n\t *
\n\t * Note that {@code gridPosition} is in units of chunks.\n\t *\n\t * @param datasetPath the dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param gridPosition position of chunk to be deleted\n\t * @throws N5Exception if the chunk exists but could not be deleted\n\t *\n\t * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted.\n\t */\n\tboolean deleteChunk(\n\t\t\tString datasetPath,\n\t\t\tDatasetAttributes datasetAttributes,\n\t\t\tlong... gridPosition) throws N5Exception;\n\n\t/**\n\t * Deletes the chunks at the given {@code gridPositions}.\n\t *
\n\t * Note that {@code gridPositions} are in units of chunks.\n\t *\n\t * @param datasetPath dataset path\n\t * @param gridPositions a list of grid positions\n\t * @return {@code true} if any of the specified chunks existed and was deleted\n\t * @throws N5Exception if any of the chunks did exist but could not be deleted\n\t */\n\tdefault boolean deleteChunks(\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal List gridPositions) throws N5Exception {\n\t\tboolean deleted = false;\n\t\tfor (long[] pos : gridPositions) {\n\t\t\tdeleted |= deleteChunk(datasetPath, datasetAttributes, pos);\n\t\t}\n\t\treturn deleted;\n\t}\n\n\t/**\n\t * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset.\n\t * The offset is given in {@link DataBlock} grid coordinates.\n\t *\n\t * @param object the object to serialize\n\t * @param datasetPath the dataset path\n\t * @param datasetAttributes the dataset attributes\n\t * @param gridPosition the grid position\n\t * @throws N5Exception the exception\n\t */\n\tdefault void writeSerializedBlock(\n\t\t\tfinal Serializable object,\n\t\t\tfinal String datasetPath,\n\t\t\tfinal DatasetAttributes datasetAttributes,\n\t\t\tfinal long... gridPosition) throws N5Exception {\n\n\t\tfinal ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();\n\t\ttry (ObjectOutputStream out = new ObjectOutputStream(byteOutputStream)) {\n\t\t\tout.writeObject(object);\n\t\t} catch (final IOException | UncheckedIOException e) {\n\t\t\tthrow new N5Exception.N5IOException(e);\n\t\t}\n\t\tfinal byte[] bytes = byteOutputStream.toByteArray();\n\t\tfinal DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes);\n\t\twriteChunk(datasetPath, datasetAttributes, dataBlock);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonSerializationContext;\nimport com.google.gson.JsonSerializer;\nimport org.janelia.saalfeldlab.n5.serialization.N5Annotations;\nimport org.janelia.saalfeldlab.n5.serialization.NameConfig;\nimport org.scijava.annotations.Index;\nimport org.scijava.annotations.IndexItem;\n\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Type;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.Map.Entry;\n\n/**\n * T adapter, auto-discovers annotated T implementations in the classpath.\n *\n * @author Caleb Hulbert\n * \n * @param \n * the class this adapter (de)serializes\n */\npublic class NameConfigAdapter implements JsonDeserializer, JsonSerializer {\n\n\tprivate static HashMap, NameConfigAdapter> adapters = new HashMap<>();\n\n\tprivate static void registerAdapter(Class cls) {\n\n\t\tadapters.put(cls, new NameConfigAdapter<>(cls));\n\t\tupdate(adapters.get(cls));\n\t}\n\tprivate final HashMap> constructors = new HashMap<>();\n\n\tprivate final HashMap> parameters = new HashMap<>();\n\tprivate final HashMap> parameterNames = new HashMap<>();\n\tprivate static ArrayList getDeclaredFields(Class clazz) {\n\n\t\tfinal ArrayList fields = new ArrayList<>();\n\t\tfields.addAll(Arrays.asList(clazz.getDeclaredFields()));\n\t\tfor (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass())\n\t\t\tfields.addAll(Arrays.asList(clazz.getDeclaredFields()));\n\t\treturn fields;\n\t}\n\n\t@SuppressWarnings(\"unchecked\")\n\tpublic static synchronized void update(final NameConfigAdapter adapter) {\n\n\t\tfinal String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value();\n\t\tfinal ClassLoader classLoader = Thread.currentThread().getContextClassLoader();\n\t\tfinal Index annotationIndex = Index.load(NameConfig.Name.class, classLoader);\n\t\tfor (final IndexItem item : annotationIndex) {\n\t\t\tClass clazz;\n\t\t\ttry {\n\t\t\t\tclazz = (Class)Class.forName(item.className());\n\n\t\t\t\tfinal NameConfig.Serialize serialize = clazz.getAnnotation(NameConfig.Serialize.class);\n\t\t\t\tif (serialize != null && !serialize.value())\n\t\t\t\t\tcontinue;\n\n\t\t\t\tfinal String name = clazz.getAnnotation(NameConfig.Name.class).value();\n\t\t\t\tfinal String type = prefix + \".\" + name;\n\n\t\t\t\tfinal Constructor constructor = clazz.getDeclaredConstructor();\n\n\t\t\t\tfinal HashMap parameters = new HashMap<>();\n\t\t\t\tfinal HashMap parameterNames = new HashMap<>();\n\t\t\t\tfinal ArrayList fields = getDeclaredFields(clazz);\n\t\t\t\tfor (final Field field : fields) {\n\t\t\t\t\tfinal NameConfig.Parameter parameter = field.getAnnotation(NameConfig.Parameter.class);\n\t\t\t\t\tif (parameter != null) {\n\n\t\t\t\t\t\tfinal String parameterName;\n\t\t\t\t\t\tif (parameter.value().equals(\"\"))\n\t\t\t\t\t\t\tparameterName = field.getName();\n\t\t\t\t\t\telse\n\t\t\t\t\t\t\tparameterName = parameter.value();\n\n\t\t\t\t\t\tparameterNames.put(field.getName(), parameterName);\n\n\t\t\t\t\t\tparameters.put(field.getName(), field);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tadapter.constructors.put(type, constructor);\n\t\t\t\tadapter.parameters.put(type, parameters);\n\t\t\t\tadapter.parameterNames.put(type, parameterNames);\n\t\t\t} catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException\n\t\t\t\t\t\t | UnsatisfiedLinkError e) {\n\n\t\t\t\tSystem.err.println(\"T '\" + item.className() + \"' could not be registered\");\n\t\t\t\te.printStackTrace(System.err);\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate final Class type;\n\n\tpublic NameConfigAdapter(Class cls) {\n\t\tthis.type = cls;\n\t}\n\n\t@Override\n\tpublic JsonElement serialize(\n\t\t\tfinal T object,\n\t\t\tfinal Type typeOfSrc,\n\t\t\tfinal JsonSerializationContext context) {\n\n\t\tfinal Class clazz = (Class)object.getClass();\n\n\t\tfinal String name = clazz.getAnnotation(NameConfig.Name.class).value();\n\t\tfinal String prefix = type.getAnnotation(NameConfig.Prefix.class).value();\n\t\tfinal String type = prefix + \".\" + name;\n\n\t\tfinal JsonObject json = new JsonObject();\n\t\tjson.addProperty(\"name\", name);\n\t\tfinal JsonObject configuration = new JsonObject();\n\n\t\tfinal HashMap parameterTypes = parameters.get(type);\n\t\tfinal HashMap parameterNameMap = parameterNames.get(type);\n\t\ttry {\n\t\t\tfor (final Entry parameterType : parameterTypes.entrySet()) {\n\t\t\t\tfinal String fieldName = parameterType.getKey();\n\t\t\t\tfinal Field field = clazz.getDeclaredField(fieldName);\n\t\t\t\tfinal boolean isAccessible = field.isAccessible();\n\t\t\t\tfield.setAccessible(true);\n\t\t\t\tfinal Object value = field.get(object);\n\t\t\t\tfield.setAccessible(isAccessible);\n\t\t\t\tfinal JsonElement serialized = context.serialize(value);\n\t\t\t\tif (field.getAnnotation(N5Annotations.ReverseArray.class) != null) {\n\t\t\t\t\tfinal JsonArray reversedArray = reverseJsonArray(serialized.getAsJsonArray());\n\t\t\t\t\tconfiguration.add(parameterNameMap.get(fieldName), reversedArray);\n\t\t\t\t} else\n\t\t\t\t\tconfiguration.add(parameterNameMap.get(fieldName), serialized);\n\n\t\t\t}\n\t\t\tif (!configuration.isEmpty())\n\t\t\t\tjson.add(\"configuration\", configuration);\n\t\t} catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {\n\t\t\tnew RuntimeException(\"Could not serialize \" + clazz.getName(), e).printStackTrace(System.err);\n\t\t\treturn null;\n\t\t}\n\n\t\treturn json;\n\t}\n\n\t@Override\n\tpublic T deserialize(\n\t\t\tfinal JsonElement json,\n\t\t\tfinal Type typeOfT,\n\t\t\tfinal JsonDeserializationContext context) throws JsonParseException {\n\n\t\tfinal String prefix = type.getAnnotation(NameConfig.Prefix.class).value();\n\n\t\tfinal JsonObject objectJson = json.getAsJsonObject();\n\t\tfinal String name = objectJson.getAsJsonPrimitive(\"name\").getAsString();\n\t\tif (name == null) {\n\t\t\treturn null;\n\t\t}\n\n\t\tfinal String type = prefix + \".\" + name;\n\n\t\tfinal JsonObject configuration = objectJson.getAsJsonObject(\"configuration\");\n\t\t/* It's ok to be null if all parameters are optional.\n\t\t* Otherwise, return*/\n\t\tif (configuration == null) {\n\t\t\tfor (final Field field : parameters.get(type).values()) {\n\t\t\t\tif (!field.getAnnotation(NameConfig.Parameter.class).optional())\n\t\t\t\t\treturn null;\n\t\t\t}\n\t\t}\n\n\t\tfinal Constructor constructor = constructors.get(type);\n\t\tconstructor.setAccessible(true);\n\t\tfinal T object;\n\t\ttry {\n\t\t\tobject = constructor.newInstance();\n\t\t\tfinal HashMap parameterTypes = parameters.get(type);\n\t\t\tfinal HashMap parameterNameMap = parameterNames.get(type);\n\t\t\tfor (final Entry parameterType : parameterTypes.entrySet()) {\n\t\t\t\tfinal String fieldName = parameterType.getKey();\n\t\t\t\tfinal String paramName = parameterNameMap.get(fieldName);\n\t\t\t\tfinal JsonElement paramJson = configuration == null ? null : configuration.get(paramName);\n\t\t\t\tfinal Field field = parameterType.getValue();\n\t\t\t\tif (paramJson != null) {\n\t\t\t\t\tfinal Object parameter;\n\t\t\t\t\tif (field.getAnnotation(N5Annotations.ReverseArray.class) != null) {\n\t\t\t\t\t\tfinal JsonArray reversedArray = reverseJsonArray(paramJson);\n\t\t\t\t\t\tparameter = context.deserialize(reversedArray, field.getType());\n\t\t\t\t\t} else\n\t\t\t\t\t\tparameter = context.deserialize(paramJson, field.getType());\n\t\t\t\t\tReflectionUtils.setFieldValue(object, fieldName, parameter);\n\t\t\t\t} else if (!field.getAnnotation(NameConfig.Parameter.class).optional()) {\n\t\t\t\t\t/* if param is null, and not optional, return null */\n\t\t\t\t\treturn null;\n\t\t\t\t}\n\t\t\t}\n\t\t} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException\n\t\t\t\t | SecurityException | NoSuchFieldException e) {\n\t\t\te.printStackTrace(System.err);\n\t\t\treturn null;\n\t\t}\n\n\t\treturn object;\n\t}\n\n\tprivate static JsonArray reverseJsonArray(JsonElement paramJson) {\n\n\t\tfinal JsonArray reversedJson = new JsonArray(paramJson.getAsJsonArray().size());\n\t\tfor (int i = paramJson.getAsJsonArray().size() - 1; i >= 0; i--) {\n\t\t\treversedJson.add(paramJson.getAsJsonArray().get(i));\n\t\t}\n\t\treturn reversedJson;\n\t}\n\n\tpublic static NameConfigAdapter getJsonAdapter(Class cls) {\n\n\t\tif (adapters.get(cls) == null)\n\t\t\tregisterAdapter(cls);\n\t\treturn (NameConfigAdapter) adapters.get(cls);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport org.janelia.saalfeldlab.n5.Compression.CompressionType;\nimport org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.serialization.NameConfig;\n\n@CompressionType(\"raw\")\n@NameConfig.Name(\"raw\")\npublic class RawCompression implements Compression, DeterministicSizeDataCodec {\n\n\tprivate static final long serialVersionUID = 7526445806847086477L;\n\n\t@Override\n\tpublic boolean equals(final Object other) {\n\n\t\treturn other != null && other.getClass() == RawCompression.class;\n\t}\n\n\t@Override\n\tpublic ReadData encode(final ReadData readData) {\n\t\treturn readData;\n\t}\n\n\t@Override\n\tpublic ReadData decode(final ReadData readData) {\n\t\treturn readData;\n\t}\n\n\t@Override\n\tpublic long encodedSize(final long size) {\n\t\treturn size;\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Modifier;\n\nclass ReflectionUtils {\n\n\tstatic void setFieldValue(\n\t\t\tfinal Object object,\n\t\t\tfinal String fieldName,\n\t\t\tfinal T value) throws NoSuchFieldException, IllegalAccessException {\n\n\t\tField modifiersField;\n\t\tboolean isModifiersAccessible;\n\t\ttry {\n\t\t\tmodifiersField = Field.class.getDeclaredField(\"modifiers\");\n\t\t\tisModifiersAccessible = modifiersField.isAccessible();\n\t\t\tmodifiersField.setAccessible(true);\n\t\t} catch (final NoSuchFieldException e) {\n\t\t\t// Java 11+ does not allow to access modifiers\n\t\t\tmodifiersField = null;\n\t\t\tisModifiersAccessible = false;\n\t\t}\n\n\t\tfinal Field field = object.getClass().getDeclaredField(fieldName);\n\t\tfinal boolean isFieldAccessible = field.isAccessible();\n\t\tfield.setAccessible(true);\n\n\t\tif (modifiersField != null) {\n\t\t\tfinal int modifiers = field.getModifiers();\n\t\t\tmodifiersField.setInt(field, modifiers & ~Modifier.FINAL);\n\t\t\tfield.set(object, value);\n\t\t\tmodifiersField.setInt(field, modifiers);\n\t\t} else {\n\t\t\tfield.set(object, value);\n\t\t}\n\n\t\tfield.setAccessible(isFieldAccessible);\n\t\tif (modifiersField != null) {\n\t\t\tmodifiersField.setAccessible(isModifiersAccessible);\n\t\t}\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\npublic class ShortArrayDataBlock extends AbstractDataBlock {\n\n\tpublic ShortArrayDataBlock(final int[] size, final long[] gridPosition, final short[] data) {\n\n\t\tsuper(size, gridPosition, data, a -> a.length);\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.nio.ByteBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\n\npublic class StringDataBlock extends AbstractDataBlock {\n\n protected static final Charset ENCODING = StandardCharsets.UTF_8;\n protected static final String NULLCHAR = \"\\0\";\n protected byte[] serializedData = null;\n protected String[] actualData = null;\n\n public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) {\n super(size, gridPosition, new String[0], a -> a.length);\n actualData = data;\n }\n\n public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) {\n super(size, gridPosition, new String[0], a -> a.length);\n serializedData = data;\n }\n\n public void readData(final ByteBuffer buffer) {\n\n\t\tif (buffer.hasArray()) {\n\t\t\tif (buffer.array() != serializedData)\n\t\t\t\tbuffer.get(serializedData);\n\t\t\tactualData = deserialize(buffer.array());\n\t\t} else\n\t\t\tactualData = ENCODING.decode(buffer).toString().split(NULLCHAR);\n }\n\n protected byte[] serialize(String[] strings) {\n final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR;\n return flattenedArray.getBytes(ENCODING);\n }\n\n protected String[] deserialize(byte[] rawBytes) {\n final String rawChars = new String(rawBytes, ENCODING);\n return rawChars.split(NULLCHAR);\n }\n\n @Override\n public int getNumElements() {\n if (serializedData == null)\n serializedData = serialize(actualData);\n return serializedData.length;\n }\n\n @Override\n public String[] getData() {\n if (actualData == null)\n actualData = deserialize(serializedData);\n return actualData;\n }\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java",
"content": "package org.janelia.saalfeldlab.n5;\n\nimport java.io.IOException;\nimport org.apache.commons.compress.compressors.xz.XZCompressorInputStream;\nimport org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;\nimport org.janelia.saalfeldlab.n5.Compression.CompressionType;\nimport org.janelia.saalfeldlab.n5.N5Exception.N5IOException;\nimport org.janelia.saalfeldlab.n5.readdata.ReadData;\nimport org.janelia.saalfeldlab.n5.serialization.NameConfig;\n\n@CompressionType(\"xz\")\n@NameConfig.Name(\"xz\")\npublic class XzCompression implements Compression {\n\n\tprivate static final long serialVersionUID = -7272153943564743774L;\n\n\t@CompressionParameter\n\t@NameConfig.Parameter\n\tprivate final int preset;\n\n\tpublic XzCompression(final int preset) {\n\n\t\tthis.preset = preset;\n\t}\n\n\tpublic XzCompression() {\n\n\t\tthis(6);\n\t}\n\n\t@Override\n\tpublic boolean equals(final Object other) {\n\n\t\tif (other == null || other.getClass() != XzCompression.class)\n\t\t\treturn false;\n\t\telse\n\t\t\treturn preset == ((XzCompression)other).preset;\n\t}\n\n\t@Override\n\tpublic ReadData decode(final ReadData readData) throws N5IOException {\n\n\t\ttry {\n\t\t\treturn ReadData.from(new XZCompressorInputStream(readData.inputStream()));\n\t\t} catch (IOException e) {\n\t\t\tthrow new N5IOException(e);\n\t\t}\n\t}\n\n\t@Override\n\tpublic ReadData encode(final ReadData readData) {\n\t\treturn readData.encode(out -> new XZCompressorOutputStream(out, preset));\n\t}\n}\n"
},
{
"path": "src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java",
"content": "package org.janelia.saalfeldlab.n5.cache;\n\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.LinkedHashSet;\n\nimport org.janelia.saalfeldlab.n5.N5Exception;\n\nimport com.google.gson.JsonElement;\n\n/*\n * A cache containing JSON attributes and children for groups and\n * datasets stored in N5 containers. Used by {@link CachedGsonKeyValueN5Reader}\n * and {@link CachedGsonKeyValueN5Writer}.\n *\n */\npublic class N5JsonCache {\n\n\tpublic static final N5CacheInfo emptyCacheInfo = new N5CacheInfo();\n\n\tpublic static final EmptyJson emptyJson = new EmptyJson();\n\n\tprotected final N5JsonCacheableContainer container;\n\n\t/**\n\t * Data object for caching meta data. Elements that are null are not yet\n\t * cached.\n\t */\n\tprotected static class N5CacheInfo {\n\n\t\tprotected final HashMap attributesCache = new HashMap<>();\n\t\tprotected LinkedHashSet children = null;\n\t\tprotected boolean isDataset = false;\n\t\tprotected boolean isGroup = false;\n\n\t\tpublic JsonElement getCache(final String normalCacheKey) {\n\n\t\t\t// synchronize the method instead?\n\t\t\tsynchronized (attributesCache) {\n\t\t\t\treturn attributesCache.get(normalCacheKey);\n\t\t\t}\n\t\t}\n\n\t\tpublic boolean containsKey(final String normalCacheKey) {\n\n\t\t\t// synchronize the method instead?\n\t\t\tsynchronized (attributesCache) {\n\t\t\t\treturn attributesCache.containsKey(normalCacheKey);\n\t\t\t}\n\t\t}\n\n\t\tpublic boolean isDataset() {\n\n\t\t\treturn isDataset;\n\t\t}\n\n\t\tpublic boolean isGroup() {\n\n\t\t\treturn isGroup;\n\t\t}\n\t}\n\n\t@SuppressWarnings(\"deprecation\")\n\tprotected static class EmptyJson extends JsonElement {\n\n\t\t@Override\n\t\tpublic JsonElement deepCopy() {\n\n\t\t\tthrow new N5Exception(\"Do not copy EmptyJson, you naughty person\");\n\t\t}\n\n\t}\n\n\tprivate final HashMap