T getAttribute(
final String pathName,
final String key,
final Type type) throws N5Exception {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
JsonElement attributes;
if (cacheMeta()) {
attributes = getCache().getAttributes(normalPathName, getAttributesKey());
} else {
attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
}
try {
return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson());
} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {
throw new N5Exception.N5ClassCastException(e);
}
}
@Override
default boolean exists(final String pathName) {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
if (cacheMeta())
return getCache().isGroup(normalPathName, getAttributesKey());
else {
return existsFromContainer(normalPathName, null);
}
}
@Override
default boolean existsFromContainer(final String normalPathName, final String normalCacheKey) {
final KeyValueAccess kva = getKeyValueAccess();
if (normalCacheKey == null)
return kva.isDirectory(kva.compose(getURI(), normalPathName));
else
return kva.isFile(kva.compose(getURI(), normalPathName, normalCacheKey));
}
@Override
default boolean groupExists(final String pathName) {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
if (cacheMeta())
return getCache().isGroup(normalPathName, null);
else {
return isGroupFromContainer(normalPathName);
}
}
@Override
default boolean isGroupFromContainer(final String normalPathName) {
return GsonKeyValueN5Reader.super.groupExists(normalPathName);
}
@Override
default boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) {
return true;
}
@Override
default boolean datasetExists(final String pathName) throws N5IOException {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
if (cacheMeta()) {
return getCache().isDataset(normalPathName, getAttributesKey());
}
return isDatasetFromContainer(normalPathName);
}
@Override
default boolean isDatasetFromContainer(final String normalPathName) throws N5IOException {
return normalGetDatasetAttributes(normalPathName) != null;
}
@Override
default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) {
return isGroupFromAttributes(normalCacheKey, attributes) && createDatasetAttributes(attributes) != null;
}
/**
* Reads or creates the attributes map of a group or dataset.
*
* @param pathName
* group path
* @return the attribute
* @throws N5IOException if an IO error occurs while reading the attribute
*/
@Override
default JsonElement getAttributes(final String pathName) throws N5IOException {
final String groupPath = N5URI.normalizeGroupPath(pathName);
/* If cached, return the cache */
if (cacheMeta()) {
return getCache().getAttributes(groupPath, getAttributesKey());
} else {
return GsonKeyValueN5Reader.super.getAttributes(groupPath);
}
}
@Override
default String[] list(final String pathName) throws N5IOException {
final String normalPath = N5URI.normalizeGroupPath(pathName);
if (cacheMeta()) {
return getCache().list(normalPath);
} else {
return GsonKeyValueN5Reader.super.list(normalPath);
}
}
@Override
default String[] listFromContainer(final String normalPathName) {
// this implementation doesn't use cache key, but rather depends on
return GsonKeyValueN5Reader.super.list(normalPathName);
}
/**
* Check for attributes that are required for a group to be a dataset.
*
* @param attributes
* to check for dataset attributes
* @return if {@link DatasetAttributes#DIMENSIONS_KEY} and
* {@link DatasetAttributes#DATA_TYPE_KEY} are present
*/
static boolean hasDatasetAttributes(final JsonElement attributes) {
if (attributes == null || !attributes.isJsonObject()) {
return false;
}
final JsonObject metadataCache = attributes.getAsJsonObject();
return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY)
&& metadataCache.has(DatasetAttributes.DATA_TYPE_KEY);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
/**
* Cached default implementation of {@link N5Writer} with JSON attributes parsed
* with {@link Gson}.
*/
public interface CachedGsonKeyValueN5Writer extends CachedGsonKeyValueN5Reader, GsonKeyValueN5Writer {
@Override
default void setVersion(final String path) throws N5Exception {
final Version version = getVersion();
if (!VERSION.isCompatible(version))
throw new N5IOException("Incompatible version " + version + " (this is " + VERSION + ").");
if (!VERSION.equals(version))
setAttribute("/", VERSION_KEY, VERSION.toString());;
}
@Override
default void createGroup(final String path) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(path);
// avoid hitting the backend if this path is already a group according to the cache
// else if exists is true (then a dataset is present) so throw an exception to avoid
// overwriting / invalidating existing data
if (groupExists(normalPath))
return;
else if (datasetExists(normalPath))
throw new N5Exception("Can't make a group on existing dataset.");
getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));
if (cacheMeta()) {
// check all nodes that are parents of the added node, if they have
// a children set, add the new child to it
getKeyValueAccess().parent(normalPath);
String[] pathParts = getKeyValueAccess().components(normalPath);
String parent = N5URI.normalizeGroupPath("/");
if (pathParts.length == 0) {
pathParts = new String[]{""};
}
for (final String child : pathParts) {
final String childPath = parent.isEmpty() ? child : parent + "/" + child;
getCache().initializeNonemptyCache(childPath, getAttributesKey());
getCache().updateCacheInfo(childPath, getAttributesKey());
// only add if the parent exists and has children cached already
if (parent != null && !child.isEmpty())
getCache().addChildIfPresent(parent, child);
parent = childPath;
}
}
}
@Override
default void writeAttributes(
final String normalGroupPath,
final JsonElement attributes) throws N5Exception {
writeAndCacheAttributes(normalGroupPath, attributes);
}
default void writeAndCacheAttributes(
final String normalGroupPath,
final JsonElement attributes) throws N5Exception {
GsonKeyValueN5Writer.super.writeAttributes(normalGroupPath, attributes);
if (cacheMeta()) {
JsonElement nullRespectingAttributes = attributes;
/*
* Gson only filters out nulls when you write the JsonElement. This
* means it doesn't filter them out when caching.
* To handle this, we explicitly writer the existing JsonElement to
* a new JsonElement.
* The output is identical to the input if:
* - serializeNulls is true
* - no null values are present
* - caching is turned off
*/
if (!getGson().serializeNulls()) {
nullRespectingAttributes = getGson().toJsonTree(attributes);
}
/* Update the cache, and write to the writer */
getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes);
}
}
@Override
default boolean remove(final String path) throws N5Exception {
// GsonKeyValueN5Writer.super.remove(path)
/*
* the lines below duplicate the single line above but would have to call
* normalizeGroupPath again the below duplicates code, but avoids extra work
*/
final String normalPath = N5URI.normalizeGroupPath(path);
final String groupPath = absoluteGroupPath(normalPath);
if (getKeyValueAccess().isDirectory(groupPath))
getKeyValueAccess().delete(groupPath);
if (cacheMeta()) {
final String parentPath = getKeyValueAccess().parent(normalPath);
getCache().removeCache(parentPath, normalPath);
}
/* an IOException should have occurred if anything had failed midway */
return true;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
/**
* Holds a channel and system-level file lock (shared for writing, non-shared
* for reading) and keeps it open until this {@code ChannelLock} is {@link
* #close() closed}.
*/
class ChannelLock implements Closeable {
private final FileChannel channel;
/**
* Hold a hard reference to the {@code FileLock} to make sure it is not
* prematurely released.
*
* NB: We do not call {@code lock.release()} in {@link #close}, because at
* this point the channel might be already closed (by an external writer).
* {@code lock.release()} will throw an exception if the channel is already
* closed. Instead, we just close the channel which will automatically
* release the lock.
*/
@SuppressWarnings({"unused", "FieldCanBeLocal"})
private final FileLock lock;
private ChannelLock(final FileChannel channel, final FileLock lock) {
this.channel = channel;
this.lock = lock;
}
public void close() throws IOException {
// NB: We do not call lock.release() here, because it may throw an
// exception if the channel is already closed. Instead, we just close
// the channel. This will automatically release the lock. (And it is ok
// to close an already closed channel.)
channel.close();
}
FileChannel getChannel() {
return channel;
}
/**
* Create a {@link FileChannel} on the given {@code path} and lock it with a
* system-level {@link FileLock}. If there is an existing overlapping file
* lock, this method will block until the existing lock is released and the
* channel could be locked (by us).
*
* The {@code FileLock} is exclusive if the {@code path} is locked {@code
* forWriting}, and shared otherwise.
*
* If the {@code path} is locked {@code forWriting} non-existing file and
* the parent directories are created as needed.
*
* @throws IOException if an error occurs while opening the channel, or if
* the calling thread is interrupted while waiting for the {@code FileLock}.
*/
static ChannelLock lock(final Path path, final boolean forWriting, final LockingPolicy policy) throws IOException {
final FileChannel channel = openFileChannel(path, forWriting);
if (policy == LockingPolicy.UNSAFE) {
return new ChannelLock(channel, null);
}
try {
while (true) {
try {
final FileLock lock = channel.lock(0, Long.MAX_VALUE, !forWriting);
return new ChannelLock(channel, lock);
} catch (final OverlappingFileLockException e) {
try {
Thread.sleep(100);
} catch (final InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while waiting for file lock", ie);
}
}
}
} catch (Exception e) {
if (policy == LockingPolicy.STRICT) {
closeQuietly(channel);
throw e;
} else {
return new ChannelLock(channel, null);
}
}
}
/**
* Opens a file channel. If the channel is opened {@code forWriting},
* then this may create the file and the parent directories as needed.
*
* @throws IOException
* if the channel cannot be opened
*/
private static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException {
if (forWriting) {
final Path parent = path.getParent();
if (parent != null) {
Files.createDirectories(parent);
}
return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
} else {
return FileChannel.open(path, StandardOpenOption.READ);
}
}
private static void closeQuietly(final FileChannel fileChannel) {
if (fileChannel != null) {
try {
fileChannel.close();
} catch (final IOException | UncheckedIOException ignored) {
}
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/Compression.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.Serializable;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.janelia.saalfeldlab.n5.codec.DataCodec;
import org.janelia.saalfeldlab.n5.codec.CodecInfo;
import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
import org.scijava.annotations.Indexable;
/**
* This interface is used to indicate that a {@link DataCodec} can be
* serialized as a "compression" for the N5 format (using the N5 API).
*
* N5Readers and N5Writers for the N5 format can declare DataCodecs that
* implement this interface so that the {@link CompressionAdapter} is used for
* serialization.
*
* See also: an alternative method for serializing general {@link CodecInfo}s is
* with the {@link NameConfigAdapter}. This interface remains for legacy
* (de)serialization.
*
* @author Stephan Saalfeld
*/
public interface Compression extends Serializable, DataCodec, DataCodecInfo {
/**
* Annotation for runtime discovery of compression schemes.
*
*/
@Retention(RetentionPolicy.RUNTIME)
@Inherited
@Target(ElementType.TYPE)
@Indexable
@interface CompressionType {
String value();
}
/**
* Annotation for runtime discovery of compression schemes.
*
*/
@Retention(RetentionPolicy.RUNTIME)
@Inherited
@Target(ElementType.FIELD)
@interface CompressionParameter {}
default String getType() {
final CompressionType compressionType = getClass().getAnnotation(CompressionType.class);
if (compressionType == null)
return null;
else
return compressionType.value();
}
@Override
default DataCodec create() {
return this;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java
================================================
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map.Entry;
import org.janelia.saalfeldlab.n5.Compression.CompressionParameter;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
import org.scijava.annotations.Index;
import org.scijava.annotations.IndexItem;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
/**
* Compression adapter, auto-discovers annotated compression implementations
* in the classpath.
*
* @author Stephan Saalfeld
*/
public class CompressionAdapter implements JsonDeserializer, JsonSerializer {
private static CompressionAdapter instance = null;
private final HashMap> compressionConstructors = new HashMap<>();
private final HashMap>> compressionParameters = new HashMap<>();
private static ArrayList getDeclaredFields(Class clazz) {
final ArrayList fields = new ArrayList<>();
fields.addAll(Arrays.asList(clazz.getDeclaredFields()));
for (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass())
fields.addAll(Arrays.asList(clazz.getDeclaredFields()));
return fields;
}
@SuppressWarnings("unchecked")
public static synchronized void update(final boolean override) {
if (override || instance == null) {
final CompressionAdapter newInstance = new CompressionAdapter();
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
final Index annotationIndex = Index.load(CompressionType.class, classLoader);
for (final IndexItem item : annotationIndex) {
Class clazz;
try {
clazz = (Class)Class.forName(item.className());
final String type = clazz.getAnnotation(CompressionType.class).value();
final Constructor constructor = clazz.getDeclaredConstructor();
final HashMap> parameters = new HashMap<>();
final ArrayList fields = getDeclaredFields(clazz);
for (final Field field : fields) {
if (field.getAnnotation(CompressionParameter.class) != null) {
parameters.put(field.getName(), field.getType());
}
}
newInstance.compressionConstructors.put(type, constructor);
newInstance.compressionParameters.put(type, parameters);
} catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException
| UnsatisfiedLinkError e) {
System.err.println("Compression '" + item.className() + "' could not be registered");
}
}
instance = newInstance;
}
}
public static void update() {
update(false);
}
@Override
public JsonElement serialize(
final Compression compression,
final Type typeOfSrc,
final JsonSerializationContext context) {
final String type = compression.getType();
final Class clazz = compression.getClass();
final JsonObject json = new JsonObject();
json.addProperty("type", type);
final HashMap> parameterTypes = compressionParameters.get(type);
try {
for (final Entry> parameterType : parameterTypes.entrySet()) {
final String name = parameterType.getKey();
final Field field = clazz.getDeclaredField(name);
final boolean isAccessible = field.isAccessible();
field.setAccessible(true);
final Object value = field.get(compression);
field.setAccessible(isAccessible);
json.add(parameterType.getKey(), context.serialize(value));
}
} catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {
e.printStackTrace(System.err);
return null;
}
return json;
}
@Override
public Compression deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {
final JsonObject jsonObject = json.getAsJsonObject();
final JsonElement jsonType = jsonObject.get("type");
if (jsonType == null)
return null;
final String type = jsonType.getAsString();
final Constructor constructor = compressionConstructors.get(type);
final Compression compression;
try {
compression = constructor.newInstance();
final HashMap> parameterTypes = compressionParameters.get(type);
for (final Entry> parameterType : parameterTypes.entrySet()) {
final String name = parameterType.getKey();
if (jsonObject.has(name)) {
final Object parameter = context.deserialize(jsonObject.get(name), parameterType.getValue());
ReflectionUtils.setFieldValue(compression, name, parameter);
}
}
} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException
| SecurityException | NoSuchFieldException e) {
e.printStackTrace(System.err);
return null;
}
return compression;
}
public static CompressionAdapter getJsonAdapter() {
if (instance == null)
update();
return instance;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
/**
* Interface for data blocks. A data block has data, a position on the block
* grid, and a size.
*
* @param type of the data contained in the DataBlock
*
* @author Stephan Saalfeld
*/
public interface DataBlock {
/**
* Returns the size of this data block.
*
* The size of a data block is expected to be smaller than or equal to the
* spacing of the block grid. The dimensionality of size is expected to be
* equal to the dimensionality of the dataset. Consistency is not enforced.
*
* @return size of the data block
*/
int[] getSize();
/**
* Returns the position of this data block on the block grid relative to dataset.
*
* The dimensionality of the grid position is expected to be equal to the
* dimensionality of the dataset. Consistency is not enforced.
*
* @return position on the block grid
*/
long[] getGridPosition();
/**
* Returns the data object held by this data block.
*
* @return data object
*/
T getData();
/**
* Returns the number of elements in this {@link DataBlock}. This number is
* not necessarily equal {@link #getNumElements(int[])
* getNumElements(getSize())}.
*
* @return the number of elements
*/
int getNumElements();
/**
* Returns the number of elements in a box of given size.
*
* @param size
* the size
* @return the number of elements
*/
static int getNumElements(final int[] size) {
int n = size[0];
for (int i = 1; i < size.length; ++i)
n *= size[i];
return n;
}
/**
* Factory for creating {@code DataBlock}.
*
* @param
* type of the data contained in the DataBlock
*/
interface DataBlockFactory {
/**
* Create a new {@link DataBlock} with the given {@code blockSize}, {@code gridPosition}, and {@code data} content.
*
* @param blockSize
* the block size
* @param gridPosition
* the grid position
* @param data
* the data object
*
* @return a new DataBlock
*/
DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/DataType.java
================================================
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Type;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
/**
* Enumerates available data types.
*
* @author Stephan Saalfeld
*/
public enum DataType {
UINT8(
"uint8",
(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(
blockSize,
gridPosition,
new byte[numElements])),
UINT16(
"uint16",
(blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(
blockSize,
gridPosition,
new short[numElements])),
UINT32(
"uint32",
(blockSize, gridPosition, numElements) -> new IntArrayDataBlock(
blockSize,
gridPosition,
new int[numElements])),
UINT64(
"uint64",
(blockSize, gridPosition, numElements) -> new LongArrayDataBlock(
blockSize,
gridPosition,
new long[numElements])),
INT8(
"int8",
(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(
blockSize,
gridPosition,
new byte[numElements])),
INT16(
"int16",
(blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(
blockSize,
gridPosition,
new short[numElements])),
INT32(
"int32",
(blockSize, gridPosition, numElements) -> new IntArrayDataBlock(
blockSize,
gridPosition,
new int[numElements])),
INT64(
"int64",
(blockSize, gridPosition, numElements) -> new LongArrayDataBlock(
blockSize,
gridPosition,
new long[numElements])),
FLOAT32(
"float32",
(blockSize, gridPosition, numElements) -> new FloatArrayDataBlock(
blockSize,
gridPosition,
new float[numElements])),
FLOAT64(
"float64",
(blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock(
blockSize,
gridPosition,
new double[numElements])),
STRING(
"string",
(blockSize, gridPosition, numElements) -> new StringDataBlock(
blockSize,
gridPosition,
new String[numElements])),
OBJECT(
"object",
(blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(
blockSize,
gridPosition,
new byte[numElements]));
private final String label;
private final DataBlockFactory dataBlockFactory;
DataType(final String label, final DataBlockFactory dataBlockFactory) {
this.label = label;
this.dataBlockFactory = dataBlockFactory;
}
@Override
public String toString() {
return label;
}
public static DataType fromString(final String string) {
for (final DataType value : values())
if (value.toString().equals(string))
return value;
return null;
}
/**
* Factory for {@link DataBlock DataBlocks}.
*
* @param blockSize
* the block size
* @param gridPosition
* the grid position
* @param numElements
* the number of elements (not necessarily one element per block
* element)
* @return the data block
*/
public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) {
return dataBlockFactory.createDataBlock(blockSize, gridPosition, numElements);
}
/**
* Factory for {@link DataBlock DataBlocks} with one data element for each
* block element (e.g. pixel image).
*
* @param blockSize
* the block size
* @param gridPosition
* the grid position
* @return the data block
*/
public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) {
return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize));
}
private interface DataBlockFactory {
DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements);
}
static public class JsonAdapter implements JsonDeserializer, JsonSerializer {
@Override
public DataType deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {
return DataType.fromString(json.getAsString());
}
@Override
public JsonElement serialize(
final DataType src,
final Type typeOfSrc,
final JsonSerializationContext context) {
return new JsonPrimitive(src.toString());
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java
================================================
package org.janelia.saalfeldlab.n5;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import org.janelia.saalfeldlab.n5.codec.BlockCodec;
import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
import org.janelia.saalfeldlab.n5.codec.CodecInfo;
import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
import org.janelia.saalfeldlab.n5.shard.DatasetAccess;
import org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess;
import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo;
import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
import java.io.Serializable;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.HashMap;
import java.util.stream.Collectors;
import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
import org.janelia.saalfeldlab.n5.codec.DatasetCodec;
import org.janelia.saalfeldlab.n5.codec.DatasetCodecInfo;
/**
* Mandatory dataset attributes:
*
*
* - long[] : dimensions
* - int[] : blockSize
* - {@link DataType} : dataType
* - {@link CodecInfo}... : encode/decode routines
*
*
* @author Stephan Saalfeld
*/
public class DatasetAttributes implements Serializable {
private static final long serialVersionUID = -4521467080388947553L;
public static final String DIMENSIONS_KEY = "dimensions";
public static final String BLOCK_SIZE_KEY = "blockSize";
public static final String SHARD_SIZE_KEY = "shardSize";
public static final String DATA_TYPE_KEY = "dataType";
public static final String COMPRESSION_KEY = "compression";
public static final String CODEC_KEY = "codecs";
public static final String[] N5_DATASET_ATTRIBUTES = new String[]{
DIMENSIONS_KEY, BLOCK_SIZE_KEY, DATA_TYPE_KEY, COMPRESSION_KEY, CODEC_KEY
};
/* version 0 */
protected static final String compressionTypeKey = "compressionType";
private final long[] dimensions;
// number of samples per chunk per dimension
private final int[] chunkSize;
// number of samples per block per dimension
// identical to chunkSize for non-sharded datasets
private final int[] blockSize;
private final DataType dataType;
private final JsonElement defaultValue;
private final BlockCodecInfo blockCodecInfo;
private final DataCodecInfo[] dataCodecInfos;
private final DatasetCodecInfo[] datasetCodecInfos;
private transient final DatasetAccess access;
public DatasetAttributes(
final long[] dimensions,
final int[] blockSize,
final DataType dataType,
final JsonElement defaultValue,
final BlockCodecInfo blockCodecInfo,
final DatasetCodecInfo[] datasetCodecInfos,
final DataCodecInfo... dataCodecInfos) {
this.dimensions = dimensions;
this.dataType = dataType;
this.blockSize = blockSize;
this.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue;
this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo;
this.datasetCodecInfos = datasetCodecInfos;
if (dataCodecInfos == null)
this.dataCodecInfos = new DataCodecInfo[0];
else
this.dataCodecInfos = Arrays.stream(dataCodecInfos)
.filter(it -> it != null && !(it instanceof RawCompression))
.toArray(DataCodecInfo[]::new);
access = createDatasetAccess();
chunkSize = access.getGrid().getBlockSize(0);
}
public DatasetAttributes(
final long[] dimensions,
final int[] outerBlockSize,
final DataType dataType,
final BlockCodecInfo blockCodecInfo,
final DatasetCodecInfo[] datasetCodecInfos,
final DataCodecInfo... dataCodecInfos) {
this(dimensions, outerBlockSize, dataType, JsonNull.INSTANCE,
blockCodecInfo, datasetCodecInfos, dataCodecInfos);
}
public DatasetAttributes(
final long[] dimensions,
final int[] outerBlockSize,
final DataType dataType,
final BlockCodecInfo blockCodecInfo,
final DataCodecInfo... dataCodecInfos) {
this(dimensions, outerBlockSize, dataType, blockCodecInfo, null, dataCodecInfos);
}
/**
* Constructs a DatasetAttributes instance with specified dimensions, block size, data type,
* and single compressor with default codec.
*
* @param dimensions the dimensions of the dataset
* @param blockSize the size of the blocks in the dataset
* @param dataType the data type of the dataset
* @param dataCodecInfos the codecs used encode/decode the data
*/
public DatasetAttributes(
final long[] dimensions,
final int[] blockSize,
final DataType dataType,
final DataCodecInfo... dataCodecInfos) {
this(dimensions, blockSize, dataType, null, dataCodecInfos);
}
/**
* Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs
*
* @param dimensions the dimensions of the dataset
* @param blockSize the size of the blocks in the dataset
* @param dataType the data type of the dataset
*/
public DatasetAttributes(
final long[] dimensions,
final int[] blockSize,
final DataType dataType) {
this(dimensions, blockSize, dataType, new DataCodecInfo[0]);
}
private DatasetAccess createDatasetAccess() {
final int m = nestingDepth(blockCodecInfo);
// There are m codecs: 1 DataBlock codecs, and m-1 shard codecs.
// The inner-most codec (the DataBlock codec) is at index 0.
final int[][] blockSizes = new int[m][];
// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs
// blockCodecInfo.create below could fail unexpecedly with invalid
// blockSizes so validate first
blockSizes[m - 1] = blockSize;
BlockCodecInfo tmpInfo = blockCodecInfo;
for (int l = m - 1; l > 0; --l) {
final ShardCodecInfo info = (ShardCodecInfo)tmpInfo;
blockSizes[l - 1] = info.getInnerBlockSize();
tmpInfo = info.getInnerBlockCodecInfo();
}
BlockCodecInfo currentBlockCodecInfo = blockCodecInfo;
DataCodecInfo[] currentDataCodecInfos = dataCodecInfos;
DatasetCodecInfo[] datasetCodecInfos = this.datasetCodecInfos;
final NestedGrid grid = new NestedGrid(blockSizes, dimensions);
final BlockCodec[] blockCodecs = new BlockCodec[m];
for (int l = m - 1; l >= 0; --l) {
blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos);
if (l > 0) {
final ShardCodecInfo info = (ShardCodecInfo) currentBlockCodecInfo;
currentBlockCodecInfo = info.getInnerBlockCodecInfo();
currentDataCodecInfos = info.getInnerDataCodecInfos();
if (info.getInnerDataCodecInfos() != null) {
if (datasetCodecInfos != null && datasetCodecInfos.length > 0) {
throw new N5Exception.N5JsonParseException("Found DatasetCodecs both inside and outside of shards. Not handled");
}
else
datasetCodecInfos = info.getInnerDatasetCodecInfos();
}
}
}
// add dataset codecs
blockCodecs[0] = blockCodecWithDatasetCodecs(this, blockCodecs[0], datasetCodecInfos);
return new DefaultDatasetAccess<>(grid, blockCodecs);
}
@SuppressWarnings("unchecked")
private static BlockCodec blockCodecWithDatasetCodecs(final DatasetAttributes attributes, final BlockCodec blockCodec,
final DatasetCodecInfo[] datasetCodecInfos) {
BlockCodec result = blockCodec;
if (datasetCodecInfos != null) {
for (final DatasetCodecInfo info : datasetCodecInfos) {
result = DatasetCodec.concatenate(info.create(attributes), (BlockCodec)result);
}
}
return result;
}
private static int nestingDepth(BlockCodecInfo info) {
if (info instanceof ShardCodecInfo) {
return 1 + nestingDepth(((ShardCodecInfo)info).getInnerBlockCodecInfo());
} else {
return 1;
}
}
protected BlockCodecInfo defaultBlockCodecInfo() {
return new N5BlockCodecInfo();
}
public long[] getDimensions() {
return dimensions;
}
public int getNumDimensions() {
return dimensions.length;
}
public int[] getChunkSize() {
return chunkSize;
}
public int[] getBlockSize() {
return blockSize;
}
public JsonElement getDefaultValue() {
return defaultValue;
}
public boolean isSharded() {
return blockCodecInfo instanceof ShardCodecInfo;
}
/**
* Only used for deserialization for N5 backwards compatibility.
* {@link Compression} is no longer a special case. Prefer to reference {@link #getDataCodecInfos()}
* Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility.
*
* Deprecated in favor of {@link #getDataCodecInfos()}.
*
* @return compression CodecInfo, if one was present, or else RawCompression
*/
@Deprecated
public Compression getCompression() {
return Arrays.stream(dataCodecInfos)
.filter(it -> it instanceof Compression)
.map(it -> (Compression)it)
.findFirst()
.orElse(new RawCompression());
}
public DataType getDataType() {
return dataType;
}
/**
* Get the {@link DatasetAccess} for this dataset.
*
* @return the {@code DatasetAccess} for this dataset
*/
protected DatasetAccess getDatasetAccess() {
return (DatasetAccess) access;
}
/**
* Returns the {@code NestedGrid} for this dataset, from which block and
* shard sizes are accessible.
*
* @return the NestedGrid
*/
public NestedGrid getNestedBlockGrid() {
return getDatasetAccess().getGrid();
}
public BlockCodecInfo getBlockCodecInfo() {
return blockCodecInfo;
}
public DataCodecInfo[] getDataCodecInfos() {
return dataCodecInfos;
}
public DatasetCodecInfo[] getDatasetCodecInfos() {
return datasetCodecInfos;
}
public String relativeBlockPath(long... position) {
return Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining("/"));
}
public HashMap asMap() {
final HashMap map = new HashMap<>();
map.put(DIMENSIONS_KEY, dimensions);
map.put(BLOCK_SIZE_KEY, chunkSize);
map.put(DATA_TYPE_KEY, dataType);
map.put(COMPRESSION_KEY, getCompression());
return map;
}
public static Builder builder(final long[] dimensions, final DataType dataType) {
return new Builder(dimensions, dataType);
}
public static Builder builder(final DatasetAttributes attributes) {
return new Builder(attributes);
}
private static final int[] DEFAULT_1D_BLOCK_SIZE = new int[]{65536};
private static final int[] DEFAULT_2D_BLOCK_SIZE = new int[]{512,512};
private static final int[] DEFAULT_3D_BLOCK_SIZE = new int[]{128,128,128};
private static final int DEFAULT_ND_DIM_LEN = 64;
protected static int[] defaultBlockSize(final long[] dimensions) {
final int[] blockSize;
if (dimensions.length == 1)
blockSize = DEFAULT_1D_BLOCK_SIZE.clone();
else if (dimensions.length == 2)
blockSize = DEFAULT_2D_BLOCK_SIZE.clone();
else if (dimensions.length == 3)
blockSize = DEFAULT_3D_BLOCK_SIZE.clone();
else {
blockSize = new int[dimensions.length];
Arrays.fill(blockSize, DEFAULT_ND_DIM_LEN);
}
for (int i = 0; i < blockSize.length; i++)
blockSize[i] = (int)Math.min(blockSize[i], dimensions[i]);
return blockSize;
}
public static class Builder {
private final long[] dimensions;
private final DataType dataType;
private int[] blockSize;
private DataCodecInfo[] dataCodecInfos = new DataCodecInfo[0];
public Builder(final long[] dimensions, final DataType dataType) {
this.dimensions = dimensions.clone();
this.dataType = dataType;
this.blockSize = defaultBlockSize(dimensions);
}
public Builder(final DatasetAttributes attributes) {
this.dimensions = attributes.getDimensions();
this.dataType = attributes.getDataType();
this.blockSize = attributes.getBlockSize();
this.dataCodecInfos = attributes.getDataCodecInfos();
}
public Builder blockSize(final int[] blockSize) {
this.blockSize = blockSize.clone();
return this;
}
/**
* Sets the compression codec. Has no effect if {@code compression} is
* null or {@link RawCompression}.
*
* @param compression the compression to use
* @return this builder
*/
public Builder compression(final Compression compression) {
if (compression != null && !(compression instanceof RawCompression))
this.dataCodecInfos = new DataCodecInfo[]{compression};
return this;
}
public DatasetAttributes build() {
final int[] resolvedBlockSize = blockSize != null ? blockSize : defaultBlockSize(dimensions);
return new DatasetAttributes(dimensions, resolvedBlockSize, dataType, new N5BlockCodecInfo(), null, dataCodecInfos);
}
}
private static DatasetAttributesAdapter adapter = null;
public static DatasetAttributesAdapter getJsonAdapter() {
if (adapter == null) {
adapter = new DatasetAttributesAdapter();
}
return adapter;
}
public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer {
@Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
if (json == null || !json.isJsonObject())
return null;
final JsonObject obj = json.getAsJsonObject();
final boolean validKeySet = obj.has(DIMENSIONS_KEY)
&& obj.has(BLOCK_SIZE_KEY)
&& obj.has(DATA_TYPE_KEY)
&& (obj.has(CODEC_KEY) || obj.has(COMPRESSION_KEY) || obj.has(compressionTypeKey));
if (!validKeySet)
return null;
final long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class);
final int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class);
final DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class);
final BlockCodecInfo blockCodecInfo;
final DataCodecInfo[] dataCodecs;
if (obj.has(CODEC_KEY)) {
final CodecInfo[] codecs = context.deserialize(obj.get(CODEC_KEY), CodecInfo[].class);
blockCodecInfo = (BlockCodecInfo)codecs[0];
dataCodecs = new DataCodecInfo[codecs.length - 1];
for (int i = 1; i < codecs.length; i++) {
dataCodecs[i - 1] = (DataCodecInfo)codecs[i];
}
} else if (obj.has(COMPRESSION_KEY)) {
final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context);
dataCodecs = new DataCodecInfo[]{compression};
blockCodecInfo = new N5BlockCodecInfo();
} else if (obj.has(compressionTypeKey)) {
final Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString());
dataCodecs = new DataCodecInfo[]{compression};
blockCodecInfo = new N5BlockCodecInfo();
} else {
return null;
}
return new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs);
}
//FIXME
// this implements multi-codec serialization for N5. We probably don't want this now
@Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) {
final JsonObject obj = new JsonObject();
obj.add(DIMENSIONS_KEY, context.serialize(src.dimensions));
obj.add(BLOCK_SIZE_KEY, context.serialize(src.chunkSize));
obj.add(DATA_TYPE_KEY, context.serialize(src.dataType));
final DataCodecInfo[] codecs = src.dataCodecInfos;
// length > 1 is actually invalid, but this is checked on construction
if (codecs.length == 0)
obj.add(COMPRESSION_KEY, context.serialize(new RawCompression()));
else
obj.add(COMPRESSION_KEY, context.serialize(codecs[0]));
return obj;
}
private static Compression getCompressionVersion0(final String compressionVersion0Name) {
switch (compressionVersion0Name) {
case "raw":
return new RawCompression();
case "gzip":
return new GzipCompression();
case "bzip2":
return new Bzip2Compression();
case "lz4":
return new Lz4Compression();
case "xz":
return new XzCompression();
}
return null;
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
public class DoubleArrayDataBlock extends AbstractDataBlock {
public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final double[] data) {
super(size, gridPosition, data, a -> a.length);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/FileKeyLockManager.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.WeakReference;
import java.nio.channels.FileLock;
import java.nio.file.Path;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static org.janelia.saalfeldlab.n5.LockingPolicy.STRICT;
/**
* Provides thread-safe and process-safe read/write locking for filesystem paths.
* Uses thread locks for JVM coordination and file locks for inter-process coordination.
*/
class FileKeyLockManager {
private static final Map managers = Collections.synchronizedMap(new EnumMap<>(LockingPolicy.class));
static FileKeyLockManager forPolicy(final LockingPolicy policy) {
return managers.computeIfAbsent(policy, FileKeyLockManager::new);
}
/**
* @deprecated use {@link FileKeyLockManager#forPolicy(LockingPolicy)}
*/
@Deprecated
static final FileKeyLockManager FILE_LOCK_MANAGER = forPolicy(STRICT);
private final LockingPolicy policy;
/**
* Create a new {@link FileKeyLockManager} with the specified locking policy.
*
* The given locking {@link LockingPolicy policy} applies to OS-level locking.
* For both the {@code STRICT} and {@code PERMISSIVE} policy, a {@link
* FileLock} is obtained. If this fails, {@code STRICT} will throw an {@code
* IOException}. {@code PERMISSIVE} will proceed without locking. {@code
* UNSAFE} will not attempt OS-level locking, however will still manage
* mutual exclusion of readers and writers in the same JVM. Trying to lock
* the same path with different locking policies will throw an {@code
* IOException}.
*
* @param policy
* the locking policy
*/
private FileKeyLockManager(final LockingPolicy policy) {
this.policy = policy;
}
private final ConcurrentHashMap locks = new ConcurrentHashMap<>();
private final ReferenceQueue refQueue = new ReferenceQueue<>();
private static class WeakValue extends WeakReference {
final String key;
WeakValue(
final String key,
final KeyLockState value,
final ReferenceQueue queue) {
super(value, queue);
this.key = key;
}
}
/**
* Remove entries from the cache whose references have been
* garbage-collected.
*/
private void cleanUp()
{
while (true) {
final WeakValue ref = (WeakValue) refQueue.poll();
if (ref == null)
break;
locks.remove(ref.key, ref);
}
}
private KeyLockState keyLockState(final Path path, final LockingPolicy policy) throws IOException {
final String key = path.toAbsolutePath().toString();
cleanUp();
final WeakValue existingRef = locks.get(key);
KeyLockState state = existingRef == null ? null : existingRef.get();
if (state != null) {
return state;
}
final KeyLockState newState = new KeyLockState(path, policy);
while (state == null) {
final WeakValue ref = locks.compute(key,
(k, v) -> (v != null && v.get() != null)
? v
: new WeakValue(k, newState, refQueue));
state = ref.get();
}
return state;
}
/**
* Acquires a read lock for the specified key. Multiple threads can hold
* read locks for the same key simultaneously.
*
* The first reader will acquire a shared file lock. Subsequent readers
* only acquire the thread-level lock.
*
* @param path
* the key (file path) to lock for reading
*
* @return a {@link LockedChannel} that must be closed when done
*
* @throws IOException
* if acquiring the file lock fails
*/
public LockedFileChannel lockForReading(final Path path) throws IOException {
return keyLockState(path, policy).acquireRead();
}
/**
* Acquires a write lock for the specified key. Only one thread can hold a
* write lock for a key at a time, and no readers can hold locks.
*
* @param path
* the file path to lock for writing
*
* @return a {@link LockedChannel} that must be closed when done
*
* @throws IOException
* if acquiring the file lock fails
*/
public LockedFileChannel lockForWriting(final Path path) throws IOException {
return keyLockState(path, policy).acquireWrite();
}
/**
* Returns the number of keys currently being tracked.
*
* @return the number of keys with associated locks
*/
int size() {
return locks.size();
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileAttribute;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.stream.Stream;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
/**
* Filesystem {@link KeyValueAccess}.
*
* @author Stephan Saalfeld
* @author Igor Pisarev
* @author Philipp Hanslovsky
*/
public class FileSystemKeyValueAccess implements KeyValueAccess {
private final FileKeyLockManager fileKeyLockManager;
public FileSystemKeyValueAccess() {
final LockingPolicy policy = LockingPolicy.fromString(System.getProperty("n5.ioPolicy", "permissive"));
this.fileKeyLockManager = FileKeyLockManager.forPolicy(policy);
}
private LockedFileChannel lockForReading(final Path path) throws N5IOException {
try {
return fileKeyLockManager.lockForReading(path);
} catch (final NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to lock file for reading: " + path, e);
}
}
private LockedFileChannel lockForWriting(final Path path) throws N5IOException {
try {
return fileKeyLockManager.lockForWriting(path);
} catch (final NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to lock file for writing: " + path, e);
}
}
@Override
public VolatileReadData createReadData(final String normalPath) {
return VolatileReadData.from(new FileLazyRead(Paths.get(normalPath)));
}
@Override
public void write(final String normalPath, final ReadData data) throws N5IOException {
final Path path = Paths.get(normalPath);
try (final LockedFileChannel channel = lockForWriting(path)) {
data.writeTo(channel.asOutputStream());
} catch (IOException e) {
throw new N5IOException(e);
}
}
@Override
public boolean isDirectory(final String normalPath) {
final Path path = Paths.get(normalPath);
return Files.isDirectory(path);
}
@Override
public boolean isFile(final String normalPath) {
final Path path = Paths.get(normalPath);
return Files.isRegularFile(path);
}
@Override
public boolean exists(final String normalPath) {
final Path path = Paths.get(normalPath);
return Files.exists(path);
}
@Override
public long size(final String normalPath) {
return size(Paths.get(normalPath));
}
private static long size(final Path path) {
try {
return Files.size(path);
} catch (NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException(e);
}
}
@Override
public String[] listDirectories(final String normalPath) throws N5IOException {
final Path path = Paths.get(normalPath);
try (final Stream pathStream = Files.list(path)) {
return pathStream
.filter(Files::isDirectory)
.map(a -> path.relativize(a).toString())
.toArray(String[]::new);
} catch (NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to list directories", e);
}
}
@Override
public String[] list(final String normalPath) throws N5IOException {
final Path path = Paths.get(normalPath);
try (final Stream pathStream = Files.list(path)) {
return pathStream
.map(a -> path.relativize(a).toString())
.toArray(String[]::new);
} catch (NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to list files", e);
}
}
@Override
public String[] components(final String path) {
final Path fsPath = Paths.get(path);
final Path root = fsPath.getRoot();
final String separator = fsPath.getFileSystem().getSeparator();
final String[] components;
int o;
if (root == null) {
components = new String[fsPath.getNameCount()];
o = 0;
} else {
components = new String[fsPath.getNameCount() + 1];
components[0] = root.toString();
o = 1;
}
for (int i = o; i < components.length; ++i) {
String name = fsPath.getName(i - o).toString();
/* Preserve trailing slash on final component if present*/
if (i == components.length - 1) {
final String trailingSeparator = path.endsWith(separator) ? separator : path.endsWith("/") ? "/" : "";
name += trailingSeparator;
}
components[i] = name;
}
return components;
}
@Override
public String parent(final String path) {
final Path parent = Paths.get(path).getParent();
if (parent == null)
return null;
else
return parent.toString();
}
@Override
public String relativize(final String path, final String base) {
final Path basePath = Paths.get(base);
return basePath.relativize(Paths.get(path)).toString();
}
/**
* Returns a normalized path. It ensures correctness on both Unix and
* Windows,
* otherwise {@code pathName} is treated as UNC path on Windows, and
* {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}.
*
* @param path the path
* @return the normalized path, without leading slash
*/
@Override
public String normalize(final String path) {
return Paths.get(path).normalize().toString();
}
@Override
public URI uri(final String normalPath) throws URISyntaxException {
// normalize make absolute the scheme specific part only
try {
final URI normalUri = URI.create(normalPath);
if (normalUri.isAbsolute()) return normalUri.normalize();
} catch (final IllegalArgumentException e) {
return new File(normalPath).toURI().normalize();
}
return new File(normalPath).toURI().normalize();
}
@Override
public String compose(final String... components) {
if (components == null || components.length == 0)
return null;
if (components.length == 1)
return Paths.get(components[0]).toString();
return Paths.get(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString();
}
@Override
public String compose(URI uri, String... components) {
Path composedPath;
if (uri.isAbsolute())
composedPath = Paths.get(uri);
else
composedPath = Paths.get(uri.toString());
for (String component : components) {
if (component == null || component.isEmpty())
continue;
composedPath = composedPath.resolve(component);
}
return composedPath.toAbsolutePath().toString();
}
@Override
public void createDirectories(final String normalPath) throws N5IOException {
try {
createDirectories(Paths.get(normalPath));
} catch (NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to create directories", e);
}
}
@Override
public void delete(final String normalPath) throws N5IOException {
try {
final Path path = Paths.get(normalPath);
if (Files.isRegularFile(path))
try (final LockedFileChannel channel = lockForWriting(path)) {
Files.delete(path);
}
else {
try (final Stream pathStream = Files.walk(path)) {
for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) {
final Path childPath = i.next();
if (Files.isRegularFile(childPath))
try (final LockedFileChannel channel = lockForWriting(childPath)) {
Files.delete(childPath);
}
else
tryDelete(childPath);
}
}
}
} catch (NoSuchFileException ignore) {
/* It doesn't exist; that's sufficient for us to not complain on a `delete` call */
} catch (IOException | UncheckedIOException e) {
throw new N5IOException("Failed to delete file at " + normalPath, e);
}
}
protected static void tryDelete(final Path path) throws IOException {
try {
Files.delete(path);
} catch (final DirectoryNotEmptyException e) {
/*
* Even though path is expected to be an empty directory, sometimes
* deletion fails on network filesystems when lock files are not
* cleared immediately after the leaves have been removed.
*/
try {
/* wait and reattempt */
Thread.sleep(100);
Files.delete(path);
} catch (final InterruptedException ex) {
e.printStackTrace();
Thread.currentThread().interrupt();
}
}
}
/**
* This is a copy of {@link Files#createDirectories(Path, FileAttribute...)}
* that follows symlinks.
*
* Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464
*
* Creates a directory by creating all nonexistent parent directories first.
* Unlike the {@link Files#createDirectories} method, an exception
* is not thrown if the directory could not be created because it already
* exists.
*
*
* The {@code attrs} parameter is optional {@link FileAttribute
* file-attributes} to set atomically when creating the nonexistent
* directories. Each file attribute is identified by its {@link
* FileAttribute#name name}. If more than one attribute of the same name is
* included in the array then all but the last occurrence is ignored.
*
*
* If this method fails, then it may do so after creating some, but not
* all, of the parent directories.
*
* @param dir
* the directory to create
*
* @param attrs
* an optional list of file attributes to set atomically when
* creating the directory
*
* @return the directory
*
* @throws UnsupportedOperationException
* if the array contains an attribute that cannot be set
* atomically
* when creating the directory
* @throws FileAlreadyExistsException
* if {@code dir} exists but is not a directory (optional
* specific
* exception)
* @throws IOException
* if an I/O error occurs
* @throws SecurityException
* in the case of the default provider, and a security manager
* is
* installed, the {@link SecurityManager#checkWrite(String)
* checkWrite}
* method is invoked prior to attempting to create a directory
* and
* its {@link SecurityManager#checkRead(String) checkRead} is
* invoked for each parent directory that is checked. If {@code
* dir} is not an absolute path then its {@link Path#toAbsolutePath
* toAbsolutePath} may need to be invoked to get its absolute
* path.
* This may invoke the security manager's {@link
* SecurityManager#checkPropertyAccess(String)
* checkPropertyAccess}
* method to check access to the system property
* {@code user.dir}
*/
protected static Path createDirectories(Path dir, final FileAttribute... attrs) throws IOException {
// attempt to create the directory
try {
createAndCheckIsDirectory(dir, attrs);
return dir;
} catch (final FileAlreadyExistsException x) {
// file exists and is not a directory
throw x;
} catch (final IOException x) {
// parent may not exist or other reason
}
SecurityException se = null;
try {
dir = dir.toAbsolutePath();
} catch (final SecurityException x) {
// don't have permission to get absolute path
se = x;
}
// find a descendant that exists
Path parent = dir.getParent();
while (parent != null) {
try {
parent.getFileSystem().provider().checkAccess(parent);
break;
} catch (final NoSuchFileException x) {
// does not exist
}
parent = parent.getParent();
}
if (parent == null) {
// unable to find existing parent
if (se == null) {
throw new FileSystemException(
dir.toString(),
null,
"Unable to determine if root directory exists");
} else {
throw se;
}
}
// create directories
Path child = parent;
for (final Path name : parent.relativize(dir)) {
child = child.resolve(name);
createAndCheckIsDirectory(child, attrs);
}
return dir;
}
/**
* This is a copy of a previous Files#createAndCheckIsDirectory(Path,
* FileAttribute...) method that follows symlinks.
*
* Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464
*
* Used by createDirectories to attempt to create a directory. A no-op if the
* directory already exists.
*
* @param dir directory path
* @param attrs file attributes
* @throws IOException the exception
*/
protected static void createAndCheckIsDirectory(
final Path dir,
final FileAttribute... attrs) throws IOException {
try {
Files.createDirectory(dir, attrs);
} catch (final FileAlreadyExistsException x) {
if (!Files.isDirectory(dir))
throw x;
}
}
/**
* Verify that the range {@code [offset, offset+length)} is fully contained in {@code [0, channelSize)}.
*
* @throws IndexOutOfBoundsException
* if range is not fully contained
*/
private static void validBounds(final long channelSize, final long offset, final long length) throws IndexOutOfBoundsException {
if (offset < 0)
throw new IndexOutOfBoundsException("offset must be > 0, but was: " + offset);
else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay
throw new IndexOutOfBoundsException("offset (" + offset + ") must be less than channel size (" + channelSize + ")");
else if (length >= 0 && offset + length > channelSize)
throw new IndexOutOfBoundsException("offset + length (" + (offset + length) + ") must be less than channel size (" + channelSize + ")");
}
private class FileLazyRead implements LazyRead {
private final Path path;
private LockedFileChannel lock; // TODO rename
FileLazyRead(final Path path) {
this.path = path;
lock = lockForReading(path);
}
@Override
public long size() throws N5IOException {
if (lock == null) {
throw new N5IOException("FileLazyRead is already closed.");
}
try {
return Files.size(path);
} catch (NoSuchFileException e) {
throw new N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5IOException(e);
}
}
@Override
public ReadData materialize(final long offset, final long length) {
if (lock == null) {
throw new N5IOException("FileLazyRead is already closed.");
}
try {
final long channelSize = lock.size();
validBounds(channelSize, offset, length);
final long size = length < 0 ? (channelSize - offset) : length;
if (size > Integer.MAX_VALUE) {
throw new IndexOutOfBoundsException("Attempt to materialize too large data");
}
final byte[] data = new byte[(int) size];
lock.read(ByteBuffer.wrap(data), offset);
return ReadData.from(data);
} catch (IOException | UncheckedIOException e) {
throw new N5Exception.N5IOException(e);
}
}
@Override
public void close() throws IOException {
if (lock != null) {
lock.close();
lock = null;
}
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
public class FloatArrayDataBlock extends AbstractDataBlock {
public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) {
super(size, gridPosition, data, a -> a.length);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.*;
import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER;
public class FsIoPolicy {
static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe());
private static boolean validBounds(long channelSize, long offset, long length) {
if (offset < 0)
throw new N5Exception("offset must be > 0, but was: " + offset);
else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay
throw new N5Exception("offset (" + offset + ") must be less than channel size (" + channelSize + ")");
else if (length >= 0 && offset + length > channelSize)
throw new N5Exception("offset + length (" + (offset + length) + ") must be less than channel size (" + channelSize + ")");
return true;
}
/**
* Opens a file channel. If the channel is opened {@code forWriting},
* then this may create the file and the parent directories as needed.
*
* @throws IOException
* if the channel cannot be opened
*/
static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException {
if (forWriting) {
final Path parent = path.getParent();
/* if not null and not directory, it will call `createDirectories` but we expect it to throw an IOException */
if (parent != null && !parent.toFile().isDirectory()) {
Files.createDirectories(parent);
}
return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
} else {
return FileChannel.open(path, StandardOpenOption.READ);
}
}
/**
* This method is necessary to handle the situtation where writing is successful, but `close` fails on the file channel.
* This has been observed to happen fairly consistently on MacOS when writing to a file mounted over SMB.
*
* @param readData to write to the {@code Path}
* @param path to write to
* @throws IOException if writing failed.
*/
private static void writeToPathIgnoreCloseException(ReadData readData, Path path) throws IOException {
FileChannel channel = openFileChannel(path, true);
OutputStream os = Channels.newOutputStream(channel);
try {
readData.writeTo(os);
os.flush();
channel.force(true);
} catch (Throwable e) {
os.close();
channel.close();
throw e;
}
/* if we get here, the write succeeded, and the os/channel may not be closed yet */
try {
os.close();
channel.close();
} catch (IOException | UncheckedIOException ignore) {
/* Ignore; we know the data was written already. */
}
}
public static class Unsafe implements IoPolicy {
@Override
public void write(String key, ReadData readData) throws IOException {
final Path path = Paths.get(key);
writeToPathIgnoreCloseException(readData, path);
}
@Override
public VolatileReadData read(final String key) throws IOException {
final Path path = Paths.get(key);
FileLazyRead fileLazyRead = new FileLazyRead(path, false);
return VolatileReadData.from(fileLazyRead);
}
@Override
public void delete(final String key) throws IOException {
final Path path = Paths.get(key);
Files.deleteIfExists(path);
}
}
public static class Atomic implements IoPolicy {
@Override
public void write(String key, ReadData readData) throws IOException {
final Path path = Paths.get(key);
try (LockedFileChannel channel = FILE_LOCK_MANAGER.lockForWriting(path)) {
readData.writeTo(channel.asOutputStream());
}
}
@Override
public VolatileReadData read(String key) throws IOException {
final Path path = Paths.get(key);
FileLazyRead fileLazyRead = new FileLazyRead(path, true);
return VolatileReadData.from(fileLazyRead);
}
@Override
public void delete(final String key) throws IOException {
final Path path = Paths.get(key);
if (!Files.isRegularFile(path))
Files.delete(path);
try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) {
Files.delete(path);
}
}
}
static class FileLazyRead implements LazyRead {
private static final Closeable NO_OP = () -> { };
private final Path path;
private Closeable lock;
FileLazyRead(final Path path) throws IOException {
this(path, true);
}
FileLazyRead(final Path path, final boolean requireLock ) throws IOException {
this.path = path;
if (requireLock)
lock = FILE_LOCK_MANAGER.lockForReading(path);
else
lock = NO_OP;
}
@Override
public long size() throws N5Exception.N5IOException {
if (lock == null) {
throw new N5Exception.N5IOException("FileLazyRead is already closed.");
}
try {
return Files.size(path);
} catch (NoSuchFileException e) {
throw new N5Exception.N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
throw new N5Exception.N5IOException(e);
}
}
@Override
public ReadData materialize(final long offset, final long length) {
if (lock == null) {
throw new N5Exception.N5IOException("FileLazyRead is already closed.");
}
ReadData readData = null;
try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) {
channel.position(offset);
final long channelSize = channel.size();
if (!validBounds(channelSize, offset, length)) {
throw new IndexOutOfBoundsException();
}
final long size = length < 0 ? (channelSize - offset) : length;
if (size > Integer.MAX_VALUE) {
throw new IndexOutOfBoundsException("Attempt to materialize too large data");
}
final byte[] data = new byte[(int) size];
final ByteBuffer buf = ByteBuffer.wrap(data);
channel.read(buf);
readData = ReadData.from(data);
} catch (final NoSuchFileException e) {
throw new N5Exception.N5NoSuchKeyException("No such file", e);
} catch (IOException | UncheckedIOException e) {
/* Occasionally (frequently for some source remote mounted file systems) this can throw exceptions during
* `channel.close()` which is called automatically in the try-with-resources block. In this case, we have
* successfully read the data, and we can return it, and ignore the exception.
* */
if (readData == null)
throw new N5Exception.N5IOException(e);
}
return readData;
}
@Override
public void close() throws IOException {
if (lock != null) {
lock.close();
lock = null;
}
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.util.List;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
import org.janelia.saalfeldlab.n5.shard.PositionValueAccess;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
/**
* {@link N5Reader} implementation through {@link KeyValueAccess} with JSON
* attributes parsed with {@link Gson}.
*
*/
public interface GsonKeyValueN5Reader extends GsonN5Reader {
KeyValueAccess getKeyValueAccess();
default boolean groupExists(final String normalPath) {
return getKeyValueAccess().isDirectory(absoluteGroupPath(normalPath));
}
@Override
default boolean exists(final String pathName) {
final String normalPath = N5URI.normalizeGroupPath(pathName);
return groupExists(normalPath) || datasetExists(normalPath);
}
@Override
default boolean datasetExists(final String pathName) throws N5Exception {
// for n5, every dataset must be a group
return getDatasetAttributes(pathName) != null;
}
/**
* Reads or creates the attributes map of a group or dataset.
*
* @param pathName
* group path
* @return the attribute
* @throws N5Exception if the attributes cannot be read
*/
@Override
default JsonElement getAttributes(final String pathName) throws N5Exception {
final String groupPath = N5URI.normalizeGroupPath(pathName);
final String attributesPath = absoluteAttributesPath(groupPath);
try (final VolatileReadData readData = getKeyValueAccess().createReadData(attributesPath);) {
if (readData == null) {
return null;
}
return GsonUtils.readAttributes(new InputStreamReader(readData.inputStream()), getGson());
} catch (final N5Exception.N5NoSuchKeyException e) {
return null;
} catch (final UncheckedIOException | N5IOException e) {
throw new N5IOException("Failed to read attributes from dataset " + pathName, e);
}
}
@Override
default DataBlock readChunk(
final String pathName,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),
convertedDatasetAttributes);
return convertedDatasetAttributes. getDatasetAccess().readChunk(posKva, gridPosition);
} catch (N5Exception.N5NoSuchKeyException e) {
return null;
}
}
@Override
default List> readChunks(
final String pathName,
final DatasetAttributes datasetAttributes,
final List blockPositions) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes);
return convertedDatasetAttributes. getDatasetAccess().readChunks(posKva, blockPositions);
}
@Override
default DataBlock readBlock(
final String pathName,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),
convertedDatasetAttributes);
return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition, shardLevel);
} catch (N5Exception.N5NoSuchKeyException e) {
return null;
}
}
@Override
default String[] list(final String pathName) throws N5Exception {
return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName));
}
/**
* Constructs the absolute path (in terms of this store) for the group or
* dataset.
*
* @param normalGroupPath
* normalized group path without leading slash
* @return the absolute path to the group
*/
default String absoluteGroupPath(final String normalGroupPath) {
return getKeyValueAccess().compose(getURI(), normalGroupPath);
}
/**
* Constructs the absolute path (in terms of this store) for the attributes
* file of a group or dataset.
*
* @param normalPath
* normalized group path without leading slash
* @return the absolute path to the attributes
*/
default String absoluteAttributesPath(final String normalPath) {
return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());
}
@Override
default boolean blockExists(
final String pathName,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(pathName);
final String blockPath = getKeyValueAccess().compose(getURI(), normalPath,
datasetAttributes.relativeBlockPath(gridPosition));
return getKeyValueAccess().isFile(blockPath);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import com.google.gson.JsonSyntaxException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.shard.PositionValueAccess;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
/**
* Default implementation of {@link N5Writer} with JSON attributes parsed with
* {@link Gson}.
*/
public interface GsonKeyValueN5Writer extends GsonN5Writer, GsonKeyValueN5Reader {
/**
* TODO This overrides the version even if incompatible, check
* if this is the desired behavior or if it is always overridden, e.g. as by
* the caching version. If this is true, delete this implementation.
*
* @param path to the group to write the version into
*/
default void setVersion(final String path) {
if (!VERSION.equals(getVersion()))
setAttribute("/", VERSION_KEY, VERSION.toString());
}
static String initializeContainer(
final KeyValueAccess keyValueAccess,
final String basePath) throws N5IOException {
final String normBasePath = keyValueAccess.normalize(basePath);
keyValueAccess.createDirectories(normBasePath);
return normBasePath;
}
@Override
default void createGroup(final String path) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(path);
getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));
}
/**
* Helper method that writes an attributes tree into the store
*
* TODO This method is not part of the public API and should be protected
* in Java versions greater than 8
*
* @param normalGroupPath
* to write the attributes to
* @param attributes
* to write
* @throws N5Exception
* if unable to write the attributes at {@code normalGroupPath}
*/
default void writeAttributes(
final String normalGroupPath,
final JsonElement attributes) throws N5Exception {
final ReadData newAttributesReadData = ReadData.from(os -> {
final OutputStreamWriter writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
GsonUtils.writeAttributes(writer, attributes, getGson());
});
try {
getKeyValueAccess().write(absoluteAttributesPath(normalGroupPath), newAttributesReadData);
} catch (UncheckedIOException | N5IOException e) {
throw new N5Exception.N5IOException("Failed to write attributes into " + normalGroupPath, e);
}
}
@Override
default void setAttributes(
final String path,
final JsonElement attributes) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(path);
if (!exists(normalPath))
throw new N5IOException("" + normalPath + " is not a group or dataset.");
writeAttributes(normalPath, attributes);
}
/**
* Helper method that reads the existing map of attributes, JSON encodes,
* inserts and overrides the provided attributes, and writes them back into
* the attributes store.
*
* TODO This method is not part of the public API and should be protected
* in Java greater than 8
*
* @param normalGroupPath
* to write the attributes to
* @param attributes
* to write
* @throws N5Exception
* if unable to read or write the attributes at
* {@code normalGroupPath}
*/
default void writeAttributes(
final String normalGroupPath,
final Map attributes) throws N5Exception {
if (attributes != null && !attributes.isEmpty()) {
JsonElement root = getAttributes(normalGroupPath);
root = root != null && root.isJsonObject()
? root.getAsJsonObject()
: new JsonObject();
root = GsonUtils.insertAttributes(root, attributes, getGson());
writeAttributes(normalGroupPath, root);
}
}
@Override
default void setAttributes(
final String path,
final Map attributes) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(path);
if (!exists(normalPath))
throw new N5IOException("" + normalPath + " is not a group or dataset.");
writeAttributes(normalPath, attributes);
}
@Override
default boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(groupPath);
final String absoluteNormalPath = getKeyValueAccess().compose(getURI(), normalPath);
final String normalKey = N5URI.normalizeAttributePath(attributePath);
if (!getKeyValueAccess().isDirectory(absoluteNormalPath))
return false;
if (attributePath.equals("/")) {
setAttributes(normalPath, JsonNull.INSTANCE);
return true;
}
final JsonElement attributes = getAttributes(normalPath);
if (GsonUtils.removeAttribute(attributes, normalKey) != null) {
setAttributes(normalPath, attributes);
return true;
}
return false;
}
@Override
default T removeAttribute(final String pathName, final String key, final Class cls) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(pathName);
final String normalKey = N5URI.normalizeAttributePath(key);
final JsonElement attributes = getAttributes(normalPath);
final T obj;
try {
obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson());
} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {
throw new N5Exception.N5ClassCastException(e);
}
if (obj != null) {
setAttributes(normalPath, attributes);
}
return obj;
}
@Override
default boolean removeAttributes(final String pathName, final List attributes) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(pathName);
boolean removed = false;
for (final String attribute : attributes) {
final String normalKey = N5URI.normalizeAttributePath(attribute);
removed |= removeAttribute(normalPath, normalKey);
}
return removed;
}
@Override
default void writeRegion(
final String datasetPath,
final DatasetAttributes datasetAttributes,
final long[] min,
final long[] size,
final DataBlockSupplier chunkSupplier,
final boolean writeFully) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);
convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully);
} catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write blocks into dataset " + datasetPath, e);
}
}
@Override
default void writeRegion(
final String datasetPath,
final DatasetAttributes datasetAttributes,
final long[] min,
final long[] size,
final DataBlockSupplier chunkSupplier,
final boolean writeFully,
final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);
convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully, exec);
} catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write blocks into dataset " + datasetPath, e);
}
}
@Override
default void writeChunks(
final String datasetPath,
final DatasetAttributes datasetAttributes,
final DataBlock... chunks) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes);
convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(chunks));
} catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write chunks into dataset " + datasetPath, e);
}
}
@Override
default void writeChunk(
final String path,
final DatasetAttributes datasetAttributes,
final DataBlock chunk) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes);
convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, chunk);
} catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write chunk " + Arrays.toString(chunk.getGridPosition()) + " into dataset " + path,
e);
}
}
@Override
default void writeBlock(
final String path,
final DatasetAttributes datasetAttributes,
final DataBlock dataBlock) throws N5Exception {
final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;
try {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes);
convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock, shardLevel);
} catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path,
e);
}
}
@Override
default boolean remove(final String path) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(path);
final String groupPath = absoluteGroupPath(normalPath);
if (getKeyValueAccess().isDirectory(groupPath))
getKeyValueAccess().delete(groupPath);
/* an IOException should have occurred if anything had failed midway */
return true;
}
@Override
default boolean deleteBlock(
final String path,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
return posKva.remove(gridPosition);
}
@Override
default boolean deleteChunk(
final String path,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
return datasetAttributes.getDatasetAccess().deleteChunk(posKva, gridPosition);
}
@Override
default boolean deleteChunks(
final String path,
final DatasetAttributes datasetAttributes,
final List gridPositions) throws N5Exception {
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
return datasetAttributes.getDatasetAccess().deleteChunks(posKva, gridPositions);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java
================================================
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Type;
import java.util.Map;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonParseException;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonSyntaxException;
/**
* {@link N5Reader} with JSON attributes parsed with {@link Gson}.
*
*/
public interface GsonN5Reader extends N5Reader {
Gson getGson();
/**
* Get the key for the {@link KeyValueAccess}, that is used for storing attributes.
* The N5 format uses "attributes.json".
*
* @return the attributes key
*/
String getAttributesKey();
@Override
default Map> listAttributes(final String pathName) throws N5Exception {
return GsonUtils.listAttributes(getAttributes(pathName));
}
@Override
default DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(pathName);
final JsonElement attributes = getAttributes(normalPath);
return createDatasetAttributes(attributes);
}
default DatasetAttributes createDatasetAttributes(final JsonElement attributes) {
final JsonDeserializationContext context = new JsonDeserializationContext() {
@Override public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException {
return getGson().fromJson(json, typeOfT);
}
};
return DatasetAttributes.getJsonAdapter().deserialize(attributes, DatasetAttributes.class, context);
}
@Override
default T getAttribute(final String pathName, final String key, final Class clazz) throws N5Exception {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
final JsonElement attributes = getAttributes(normalPathName);
try {
return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson());
} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {
throw new N5Exception.N5ClassCastException(e);
}
}
@Override
default T getAttribute(final String pathName, final String key, final Type type) throws N5Exception {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
final JsonElement attributes = getAttributes(normalPathName);
try {
return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson());
} catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {
throw new N5Exception.N5ClassCastException(e);
}
}
/**
* Reads or the attributes of a group or dataset.
*
* @param pathName
* group path
* @return the attributes identified by pathName
* @throws N5Exception if the attribute cannot be returned
*/
JsonElement getAttributes(final String pathName) throws N5Exception;
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java
================================================
package org.janelia.saalfeldlab.n5;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
/**
* {@link N5Writer} with JSON attributes parsed with {@link Gson}.
*
*/
public interface GsonN5Writer extends GsonN5Reader, N5Writer {
/**
* Set the attributes of a group. This result of this method is equivalent
* with {@link N5Writer#setAttribute(String, String, Object) N5Writer#setAttribute(groupPath, "/", attributes)}.
*
* @param groupPath
* to write the attributes to
* @param attributes
* to write
* @throws N5Exception if the attributes cannot be set
*/
void setAttributes(
final String groupPath,
final JsonElement attributes) throws N5Exception;
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException;
/**
* Utility class for working with JSON.
*
* @author Stephan Saalfeld
*/
public interface GsonUtils {
/**
* Reads the attributes json from a given {@link Reader}.
*
* @param reader
* the reader
* @param gson
* to parse Json from the {@code reader}
* @return the root {@link JsonObject} of the attributes
*/
static JsonElement readAttributes(final Reader reader, final Gson gson) {
return gson.fromJson(reader, JsonElement.class);
}
static T readAttribute(
final JsonElement root,
final String normalizedAttributePath,
final Class cls,
final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {
return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson);
}
static T readAttribute(
final JsonElement root,
final String normalizedAttributePath,
final Type type,
final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {
final JsonElement attribute = getAttribute(root, normalizedAttributePath);
return parseAttributeElement(attribute, gson, type);
}
/**
* Deserialize the {@code attribute} as {@link Type type} {@code T}.
*
* @param attribute
* to deserialize as {@link Type type}
* @param gson
* used to deserialize {@code attribute}
* @param type
* to deserialize {@code attribute} as
* @param
* return type represented by {@link Type type}
* @return the deserialized attribute object, or {@code null} if
* {@code attribute} cannot deserialize to {@code T}
*/
@SuppressWarnings("unchecked")
static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) throws JsonSyntaxException, NumberFormatException, ClassCastException {
if (attribute == null)
return null;
final Class clazz = (type instanceof Class) ? ((Class)type) : null;
if (clazz != null && clazz.isAssignableFrom(HashMap.class)) {
final Type mapType = new TypeToken>() {
}.getType();
final Map retMap = gson.fromJson(attribute, mapType);
// noinspection unchecked
return (T)retMap;
}
if (attribute instanceof JsonArray) {
final JsonArray array = attribute.getAsJsonArray();
try {
final T retArray = GsonUtils.getJsonAsArray(gson, array, type);
if (retArray != null)
return retArray;
} catch (final JsonSyntaxException e) {
if (type == String.class)
//noinspection unchecked
return (T)gson.toJson(attribute);
}
}
try {
final T parsedResult = gson.fromJson(attribute, type);
if (parsedResult == null)
throw new ClassCastException("Cannot parse json as type " + type.getTypeName());
return parsedResult;
} catch (final JsonSyntaxException e) {
if (type == String.class)
//noinspection unchecked
return (T)gson.toJson(attribute);
throw e;
}
}
/**
* Return the attribute at {@code normalizedAttributePath} as a
* {@link JsonElement}. Does not attempt to parse the attribute.
* to search for the {@link JsonElement} at location
* {@code normalizedAttributePath}
*
* @param root
* containing an attribute at normalizedAttributePath
* @param normalizedAttributePath
* to the attribute
* @return the attribute as a {@link JsonElement}.
*/
static JsonElement getAttribute(JsonElement root, final String normalizedAttributePath) {
final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) {
return null;
}
root = jsonArray.get(index);
} else {
return null;
}
}
}
return root;
}
/**
* Best effort implementation of {@link N5Reader#listAttributes(String)}
* with limited type resolution. Possible return types are
*
* - null
* - boolean
* - double
* - String
* - Object
* - boolean[]
* - double[]
* - String[]
* - Object[]
*
*
* @param root
* the json element
* @return the attribute map
*/
static Map> listAttributes(final JsonElement root) throws N5JsonParseException {
if (root == null) {
return new HashMap<>();
}
if (!root.isJsonObject()) {
throw new N5JsonParseException("JsonElement found, but was not JsonObject");
}
final HashMap> attributes = new HashMap<>();
root.getAsJsonObject().entrySet().forEach(entry -> {
final Class clazz;
final String key = entry.getKey();
final JsonElement jsonElement = entry.getValue();
if (jsonElement.isJsonNull())
clazz = null;
else if (jsonElement.isJsonPrimitive())
clazz = classForJsonPrimitive((JsonPrimitive)jsonElement);
else if (jsonElement.isJsonArray()) {
final JsonArray jsonArray = (JsonArray)jsonElement;
Class arrayElementClass = Object.class;
if (jsonArray.size() > 0) {
final JsonElement firstElement = jsonArray.get(0);
if (firstElement.isJsonPrimitive()) {
arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive());
for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) {
final JsonElement element = jsonArray.get(i);
if (element.isJsonPrimitive()) {
final Class nextArrayElementClass = classForJsonPrimitive(
element.getAsJsonPrimitive());
if (nextArrayElementClass != arrayElementClass)
if (nextArrayElementClass == double.class && arrayElementClass == long.class)
arrayElementClass = double.class;
else {
arrayElementClass = Object.class;
break;
}
} else {
arrayElementClass = Object.class;
break;
}
}
}
clazz = Array.newInstance(arrayElementClass, 0).getClass();
} else
clazz = Object[].class;
} else
clazz = Object.class;
attributes.put(key, clazz);
});
return attributes;
}
static T getJsonAsArray(final Gson gson, final JsonArray array, final Class cls) {
return getJsonAsArray(gson, array, TypeToken.get(cls).getType());
}
@SuppressWarnings("unchecked")
static T getJsonAsArray(final Gson gson, final JsonArray array, final Type type) {
final Class clazz = (type instanceof Class) ? ((Class)type) : null;
if (type == boolean[].class) {
final boolean[] retArray = new boolean[array.size()];
for (int i = 0; i < array.size(); i++) {
final Boolean value = gson.fromJson(array.get(i), boolean.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == double[].class) {
final double[] retArray = new double[array.size()];
for (int i = 0; i < array.size(); i++) {
final double value = gson.fromJson(array.get(i), double.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == float[].class) {
final float[] retArray = new float[array.size()];
for (int i = 0; i < array.size(); i++) {
final float value = gson.fromJson(array.get(i), float.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == long[].class) {
final long[] retArray = new long[array.size()];
for (int i = 0; i < array.size(); i++) {
final long value = gson.fromJson(array.get(i), long.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == short[].class) {
final short[] retArray = new short[array.size()];
for (int i = 0; i < array.size(); i++) {
final short value = gson.fromJson(array.get(i), short.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == int[].class) {
final int[] retArray = new int[array.size()];
for (int i = 0; i < array.size(); i++) {
final int value = gson.fromJson(array.get(i), int.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == byte[].class) {
final byte[] retArray = new byte[array.size()];
for (int i = 0; i < array.size(); i++) {
final byte value = gson.fromJson(array.get(i), byte.class);
retArray[i] = value;
}
return (T)retArray;
} else if (type == char[].class) {
final char[] retArray = new char[array.size()];
for (int i = 0; i < array.size(); i++) {
final char value = gson.fromJson(array.get(i), char.class);
retArray[i] = value;
}
return (T)retArray;
} else if (clazz != null && clazz.isArray()) {
final Class componentCls = clazz.getComponentType();
final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size());
for (int i = 0; i < array.size(); i++) {
clsArray[i] = gson.fromJson(array.get(i), componentCls);
}
// noinspection unchecked
return (T)clsArray;
}
return null;
}
/**
* Return a reasonable class for a {@link JsonPrimitive}. Possible return
* types are
*
* - boolean
* - double
* - String
* - Object
*
*
* @param jsonPrimitive
* the json primitive
* @return the class
*/
static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) {
if (jsonPrimitive.isBoolean())
return boolean.class;
else if (jsonPrimitive.isNumber()) {
final Number number = jsonPrimitive.getAsNumber();
if (number.longValue() == number.doubleValue())
return long.class;
else
return double.class;
} else if (jsonPrimitive.isString())
return String.class;
else
return Object.class;
}
/**
* If there is an attribute in {@code root} such that it can be parsed and
* deserialized as {@code T},
* then remove it from {@code root}, write {@code root} to the
* {@code writer}, and return the removed attribute.
*
* If there is an attribute at the location specified by
* {@code normalizedAttributePath} but it cannot be deserialized to
* {@code T}, then it is not removed.
*
* If nothing is removed, then {@code root} is not written to the
* {@code writer}.
* to write the modified {@code root} to after removal of the attribute to
* remove the attribute from
*
*
* @param the type of the attribute to be removed
* @param writer to write the modified JsonElement, which no longer contains the removed attribute
* @param root element to remove the attribute from
* @param normalizedAttributePath
* to the attribute location of the attribute to remove to
* deserialize the attribute with of the removed attribute
* @param cls of the type of the attribute to be removed
* @param gson used to deserialize the JsonElement as {@code T}
* @return the removed attribute, or null if nothing removed
* @throws IOException
* if unable to write to the {@code writer}
*/
static T removeAttribute(
final Writer writer,
final JsonElement root,
final String normalizedAttributePath,
final Class cls,
final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException {
final T removed = removeAttribute(root, normalizedAttributePath, cls, gson);
if (removed != null) {
writeAttributes(writer, root, gson);
}
return removed;
}
/**
* If there is an attribute in {@code root} at location
* {@code normalizedAttributePath} then remove it from {@code root}..
* to write the modified {@code root} to after removal of the attribute to
* remove the attribute from
*
* @param writer to write the modified JsonElement, which no longer contains the removed attribute
* @param root to remove the attribute from
* @param normalizedAttributePath
* to the attribute location to deserialize the attribute with
* @param gson to write the attribute to the {@code writer}
* @return if the attribute was removed or not
* @throws IOException if an error occurs while writing to the {@code writer}
*/
static boolean removeAttribute(
final Writer writer,
final JsonElement root,
final String normalizedAttributePath,
final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException {
final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson);
if (removed != null) {
writeAttributes(writer, root, gson);
return true;
}
return false;
}
/**
* If there is an attribute in {@code root} such that it can be parsed and
* desrialized as {@code T},
* then remove it from {@code root} and return the removed attribute.
*
* If there is an attribute at the location specified by
* {@code normalizedAttributePath} but it cannot be deserialized to
* {@code T}, then it is not removed.
* to remove the attribute from
*
* @param the type of the attribute to be removed
* @param root element to remove the attribute from
* @param normalizedAttributePath
* to the attribute location of the attribute to remove to
* deserialize the attribute with of the removed attribute
* @param cls of the type of the attribute to be removed
* @param gson used to deserialize the JsonElement as {@code T}
* @return the removed attribute, or null if nothing removed
* @throws JsonSyntaxException if the attribute is not valid json
* @throws NumberFormatException if {@code T} is a {@link Number} but the attribute cannot be parsed as {@code T}
* @throws ClassCastException if an attribute exists at this path, but is not of type {@code T}
*/
static T removeAttribute(
final JsonElement root,
final String normalizedAttributePath,
final Class cls,
final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException {
final T attribute = GsonUtils.readAttribute(root, normalizedAttributePath, cls, gson);
if (attribute != null) {
removeAttribute(root, normalizedAttributePath);
}
return attribute;
}
/**
* Remove and return the attribute at {@code normalizedAttributePath} as a
* {@link JsonElement}. Does not attempt to parse the attribute. to search
* for the {@link JsonElement} at location {@code normalizedAttributePath}
*
* @param root
* the root JsonElement
* @param normalizedAttributePath
* to the attribute
* @return the attribute as a {@link JsonElement}.
*/
static JsonElement removeAttribute(JsonElement root, final String normalizedAttributePath) {
final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) {
return null;
}
root = jsonArray.get(index);
if (i == pathParts.length - 1) {
jsonArray.remove(index);
}
} else {
return null;
}
}
}
return root;
}
/**
* Inserts {@code attribute} into {@code root} at location
* {@code normalizedAttributePath} and write the resulting {@code root}.
*
* If {@code root} is not a {@link JsonObject}, then it is overwritten with
* an object containing {@code "normalizedAttributePath": attribute }
*
* @param writer
* the writer
* @param root
* the root json element
* @param normalizedAttributePath
* the attribute path
* @param attribute
* the attribute
* @param gson
* the gson
* @param
* the attribute type
* @throws IOException
* the exception
*/
static void writeAttribute(
final Writer writer,
JsonElement root,
final String normalizedAttributePath,
final T attribute,
final Gson gson) throws IOException {
root = insertAttribute(root, normalizedAttributePath, attribute, gson);
writeAttributes(writer, root, gson);
}
/**
* Writes the attributes JsonElement to a given {@link Writer}.
* This will overwrite any existing attributes.
*
* @param writer
* the writer
* @param root
* the root json element
* @param gson
* the gson
* @param
* the attribute type
* @throws IOException
* the exception
*/
static void writeAttributes(
final Writer writer,
final JsonElement root,
final Gson gson) throws IOException {
gson.toJson(root, writer);
writer.flush();
}
static JsonElement insertAttributes(JsonElement root, final Map attributes, final Gson gson) {
for (final Map.Entry attribute : attributes.entrySet()) {
root = insertAttribute(root, N5URI.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson);
}
return root;
}
static JsonElement insertAttribute(
JsonElement root,
final String normalizedAttributePath,
final T attribute,
final Gson gson) {
LinkedAttributePathToken pathToken = N5URI.getAttributePathTokens(normalizedAttributePath);
/* No path to traverse or build; just return the value */
if (pathToken == null)
return gson.toJsonTree(attribute);
JsonElement json = root;
while (pathToken != null) {
final JsonElement parent = pathToken.setAndCreateParentElement(json);
/*
* We may need to create or override the existing root if it is
* non-existent or incompatible.
*/
final boolean rootOverriden = json == root && parent != json;
if (root == null || rootOverriden) {
root = parent;
}
json = pathToken.writeChild(gson, attribute);
pathToken = pathToken.next();
}
return root;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipParameters;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@CompressionType("gzip")
@NameConfig.Name("gzip")
public class GzipCompression implements Compression {
private static final long serialVersionUID = 8630847239813334263L;
/**
* Explicit equivalent of {@link java.util.zip.Deflater#DEFAULT_COMPRESSION}: zlib defines
* level 6 as "a default compromise between speed and compression." An explicit value is used
* instead of {@code DEFAULT_COMPRESSION} (-1) because -1 is not a valid level for Zarr codecs.
*
* @see zlib Manual
*/
private static final int N5_DEFAULT_GZIP_LEVEL = 6;
@CompressionParameter
@NameConfig.Parameter
private final int level;
/**
* This is not a NameConfig.Parameter because this parameter must not be
* serialized for zarr
*/
@CompressionParameter
private final boolean useZlib;
private final transient GzipParameters parameters = new GzipParameters();
public GzipCompression() {
this(N5_DEFAULT_GZIP_LEVEL);
}
public GzipCompression(final int level) {
this(level, false);
}
public GzipCompression(final int level, final boolean useZlib) {
this.level = level;
this.useZlib = useZlib;
}
@Override
public boolean equals(final Object other) {
if (other == null || other.getClass() != GzipCompression.class)
return false;
else {
final GzipCompression gz = ((GzipCompression)other);
return useZlib == gz.useZlib && level == gz.level;
}
}
private InputStream decode(final InputStream in) throws IOException {
if (useZlib) {
return new InflaterInputStream(in);
} else {
return GzipCompressorInputStream.builder()
.setInputStream(in)
.setDecompressConcatenated(true)
.get();
}
}
@Override
public ReadData decode(final ReadData readData) throws N5IOException {
try {
return ReadData.from(decode(readData.inputStream()));
} catch (IOException e) {
throw new N5IOException(e);
}
}
@Override
public ReadData encode(final ReadData readData) {
if (useZlib) {
return readData.encode(out -> new DeflaterOutputStream(out, new Deflater(level)));
} else {
return readData.encode(out -> {
parameters.setCompressionLevel(level);
return new GzipCompressorOutputStream(out, parameters);
});
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java
================================================
package org.janelia.saalfeldlab.n5;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.function.TriFunction;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.http.ListResponseParser;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.channels.NonWritableChannelException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
/**
* A read-only {@link KeyValueAccess} implementation using HTTP. As a result, calling lockForWriting, createDirectories, or delete will throw an {@link N5Exception}.
*
* The behavior of list, listDirectories, and isDirectory will depend on the server configuration. See the documentation of those methods for details.
*
* Methods that take a "normalPath" as an argument expect absolute URIs.
*/
public class HttpKeyValueAccess implements KeyValueAccess {
public static final String HEAD = "HEAD";
public static final String GET = "GET";
public static final String RANGE = "Range";
public static final String ACCEPT_RANGE = "Accept-Range";
public static final String BYTES = "bytes";
private int readTimeoutMilliseconds;
private int connectionTimeoutMilliseconds;
private ListResponseParser listResponseParser = ListResponseParser.defaultListParser();
private ListResponseParser listDirectoryResponseParser = ListResponseParser.defaultDirectoryListParser();
/**
* Opens an {@link HttpKeyValueAccess}
*
* @throws N5IOException if the access could not be created
*/
public HttpKeyValueAccess() {
readTimeoutMilliseconds = 5000;
connectionTimeoutMilliseconds = 5000;
}
public void setReadTimeout(int readTimeoutMilliseconds) {
this.readTimeoutMilliseconds = readTimeoutMilliseconds;
}
public void setConnectionTimeout(int connectionTimeoutMilliseconds) {
this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
}
public void setListParser(final ListResponseParser parser) {
listResponseParser = parser;
}
public void setListDirectoryParser(final ListResponseParser parser) {
listDirectoryResponseParser = parser;
}
@Override
public String normalize(final String path) {
return N5URI.normalizeGroupPath(path);
}
@Override
public URI uri(final String normalPath) throws URISyntaxException {
return new URI(normalPath);
}
/**
* Test whether the {@code normalPath} exists.
*
* Removes leading slash from {@code normalPath}, and then checks whether
* either {@code path} or {@code path + "/"} is a key.
*
* @param normalPath is expected to be in normalized form, no further efforts are
* made to normalize it.
* @return {@code true} if {@code path} exists, {@code false} otherwise
*/
@Override
public boolean exists(final String normalPath) {
try {
requireValidHttpResponse(normalPath, "HEAD", "Error checking existence: " + normalPath, true);
return true;
} catch (N5Exception.N5NoSuchKeyException e) {
return false;
}
}
@Override public long size(String normalPath) {
final HttpURLConnection head = requireValidHttpResponse(normalPath, "HEAD", "Error checking existence: " + normalPath, true);
return head.getContentLengthLong();
}
/**
* Test whether the path is a directory.
*
* Appends trailing "/" to {@code normalPath} if there is none, removes
* leading "/", and then checks whether resulting {@code path} is a key.
*
* @param normalPath is expected to be in normalized form, no further efforts are
* made to normalize it.
* @return {@code true} if {@code path} (with trailing "/") exists as a key,
* {@code false} otherwise
*/
@Override
public boolean isDirectory(final String normalPath) {
try {
requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, false, (code, msg,http) -> {
final N5Exception cause = validExistsResponse(code, "Error checking directory: " + normalPath, msg, true);
if (code >= 300 && code < 400) {
final String redirectLocation = http.getHeaderField("Location");
if (!(redirectLocation.endsWith("/") || redirectLocation.endsWith("index.html")))
return new N5Exception.N5NoSuchKeyException("Found File at " + normalPath + " but was not directory");
return null;
}
return cause;
});
return true;
} catch (N5Exception e) {
return false;
}
}
private static String getDirectoryPath(String normalPath) {
final String directoryNormalPath;
if (normalPath.endsWith("/"))
directoryNormalPath = normalPath;
else
directoryNormalPath = normalPath + "/";
return directoryNormalPath;
}
/**
* Test whether the path is a file.
*
* Checks whether {@code normalPath} has no trailing "/", then removes
* leading "/" and checks whether the resulting {@code path} is a key.
*
* @param normalPath is expected to be in normalized form, no further efforts are
* made to normalize it.
* @return {@code true} if {@code path} exists as a key and has no trailing
* slash, {@code false} otherwise
*/
@Override
public boolean isFile(final String normalPath) {
/* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */
try {
requireValidHttpResponse(getFilePath(normalPath), HEAD, false, (code, msg, http) -> {
final N5Exception cause = validExistsResponse(code, "Error accessing file: " + normalPath, msg, true);
if (code >= 300 && code < 400) {
final String redirectLocation = http.getHeaderField("Location");
if (redirectLocation.endsWith("/") || redirectLocation.endsWith("index.html"))
return new N5Exception.N5NoSuchKeyException("Found key at " + normalPath + " but was directory");
}
return cause;
});
return true;
} catch (N5Exception e) {
return false;
}
}
private static String getFilePath(String normalPath) {
final String fileNormalPath = normalPath.replaceAll("/+$", "");
return fileNormalPath;
}
private HttpURLConnection httpRequest(String normalPath, String method) throws IOException {
final URL url = URI.create(normalPath).toURL();
final HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setReadTimeout(readTimeoutMilliseconds);
connection.setConnectTimeout(connectionTimeoutMilliseconds);
connection.setRequestMethod(method);
return connection;
}
@Override
public VolatileReadData createReadData(final String normalPath) {
return VolatileReadData.from(new HttpLazyRead(normalPath));
}
public LockedChannel lockForReading(final String normalPath) throws N5IOException {
//TODO Caleb: Maybe check exists lazily when attempting to read
try {
if (!exists(normalPath))
throw new N5Exception.N5NoSuchKeyException("Key does not exist: " + normalPath);
return new HttpObjectChannel(uri(normalPath), 0, -1);
} catch (URISyntaxException e) {
throw new N5Exception("Invalid URI Syntax", e);
}
}
@Override
public LockedChannel lockForWriting(final String normalPath) throws N5IOException {
throw new N5Exception("HttpKeyValueAccess is read-only");
}
@Override
public void write(final String normalPath, final ReadData data) throws N5IOException {
throw new N5Exception("HttpKeyValueAccess is read-only");
}
/**
* List all 'directory'-like children of a path.
*
* Will throw an N5IOException both if a connection to the server can not be established, or the server does not allow listing.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the directories
* @throws N5IOException
* if an error occurs during listing
*/
@Override
public String[] listDirectories(final String normalPath) throws N5IOException {
return queryListEntries(normalPath, listDirectoryResponseParser, true);
}
/**
* List all children of a path.
*
* Will throw an N5IOException both if a connection to the server can not be
* established, or the server does not allow listing.
*
* @param normalPath
* is expected to be in normalized form, no further efforts are
* made to normalize it.
* @return the the child paths
* @throws N5IOException
* if an error occurs during listing
*/
@Override
public String[] list(final String normalPath) throws N5IOException {
return queryListEntries(normalPath, listResponseParser, true);
}
private String[] queryListEntries(String normalPath, ListResponseParser parser, boolean allowRedirect) throws N5IOException{
final HttpURLConnection http = requireValidHttpResponse(normalPath, GET, "Error listing directory at " + normalPath, allowRedirect);
try {
final String listResponse = responseToString(http.getInputStream());
return parser.parseListResponse(listResponse);
} catch (IOException e) {
throw new N5IOException("Error listing directory at " + normalPath, e);
}
}
private static N5Exception validExistsResponse(int code, String responseMsg, String message, boolean allowRedirect) {
if (code >= 200 && code < (allowRedirect ? 400 : 300)) return null;
final RuntimeException cause = new RuntimeException(message + "( "+ responseMsg + ")(" + code + ")");
if (code == 404 | code == 410)
return new N5Exception.N5NoSuchKeyException(message, cause);
return new N5Exception(message, cause);
}
private HttpURLConnection requireValidHttpResponse(String uri, String method, String message, boolean allowRedirect) throws N5Exception {
return requireValidHttpResponse(uri, method, (code, msg, http) -> validExistsResponse(code, msg, message, allowRedirect));
}
private HttpURLConnection requireValidHttpResponse(String uri, String method, TriFunction filterCode) throws N5Exception {
return requireValidHttpResponse(uri, method, true, filterCode);
}
private HttpURLConnection requireValidHttpResponse(String uri, String method, boolean followRedirects, TriFunction filterCode) throws N5Exception {
final int code;
final HttpURLConnection http;
final String responseMsg;
try {
http = httpRequest(uri, method);
http.setInstanceFollowRedirects(followRedirects);
code = http.getResponseCode();
responseMsg = http.getResponseMessage();
} catch (IOException e) {
throw new N5IOException("Could not validate HTTP Response", e);
}
final N5Exception cause = filterCode.apply(code, responseMsg, http);
if (cause != null) throw cause;
return http;
}
private String responseToString(InputStream inputStream) throws IOException {
return IOUtils.toString(inputStream, StandardCharsets.UTF_8.name());
}
@Override
public void createDirectories(final String normalPath) {
throw new N5Exception("HttpKeyValueAccess is read-only");
}
@Override
public void delete(final String normalPath) {
throw new N5Exception("HttpKeyValueAccess is read-only");
}
private class HttpObjectChannel implements LockedChannel {
protected final URI uri;
private final long startByte;
private final long size;
private final ArrayList resources = new ArrayList<>();
protected HttpObjectChannel(final URI uri, long startByte, long size) {
this.uri = uri;
this.startByte = startByte;
this.size = size;
}
private boolean isPartialRead() {
return startByte > 0 || (size >= 0 && size != Long.MAX_VALUE);
}
@Override
public InputStream newInputStream() throws N5IOException {
try {
HttpURLConnection conn = (HttpURLConnection)uri.toURL().openConnection();
if (isPartialRead()) {
conn.setRequestProperty(RANGE, rangeString());
final String acceptRanges = conn.getHeaderField(ACCEPT_RANGE);
if (acceptRanges == null || !acceptRanges.equals(BYTES)) {
conn.disconnect();
conn = (HttpURLConnection)uri.toURL().openConnection();
return ReadData.from(conn.getInputStream()).materialize().slice(startByte, size).inputStream();
}
}
return conn.getInputStream();
} catch (FileNotFoundException e) {
/*default HttpURLConnection throws FileNotFoundException on 404 or 410 */
throw new N5Exception.N5NoSuchKeyException("Could not open stream for " + uri, e);
} catch (IOException e) {
throw new N5IOException("Could not open stream for " + uri, e);
}
}
private String rangeString() {
final String lastByte = (size > 0) ? Long.toString(startByte + size - 1) : "";
return String.format("%s=%d-%s", BYTES, startByte, lastByte);
}
@Override
public Reader newReader() {
final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8);
synchronized (resources) {
resources.add(reader);
}
return reader;
}
@Override
public OutputStream newOutputStream() {
throw new NonWritableChannelException();
}
@Override
public Writer newWriter() {
throw new NonWritableChannelException();
}
@Override
public void close() throws IOException {
synchronized (resources) {
for (final Closeable resource : resources) {
resource.close();
}
resources.clear();
}
}
}
private class HttpLazyRead implements LazyRead {
private final String normalKey;
HttpLazyRead(String normalKey) {
this.normalKey = normalKey;
}
@Override
public long size() {
return HttpKeyValueAccess.this.size(normalKey);
}
@Override
public ReadData materialize(long offset, long length) {
try (final HttpObjectChannel ch = new HttpObjectChannel(uri(normalKey), offset, length)) {
return ReadData.from(ch.newInputStream()).materialize();
} catch (IOException e) {
throw new N5IOException(e);
} catch (URISyntaxException e) {
throw new N5Exception(e);
}
}
@Override
public void close() {
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
public class IntArrayDataBlock extends AbstractDataBlock {
public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) {
super(size, gridPosition, data, a -> a.length);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
import java.io.IOException;
public interface IoPolicy {
void write(String key, ReadData readData) throws IOException;
VolatileReadData read(String key) throws IOException;
void delete(String key) throws IOException;
static IoPolicy withFallback(IoPolicy primary, IoPolicy fallback) {
return new IoPolicy() {
@Override
public void write(String key, ReadData readData) throws IOException {
try {
primary.write(key, readData);
} catch (IOException e) {
fallback.write(key, readData);
}
}
@Override
public VolatileReadData read(String key) throws IOException {
try {
return primary.read(key);
} catch (IOException e) {
return fallback.read(key);
}
}
@Override
public void delete(String key) throws IOException {
try {
primary.delete(key);
} catch (IOException e) {
fallback.delete(key);
}
}
};
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/KeyLockState.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.Semaphore;
/**
* Per-key state that tracks both thread locks and file locks.
*/
class KeyLockState {
private final Path path;
private final LockingPolicy policy;
public KeyLockState(final Path path, LockingPolicy policy) {
this.path = path;
this.policy = policy;
}
/**
* The current system-level file lock (shared for reading, exclusive for
* writing). Is {@link ChannelLock#close closed} when the last {@link
* #releaseRead() Reader is released}, or the (one and only) {@link
* #releaseWrite() Writer is released}.
*/
private volatile ChannelLock channelLock;
/**
* Multiple Readers coordinate via this mutex. {@code numReaders} may only
* be modified when {@code readerMutex} is held.
*/
private final Semaphore readerMutex = new Semaphore(1);
private int numReaders = 0;
/**
* This coordinates mutual exclusion between one writer and (the first of)
* multiple readers. {@code channelLock} may only be created or closed when
* {@code channelLockMutex} is held.
*/
private final Semaphore channelLockMutex = new Semaphore(1);
LockedFileChannel acquireRead() throws IOException {
try {
readerMutex.acquire();
try {
if (numReaders == 0) {
// We are the first Reader, and are responsible for creating the channelLock
// (Other concurrent Readers will still be blocked in readerMutex.)
// If a Writer is still open, this will block us until the Writer is closed.
channelLockMutex.acquire();
try {
channelLock = ChannelLock.lock(path, false, policy);
} catch (IOException e) {
// Something went wrong. Back off.
channelLockMutex.release();
throw e;
}
}
// We have a FileChannel.
// Create a LockedFileChannel that will releaseRead() when it is closed.
++numReaders;
return new LockedFileChannel(channelLock.getChannel(), this::releaseRead);
} finally {
readerMutex.release();
}
} catch (InterruptedException e) {
throw new IOException(e);
}
}
void releaseRead() throws IOException {
try {
readerMutex.acquire();
try {
--numReaders;
if (numReaders == 0) {
// We were the last Reader, and are responsible for releasing the channelLock
releaseChannelLock();
}
} finally {
readerMutex.release();
}
} catch (InterruptedException e) {
throw new IOException(e);
}
}
private void releaseChannelLock() throws IOException {
try {
channelLock.close();
} finally {
channelLockMutex.release();
}
}
LockedFileChannel acquireWrite() throws IOException {
try {
// If another Writer or Reader is still open, this will block until it is closed.
channelLockMutex.acquire();
try {
channelLock = ChannelLock.lock(path, true, policy);
} catch (IOException e) {
// Something went wrong. Back off.
channelLockMutex.release();
throw e;
}
// We have a WRITE ChannelLock.
// Create a LockedFileChannel that will releaseWrite() when it is closed.
return new LockedFileChannel(channelLock.getChannel(), this::releaseWrite);
} catch (InterruptedException e) {
throw new IOException(e);
}
}
void releaseWrite() throws IOException {
releaseChannelLock();
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystem;
import java.util.Arrays;
import java.util.stream.Collectors;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
/**
* Key value read primitives used by {@link N5KeyValueReader}
* implementations. This interface implements a subset of access primitives
* provided by {@link FileSystem} to reduce the implementation burden for
* backends lacking a {@link FileSystem} implementation (such as AWS-S3).
*
* @author Stephan Saalfeld
*/
public interface KeyValueAccess {
/**
* Split a path string into its components.
*
* @param path
* the path
* @return the path components
*/
default String[] components( final String path ) {
String[] components = Arrays.stream(path.split("/"))
.filter(x -> !x.isEmpty())
.toArray(String[]::new);
if (components.length == 0)
return path.startsWith("/") ? new String[]{"/"} : new String[]{""};
if (path.startsWith("/") && !components[0].equals("/")) {
final String[] prependRoot = new String[components.length + 1];
prependRoot[0] = "/";
System.arraycopy(components, 0, prependRoot, 1, components.length);
components = prependRoot;
}
if (path.endsWith("/") && !components[components.length - 1].endsWith("/")) {
components[components.length - 1] = components[components.length - 1] + "/";
}
return components;
}
/**
* Compose a path from a base uri and subsequent components.
*
* @param uri the base path uri
* @param components the path components
* @return the path
*/
default String compose( final URI uri, final String... components ) {
int firstNonEmptyIdx = 0;
while (firstNonEmptyIdx < components.length && (components[firstNonEmptyIdx] == null || components[firstNonEmptyIdx].isEmpty())) {
firstNonEmptyIdx++;
}
/*If there are no non-empty components, there is nothing to compose against; return the uri. */
if (components.length == firstNonEmptyIdx)
return uri.toString();
/* allocate space for the initial path and the new components, skipping empty strings */
final int nonEmptysize = components.length - firstNonEmptyIdx;
final String[] allComponents = new String[1 + nonEmptysize];
if (uri.getPath().isEmpty())
//TODO Caleb: This `isEmpty()` check is only necessary for Java 8. In newer versions
// URI resolution is updated so that resolving and empty path with a new path adds
// a leading `/` between the rest of the URI and the path part. In Java 8 it doesn't
// add the `/` so it ends up directly concatenating the path part with URI
allComponents[0] = "/";
else
allComponents[0] = uri.getPath();
System.arraycopy(components, firstNonEmptyIdx, allComponents, 1, nonEmptysize);
URI composedUri = uri;
for (int i = 0; i < allComponents.length; i++) {
final String component = allComponents[i];
if (component == null || component.isEmpty())
continue;
else if (component.endsWith("/") || i == allComponents.length - 1)
composedUri = composedUri.resolve(N5URI.encodeAsUriPath(component));
else
composedUri = composedUri.resolve(N5URI.encodeAsUriPath(component + "/"));
}
return composedUri.toString();
}
@Deprecated
default String compose( final String... components ) {
return normalize(
Arrays.stream(components)
.filter(x -> !x.isEmpty())
.collect(Collectors.joining("/"))
);
}
/**
* Get the parent of a path string.
*
* @param path
* the path
* @return the parent path or null if the path has no parent
*/
default String parent( final String path ) {
final String removeTrailingSlash = path.replaceAll("/+$", "");
return normalize(N5URI.getAsUri(removeTrailingSlash).resolve("").toString());
}
/**
* Relativize path relative to base.
*
* @param path
* the path
* @param base
* the base path
* @return the result or null if the path has no parent
*/
default String relativize( final String path, final String base ) {
try {
/*
* Must pass absolute path to `uri`. if it already is, this is
* redundant, and has no impact on the result. It's not true that
* the inputs are always referencing absolute paths, but it doesn't
* matter in this case, since we only care about the relative
* portion of `path` to `base`, so the result always ignores the
* absolute prefix anyway.
*/
return normalize(uri("/" + base).relativize(uri("/" + path)).toString());
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e);
}
}
/**
* Normalize a path to canonical form. All paths pointing to the same
* location return the same output. This is most important for cached
* data pointing at the same location getting the same key.
*
* @param path
* the path
* @return the normalized path
*/
String normalize( final String path );
/**
* Get the absolute (including scheme) {@link URI} of the given path
*
* @param uriString
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return absolute URI
* @throws URISyntaxException if the given path is not a proper URI
*/
default URI uri(final String uriString) throws URISyntaxException {
try {
return URI.create(uriString);
} catch (Exception ignore) {
return N5URI.encodeAsUri(uriString);
}
}
/**
* Test whether the path exists.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return true if the path exists
*/
boolean exists( final String normalPath );
/**
* Returns the size in bytes of the object at the given normalPath if it exists.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the size of the object in bytes.
* @throws N5Exception.N5NoSuchKeyException if the given key does not exist
*/
long size( final String normalPath ) throws N5Exception.N5NoSuchKeyException;
/**
* Test whether the path is a directory.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return true if the path is a directory
*/
boolean isDirectory( String normalPath );
/**
* Test whether the path is a file.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return true if the path is a file
*/
boolean isFile( String normalPath ); // TODO: Looks un-used. Remove?
/**
* Create a {@link VolatileReadData} through which data at the normal key
* can be read.
*
* Implementations should read lazily if possible. Consumers may call {@link
* ReadData#materialize()} to force a read operation if needed.
*
* If supported by this KeyValueAccess implementation, partial reads are
* possible by {@link ReadData#slice slicing} the returned {@code ReadData}.
*
* The resulting {@code VolatileReadData} is potentially lazy. If the requested
* key does not exist, it will throw {@code N5NoSuchKeyException}. Whether
* the exception is thrown when {@link KeyValueAccess#createReadData(String)}] is called,
* or when trying to materialize the {@code VolatileReadData} is implementation dependent.
*
* @param normalPath
* is expected to be in normalized form, no further efforts are made to normalize it
*
* @return a ReadData
*
* @throws N5IOException
* if an error occurs
*/
VolatileReadData createReadData(final String normalPath) throws N5IOException;
/**
* Write {@code data} to the given {@code normalPath}.
*
* Existing data at {@code normalPath} will be overridden.
*
* @param normalPath
* is expected to be in normalized form, no further efforts are made to normalize it
* @param data
* the data to write
*
* @throws N5IOException
* if an error occurs
*/
void write(String normalPath, ReadData data) throws N5IOException;
/**
* Create a lock on a path for reading. This isn't meant to be kept
* around. Create, use, [auto]close, e.g.
*
* try (final lock = store.lockForReading()) {
* ...
* }
*
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the locked channel
* @throws N5IOException
* if a locked channel could not be created
* @deprecated migrate to {@link KeyValueAccess#createReadData(String)}
*/
@Deprecated
default LockedChannel lockForReading( final String normalPath ) throws N5IOException {
return new BufferedKvaLockedChannel(this, normalPath);
}
/**
* Create an exclusive lock on a path for writing. If the file doesn't exist
* yet, it will be created, including all directories leading up to it. This
* lock isn't meant to be kept around. Create, use, [auto]close, e.g.
*
* try (final lock = store.lockForWriting()) {
* ...
* }
*
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the locked channel
* @throws N5IOException
* if a locked channel could not be created
* @deprecated migrate to {@link KeyValueAccess#write(String, ReadData)}
*/
@Deprecated
default LockedChannel lockForWriting( final String normalPath ) throws N5IOException {
return new BufferedKvaLockedChannel(this, normalPath);
}
/**
* List all 'directory'-like children of a path.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the directories
* @throws N5IOException
* if an error occurs during listing
*/
String[] listDirectories( final String normalPath ) throws N5IOException;
/**
* List all children of a path.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @return the the child paths
* @throws N5IOException if an error occurs during listing
*/
String[] list( final String normalPath ) throws N5IOException;
/**
* Create a directory and all parent paths along the way. The directory
* and parent paths are discoverable. On a filesystem, this usually means
* that the directories exist, on a key value store that is unaware of
* directories, this may be implemented as creating an object for each path.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @throws N5IOException
* if an error occurs during creation
*/
void createDirectories( final String normalPath ) throws N5IOException;
/**
* Delete a path. If the path is a directory, delete it recursively.
*
* @param normalPath
* is expected to be in normalized form, no further
* efforts are made to normalize it.
* @throws N5IOException
* if an error occurs during deletion
*/
void delete( final String normalPath ) throws N5IOException;
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java
================================================
package org.janelia.saalfeldlab.n5;
import java.util.Iterator;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
public abstract class LinkedAttributePathToken implements Iterator> {
/**
* The JsonElement which contains the mapping that this token represents.
*/
protected T parentJson;
/**
* The token representing the subsequent path element.
*/
protected LinkedAttributePathToken childToken;
/**
* @return a reference object of type {@code T}
*/
public abstract T getJsonType();
/**
* This method will return the child element, if the {@link #parentJson}
* contains a child that
* is valid for this token. If the {@link #parentJson} does not have a
* child, this method will
* create it. If the {@link #parentJson} does have a child, but it is not
* {@link #jsonCompatible(JsonElement)}
* with our {@link #childToken}, then this method will also create a new
* (compatible) child, and replace
* the existing incompatible child.
*
* @return the resulting child element
*/
public abstract JsonElement getOrCreateChildElement();
/**
* This method will write into {@link #parentJson} the subsequent
* {@link JsonElement}.
*
* The written JsonElement will EITHER be:
*
* - The result of serializing {@code value} ( if {@link #childToken} is
* {@code null} )
* - The {@link JsonElement} which represents our {@link #childToken} (See
* {@link #getOrCreateChildElement()} )
*
*
* @param gson
* instance used to serialize {@code value }
* @param value
* to write
* @return the object that was written.
*/
public JsonElement writeChild(final Gson gson, final Object value) {
if (childToken != null)
/*
* If we have a child, get/create it and set current json element to
* it
*/
return getOrCreateChildElement();
else {
/* We are done, no token remaining */
writeValue(gson, value);
return getChildElement();
}
}
/**
* Check if the provided {@code json} is compatible with this token.
*
* Compatibility means that the provided {@code JsonElement} does not
* explicitly conflict with this token. That means that {@code null} is
* always compatible. The only real incompatibility is when the
* {@code JsonElement}
* that we receive is different that we expect.
*
* @param json
* element that we are testing for compatibility.
* @return false if {@code json} is not null and is not of type {@code T}
*/
@SuppressWarnings("BooleanMethodIsAlwaysInverted")
public boolean jsonCompatible(final JsonElement json) {
return json == null || json.getClass() == getJsonType().getClass();
}
/**
* Write {@code value} into {@link #parentJson}.
*
* Should only be called when {@link #childToken} is null (i.e. this is the
* last token in the attribute path).
*
* @param gson
* instance used to serialize {@code value}
* @param value
* to serialize
*/
protected abstract void writeValue(Gson gson, Object value);
/**
* Write the {@link JsonElement} the corresponds to {@link #childToken} into
* {@link #parentJson}.
*/
protected abstract void writeChildElement();
/**
* @return the element that is represented by {@link #childToken} if
* present.
*/
protected abstract JsonElement getChildElement();
/**
* If {@code json} is compatible with the type or token that we are, then
* set {@link #parentJson} to {@code json}.
*
* However, if {@code json} is either {@code null} or not
* {@link #jsonCompatible(JsonElement)} then
* {@link #parentJson} will be set to a new instance of {@code T}.
*
* @param json
* to attempt to set to {@link #parentJson}
* @return the value set to {@link #parentJson}.
*/
@SuppressWarnings("unchecked")
protected JsonElement setAndCreateParentElement(final JsonElement json) {
if (json == null || !jsonCompatible(json)) {
// noinspection unchecked
parentJson = (T)getJsonType().deepCopy();
} else {
// noinspection unchecked
parentJson = (T)json;
}
return parentJson;
}
/**
* @return if we have a {@link #childToken}.
*/
@Override
public boolean hasNext() {
return childToken != null;
}
/**
* @return {@link #childToken}
*/
@Override
public LinkedAttributePathToken next() {
return childToken;
}
public static class ObjectAttributeToken extends LinkedAttributePathToken {
private static final JsonObject JSON_OBJECT = new JsonObject();
private final String key;
public ObjectAttributeToken(final String key) {
this.key = key;
}
/**
* @return the {@link #key} this token maps it's
* {@link #getChildElement()} to.
*/
public String getKey() {
return key;
}
@Override
public String toString() {
return getKey();
}
@Override
public JsonObject getJsonType() {
return JSON_OBJECT;
}
@Override
public JsonElement getOrCreateChildElement() {
final JsonElement childElement = parentJson.getAsJsonObject().get(key);
if (!parentJson.has(key) || childToken != null && !childToken.jsonCompatible(childElement)) {
writeChildElement();
}
return getChildElement();
}
@Override
protected JsonElement getChildElement() {
return parentJson.get(key);
}
@Override
protected void writeValue(final Gson gson, final Object value) {
parentJson.add(key, gson.toJsonTree(value));
}
@Override
protected void writeChildElement() {
if (childToken != null)
parentJson.add(key, childToken.getJsonType().deepCopy());
}
}
public static class ArrayAttributeToken extends LinkedAttributePathToken {
private static final JsonArray JSON_ARRAY = new JsonArray();
private final int index;
public ArrayAttributeToken(final int index) {
this.index = index;
}
/**
* @return the {@link #index} this token maps it's
* {@link #getChildElement()} to.
*/
public int getIndex() {
return index;
}
@Override
public String toString() {
return "[" + getIndex() + "]";
}
@Override
public JsonArray getJsonType() {
return JSON_ARRAY;
}
@Override
public JsonElement getOrCreateChildElement() {
if (index >= parentJson.size() || childToken != null && !childToken.jsonCompatible(parentJson.get(index)))
writeChildElement();
return parentJson.get(index);
}
@Override
protected void writeChildElement() {
if (childToken != null) {
fillArrayToIndex(parentJson, index, null);
parentJson.set(index, childToken.getJsonType().deepCopy());
}
}
@Override
protected JsonElement getChildElement() {
return parentJson.get(index);
}
@Override
protected void writeValue(final Gson gson, final Object value) {
fillArrayToIndex(parentJson, index, value);
parentJson.set(index, gson.toJsonTree(value));
}
/**
* Fill {@code array} up to, and including, {@code index} with a default
* value, determined by the type of {@code value}.
* Importantly, this does NOT set {@code array[index]=value}.
*
* @param array
* to fill
* @param index
* to fill the array to (inclusive)
* @param value
* used to determine the default array fill value
*/
private static void fillArrayToIndex(final JsonArray array, final int index, final Object value) {
final JsonElement fillValue;
if (valueRepresentsANumber(value)) {
fillValue = new JsonPrimitive(0);
} else {
fillValue = JsonNull.INSTANCE;
}
for (int i = array.size(); i <= index; i++) {
array.add(fillValue);
}
}
/**
* Check if {@value} represents a {@link Number}.
* True if {@code value} either:
*
* - is a {@code Number}, or;
* - is a {@link JsonPrimitive} where
* {@link JsonPrimitive#isNumber()}
*
*
* @param value
* we wish to check if represents a {@link Number} or not.
* @return true if {@code value} represents a {@link Number}
*/
private static boolean valueRepresentsANumber(final Object value) {
return value instanceof Number || (value instanceof JsonPrimitive && ((JsonPrimitive)value).isNumber());
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import java.io.Closeable;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
/**
* A lock on a path that can create a {@link Reader}, {@link Writer},
* {@link InputStream}, or {@link OutputStream}.
*
* @author Stephan Saalfeld
*/
public interface LockedChannel extends Closeable {
/**
* Create a UTF-8 {@link Reader}.
*
* @return the reader
* @throws N5IOException
* if the reader could not be created
*/
Reader newReader() throws N5IOException;
/**
* Create a new {@link InputStream}.
*
* @return the input stream
* @throws N5IOException
* if an input stream could not be created
*/
InputStream newInputStream() throws N5IOException;
/**
* Create a new UTF-8 {@link Writer}.
*
* @return the writer
* @throws N5IOException
* if a writer could not be created
*/
Writer newWriter() throws N5IOException;
/**
* Create a new {@link OutputStream}.
*
* @return the output stream
* @throws N5IOException
* if an output stream could not be created
*/
OutputStream newOutputStream() throws N5IOException;
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
/**
* LockedFileChannel implementation for both read and write operations.
*
* When closing this {@code LockedFileChannel}, {@code releaseLock} is called,
* but the {@code channel} is not closed. The channel may be shared among
* multiple {@code LockedFileChannel} for concurrent readers. {@code
* releaseLock} should take care of closing the channel when the last reader (or
* the only writer) is closed.
*/
class LockedFileChannel implements Closeable {
// TODO: Consider splitting LockedFileChannel into read-only and write-only part.
// TODO: Consider removing LockedFileChannel and having
// FileKeyLockManager.acquireRead() and FileKeyLockManager.acquireWrite()
// return appropriately wrapped SeekableByteChannel.
private final FileChannel channel;
private ReleaseLock releaseLock;
@FunctionalInterface
public interface ReleaseLock {
void release() throws IOException;
}
LockedFileChannel(final FileChannel channel, final ReleaseLock releaseLock) {
this.channel = channel;
this.releaseLock = releaseLock;
}
/**
* Returns the size of this channel's file.
*
* See {@link FileChannel#size()}.
*/
public long size() throws IOException {
return channel.size();
}
/**
* Reads a sequence of bytes from this channel into the given buffer,
* starting at the given file position.
*
* See {@link FileChannel#read(ByteBuffer, long)}.
*/
public int read(final ByteBuffer dst, final long position) throws IOException {
return channel.read(dst, position);
}
/**
* Return an {@link OutputStream} that writes into this channel.
* Closing the OutputStream will close this channel.
*/
public OutputStream asOutputStream() {
return Channels.newOutputStream(new ClosingChannelWrapper());
}
volatile boolean firstWrite = true;
private class ClosingChannelWrapper implements WritableByteChannel {
@Override
public int write(final ByteBuffer src) throws IOException {
if (!firstWrite)
return channel.write(src);
try {
channel.truncate(0);
return channel.write(src);
} finally {
firstWrite = false;
}
}
@Override
public boolean isOpen() {
return channel.isOpen();
}
@Override
public void close() throws IOException {
channel.close();
LockedFileChannel.this.close();
}
}
@Override
public void close() throws IOException {
if (releaseLock != null) {
releaseLock.release();
releaseLock = null;
// Mote that setting releaseLock=null here drops the (method)
// reference to LockKeyState, which potentially allows clearing the
// WeakReference that FileLockManager holds.
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/LockingPolicy.java
================================================
package org.janelia.saalfeldlab.n5;
/**
* File locking policy.
*
* Usually, we want to coordinate reads and writs to a container such that
*
* - multiple readers can access a key simultaneously (blocking all writers).
* - A writer should have exclusive access to a key (blocking all other readers and writers).
*
* However, this cannot always be enforced for all backends:
* For SMB on macOS OS-level file locking is broken.
* For AWS S3 and Google Cloud we can detect (but not prevent) concurrent modifications.
*
* Sometimes we know that we can disregard locking, for example in read-only settings.
*
* {@code IoPolicy} can be used to configure locking policy for backends ({@link
* KeyValueAccess}) that support various locking strategies (potentially coming
* with performance trade-offs).
*
* The policy values ({@link #STRICT}, {@link #UNSAFE}, {@link #PERMISSIVE})
* specify intent. Detailed interpretation is up to the backend implementation.
*/
public enum LockingPolicy {
/**
* Protect all reads and writes by locks.
* Fail if locking is not possible.
*/
STRICT,
/**
* Reads and writes are unprotected.
*/
UNSAFE,
/**
* Try to lock for all reads and writes.
* Fall back to unprotected reads and writes if locking is not possible.
* This is the default.
*/
PERMISSIVE;
static LockingPolicy fromString(final String s) {
if ("strict".equalsIgnoreCase(s))
return STRICT;
else if ("unsafe".equalsIgnoreCase(s))
return UNSAFE;
// else if ("permissive".equalsIgnoreCase(s))
// return PERMISSIVE;
else
return PERMISSIVE;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
public class LongArrayDataBlock extends AbstractDataBlock {
public LongArrayDataBlock(final int[] size, final long[] gridPosition, final long[] data) {
super(size, gridPosition, data, a -> a.length);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java
================================================
package org.janelia.saalfeldlab.n5;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@CompressionType("lz4")
@NameConfig.Name("lz4")
public class Lz4Compression implements Compression {
private static final long serialVersionUID = -9071316415067427256L;
@CompressionParameter
@NameConfig.Parameter
private final int blockSize;
public Lz4Compression(final int blockSize) {
this.blockSize = blockSize;
}
public Lz4Compression() {
this(1 << 16);
}
@Override
public boolean equals(final Object other) {
if (other == null || other.getClass() != Lz4Compression.class)
return false;
else
return blockSize == ((Lz4Compression)other).blockSize;
}
@Override
public ReadData decode(final ReadData readData) {
return ReadData.from(new LZ4BlockInputStream(readData.inputStream()));
}
@Override
public ReadData encode(final ReadData readData) {
return readData.encode(out -> new LZ4BlockOutputStream(out, blockSize));
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java
================================================
package org.janelia.saalfeldlab.n5;
public class N5Exception extends RuntimeException {
public N5Exception() {
super();
}
public N5Exception(final String message) {
super(message);
}
public N5Exception(final String message, final Throwable cause) {
super(message, cause);
}
public N5Exception(final Throwable cause) {
super(cause);
}
protected N5Exception(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public static class N5IOException extends N5Exception{
public N5IOException(final String message) {
super(message);
}
public N5IOException(final String message, final Throwable cause) {
super(message, cause);
}
public N5IOException(final Throwable cause) {
super(cause);
}
protected N5IOException(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
/**
* This excpetion represents the situation when an attribute is requested by key as a specific Class,
* that attribute key does exist, but is not parseable as the desired Class
*/
public static class N5ClassCastException extends N5Exception {
public N5ClassCastException(final Class cls) {
super("Cannot cast as class " + cls.getName());
}
public N5ClassCastException(final String message) {
super(message);
}
public N5ClassCastException(final String message, final Throwable cause) {
super(message, cause);
}
public N5ClassCastException(final Throwable cause) {
super(cause);
}
protected N5ClassCastException(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
public static class N5NoSuchKeyException extends N5IOException {
public N5NoSuchKeyException(final String message) {
super(message);
}
public N5NoSuchKeyException(final String message, final Throwable cause) {
super(message, cause);
}
public N5NoSuchKeyException(final Throwable cause) {
super(cause);
}
protected N5NoSuchKeyException(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
/**
* Exception to represent an error when attempting to parse json attributes
*/
public static class N5JsonParseException extends N5Exception {
public N5JsonParseException(final String message) {
super(message);
}
public N5JsonParseException(final String message, final Throwable cause) {
super(message, cause);
}
public N5JsonParseException(final Throwable cause) {
super(cause);
}
protected N5JsonParseException(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
public static class N5ConcurrentModificationException extends N5IOException {
public N5ConcurrentModificationException(final String message) {
super(message);
}
public N5ConcurrentModificationException(final String message, final Throwable cause) {
super(message, cause);
}
public N5ConcurrentModificationException(final Throwable cause) {
super(cause);
}
protected N5ConcurrentModificationException(
final String message,
final Throwable cause,
final boolean enableSuppression,
final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java
================================================
package org.janelia.saalfeldlab.n5;
import java.nio.file.FileSystems;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* {@link N5Reader} implementation through {@link KeyValueAccess} with JSON
* attributes parsed with {@link Gson}.
*
* @author Stephan Saalfeld
* @author Igor Pisarev
* @author Philipp Hanslovsky
*/
//public class N5FSReader extends N5KeyValueReader {
public class N5FSReader extends N5KeyValueReader {
/**
* Opens an {@link N5FSReader} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* @param basePath
* N5 base path
* @param gsonBuilder
* the gson builder
* @param cacheMeta
* cache attributes and meta data
* Setting this to true avoids frequent reading and parsing of
* JSON encoded attributes and other meta data that requires
* accessing the store. This is most interesting for high latency
* backends. Changes of cached attributes and meta data by an
* independent writer on the same container will not be tracked.
*
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta)
throws N5Exception {
super(
new FileSystemKeyValueAccess(),
basePath,
gsonBuilder,
cacheMeta);
if (!exists("/"))
throw new N5Exception.N5IOException("No container exists at " + basePath);
}
/**
* Opens an {@link N5FSReader} at a given base path.
*
* @param basePath
* N5 base path
* @param cacheMeta
* cache attributes and meta data
* Setting this to true avoids frequent reading and parsing of
* JSON encoded attributes and other meta data that requires
* accessing the store. This is most interesting for high latency
* backends. Changes of cached attributes and meta data by an
* independent writer on the same container will not be tracked.
*
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
public N5FSReader(final String basePath, final boolean cacheMeta) throws N5Exception {
this(basePath, new GsonBuilder(), cacheMeta);
}
/**
* Opens an {@link N5FSReader} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* @param basePath
* N5 base path
* @param gsonBuilder
* the gson builder
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception {
this(basePath, gsonBuilder, false);
}
/**
* Opens an {@link N5FSReader} at a given base path.
*
* @param basePath
* N5 base path
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
public N5FSReader(final String basePath) throws N5Exception {
this(basePath, new GsonBuilder(), false);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java
================================================
package org.janelia.saalfeldlab.n5;
import java.nio.file.FileSystems;
import com.google.gson.GsonBuilder;
/**
* Filesystem {@link N5Writer} implementation with version compatibility check.
*
* @author Stephan Saalfeld
*/
public class N5FSWriter extends N5KeyValueWriter {
/**
* Opens an {@link N5FSWriter} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* If the base path does not exist, it will be created.
*
* If the base path exists and if the N5 version of the container is
* compatible with this implementation, the N5 version of this container
* will be set to the current N5 version of this implementation.
*
* @param basePath
* n5 base path
* @param gsonBuilder
* the gson builder
* @param cacheAttributes
* cache attributes and meta data
* Setting this to true avoidsfrequent reading and parsing of
* JSON encoded attributes andother meta data that requires
* accessing the store. This ismost interesting for high latency
* backends. Changes of cachedattributes and meta data by an
* independent writer on the samecontainer will not be tracked.
*
* @throws N5Exception
* if the base path cannot be written to or cannot be created,
* if the N5 version of the container is not compatible with
* this implementation.
*/
public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes)
throws N5Exception {
super(
new FileSystemKeyValueAccess(),
basePath,
gsonBuilder,
cacheAttributes);
}
/**
* Opens an {@link N5FSWriter} at a given base path.
*
* If the base path does not exist, it will be created.
*
* If the base path exists and if the N5 version of the container is
* compatible with this implementation, the N5 version of this container
* will be set to the current N5 version of this implementation.
*
* @param basePath
* base path
* @param cacheAttributes
* attributes and meta data
* Setting this to true avoidsfrequent reading and parsing of
* JSON encoded attributes andother meta data that requires
* accessing the store. This ismost interesting for high latency
* backends. Changes of cachedattributes and meta data by an
* independent writer on the samecontainer will not be tracked.
*
* @throws N5Exception
* if the base path cannot be written to or cannot be created,
* if the N5 version of the container is not compatible with
* this implementation.
*/
public N5FSWriter(final String basePath, final boolean cacheAttributes) throws N5Exception {
this(basePath, new GsonBuilder(), cacheAttributes);
}
/**
* Opens an {@link N5FSWriter} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* If the base path does not exist, it will be created.
*
*
* If the base path exists and if the N5 version of the container is
* compatible with this implementation, the N5 version of this container
* will be set to the current N5 version of this implementation.
*
*
* @param basePath
* base path
* @param gsonBuilder
* gson builder
*
* @throws N5Exception
* if the base path cannot be written to or cannot be created,
* if the N5 version of the container is not compatible with
* this implementation.
*/
public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception {
this(basePath, gsonBuilder, false);
}
/**
* Opens an {@link N5FSWriter} at a given base path.
*
* If the base path does not exist, it will be created.
*
*
* If the base path exists and if the N5 version of the container is
* compatible with this implementation, the N5 version of this container
* will be set to the current N5 version of this implementation.
*
*
* @param basePath
* n5 base path
*
* @throws N5Exception
* if the base path cannot be written to or cannot be created,
* if the N5 version of the container is not compatible with
* this implementation.
*/
public N5FSWriter(final String basePath) throws N5Exception {
this(basePath, new GsonBuilder());
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java
================================================
package org.janelia.saalfeldlab.n5;
import java.net.URI;
import java.net.URISyntaxException;
import com.google.gson.JsonElement;
import org.janelia.saalfeldlab.n5.cache.N5JsonCache;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
/**
* {@link N5Reader} implementation through {@link KeyValueAccess} with JSON
* attributes parsed with {@link Gson}.
*
* @author Stephan Saalfeld
* @author Igor Pisarev
* @author Philipp Hanslovsky
*/
public class N5KeyValueReader implements CachedGsonKeyValueN5Reader {
public static final String ATTRIBUTES_JSON = "attributes.json";
protected final KeyValueAccess keyValueAccess;
protected final Gson gson;
protected final boolean cacheMeta;
protected URI uri;
private final N5JsonCache cache;
/**
* Opens an {@link N5KeyValueReader} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* @param keyValueAccess
* the KeyValueAccess backend used
* @param basePath
* N5 base path
* @param gsonBuilder
* the GsonBuilder
* @param cacheMeta
* cache attributes and meta data
* Setting this to true avoidsfrequent reading and parsing of
* JSON encoded attributes andother meta data that requires
* accessing the store. This ismost interesting for high latency
* backends. Changes of cachedattributes and meta data by an
* independent writer will not betracked.
*
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
public N5KeyValueReader(
final KeyValueAccess keyValueAccess,
final String basePath,
final GsonBuilder gsonBuilder,
final boolean cacheMeta)
throws N5Exception {
this(true, keyValueAccess, basePath, gsonBuilder, cacheMeta, true);
}
/**
* Opens an {@link N5KeyValueReader} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
* @param checkVersion
* the version check
* @param keyValueAccess
* the backend KeyValueAccess used
* @param basePath
* base path
* @param gsonBuilder
* the GsonBuilder
* @param cacheMeta
* cache attributes and meta data Setting this to true avoids
* frequent reading and parsing of JSON encoded attributes and
* other meta data that requires accessing the store. This is
* most interesting for high latency backends. Changes of cached
* attributes and meta data by an independent writer will not be
* tracked.
* @param checkExists
* if true, an N5IOException will be thrown if a container does
* not exist at the specified location
* @throws N5Exception
* if the base path cannot be read or does not exist, if the N5
* version of the container is not compatible with this
* implementation.
*/
protected N5KeyValueReader(
final boolean checkVersion,
final KeyValueAccess keyValueAccess,
final String basePath,
final GsonBuilder gsonBuilder,
final boolean cacheMeta,
final boolean checkExists)
throws N5Exception {
this.keyValueAccess = keyValueAccess;
this.gson = registerGson(gsonBuilder).create();
this.cacheMeta = cacheMeta;
if (this.cacheMeta)
this.cache = newCache();
else
this.cache = null;
try {
uri = keyValueAccess.uri(basePath);
} catch (final URISyntaxException e) {
throw new N5Exception(e);
}
boolean versionFound = false;
if (checkVersion) {
/* Existence checks, if any, go in subclasses */
/* Check that version (if there is one) is compatible. */
final Version version = getVersion();
versionFound = !version.equals(NO_VERSION);
if (!VERSION.isCompatible(version))
throw new N5Exception.N5IOException(
"Incompatible version " + version + " (this is " + VERSION + ").");
}
// if a version was found, the container exists - don't need to check again
if (checkExists && (!versionFound && !inferExistence("/")))
throw new N5Exception.N5IOException("No container exists at " + basePath);
}
private boolean inferExistence(String path) {
final JsonElement attributes = getAttributes(path);
return attributes != null || exists(path);
}
protected GsonBuilder registerGson(final GsonBuilder gsonBuilder) {
gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());
gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());
gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter());
gsonBuilder.disableHtmlEscaping();
return gsonBuilder;
}
@Override
public String getAttributesKey() {
return ATTRIBUTES_JSON;
}
@Override
public Gson getGson() {
return gson;
}
@Override
public KeyValueAccess getKeyValueAccess() {
return keyValueAccess;
}
@Override
public URI getURI() {
return uri;
}
@Override
public boolean cacheMeta() {
return cacheMeta;
}
@Override
public N5JsonCache getCache() {
return this.cache;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java
================================================
package org.janelia.saalfeldlab.n5;
import com.google.gson.GsonBuilder;
/**
* Filesystem {@link N5Writer} implementation with version compatibility check.
*
* @author Stephan Saalfeld
*/
public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyValueN5Writer {
/**
* Opens an {@link N5KeyValueWriter} at a given base path with a custom
* {@link GsonBuilder} to support custom attributes.
*
*
* If the base path does not exist, it will be created.
*
* If the base path exists and if the N5 version of the container is
* compatible with this implementation, the N5 version of this container
* will be set to the current N5 version of this implementation.
*
* @param keyValueAccess
* the backend key value access to use
* @param basePath
* n5 base path
* @param gsonBuilder
* the gson builder
* @param cacheAttributes
* Setting this to true avoids frequent reading and parsing of
* JSON encoded attributes, this is most interesting for high
* latency file systems. Changes of attributes by an independent
* writer will not be tracked.
* @throws N5Exception
* if the base path cannot be written to or cannot be created,
* if the N5 version of the container is not compatible with
* this implementation.
*/
public N5KeyValueWriter(
final KeyValueAccess keyValueAccess,
final String basePath,
final GsonBuilder gsonBuilder,
final boolean cacheAttributes)
throws N5Exception {
super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes, false);
Version version = null;
try {
version = getVersion();
if (!VERSION.isCompatible(version))
throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ").");
} catch (final NullPointerException e) {}
if (version == null || version.equals(new Version(0, 0, 0, ""))) {
createGroup("/");
setVersion("/");
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.io.UncheckedIOException;
import java.lang.reflect.Type;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
/**
* A simple structured container for hierarchies of chunked
* n-dimensional datasets and attributes.
*
* @author Stephan Saalfeld
* @see "https://github.com/axtimwalde/n5"
*/
public interface N5Reader extends AutoCloseable {
class Version {
private final int major;
private final int minor;
private final int patch;
private final String suffix;
public Version(
final int major,
final int minor,
final int patch,
final String rest) {
this.major = major;
this.minor = minor;
this.patch = patch;
this.suffix = rest;
}
public Version(
final int major,
final int minor,
final int patch) {
this(major, minor, patch, "");
}
/**
* Creates a version from a SemVer compatible version string.
*
* If the version string is null or not a SemVer version, this
* version will be "0.0.0"
*
*
* @param versionString
* the string representation of the version
*/
public Version(final String versionString) {
boolean isSemVer = false;
if (versionString != null) {
final Matcher matcher = Pattern.compile("(\\d+)(\\.(\\d+))?(\\.(\\d+))?(.*)").matcher(versionString);
isSemVer = matcher.find();
if (isSemVer) {
major = Integer.parseInt(matcher.group(1));
final String minorString = matcher.group(3);
if (!minorString.equals(""))
minor = Integer.parseInt(minorString);
else
minor = 0;
final String patchString = matcher.group(5);
if (!patchString.equals(""))
patch = Integer.parseInt(patchString);
else
patch = 0;
suffix = matcher.group(6);
} else {
major = 0;
minor = 0;
patch = 0;
suffix = "";
}
} else {
major = 0;
minor = 0;
patch = 0;
suffix = "";
}
}
public final int getMajor() {
return major;
}
public final int getMinor() {
return minor;
}
public final int getPatch() {
return patch;
}
public final String getSuffix() {
return suffix;
}
@Override
public String toString() {
final StringBuilder s = new StringBuilder();
s.append(major);
s.append(".");
s.append(minor);
s.append(".");
s.append(patch);
s.append(suffix);
return s.toString();
}
@Override
public boolean equals(final Object other) {
if (other instanceof Version) {
final Version otherVersion = (Version)other;
return (major == otherVersion.major) &
(minor == otherVersion.minor) &
(patch == otherVersion.patch) &
(suffix.equals(otherVersion.suffix));
} else
return false;
}
/**
* Returns true if this implementation is compatible with a given
* version.
*
* Currently, this means that the version is less than or equal to
* 1.X.X.
*
* @param version
* the version
* @return true if this version is compatible
*/
public boolean isCompatible(final Version version) {
return version.getMajor() <= major;
}
}
/**
* SemVer version of this N5 spec.
*/
Version NO_VERSION = new Version(0, 0, 0);
/**
* SemVer version of this N5 spec.
*/
Version VERSION = new Version(4, 0, 0);
/**
* Version attribute key.
*/
String VERSION_KEY = "n5";
/**
* Get the SemVer version of this container as specified in the 'version'
* attribute of the root group.
*
* If no version is specified or the version string does not conform to
* the SemVer format, 0.0.0 will be returned. For incomplete versions,
* such as 1.2, the missing elements are filled with 0, i.e. 1.2.0 in this
* case.
*
* @return the version
* @throws N5Exception
* the exception
*/
default Version getVersion() throws N5Exception {
return new Version(getAttribute("/", VERSION_KEY, String.class));
}
/**
* Returns the URI of the container's root.
*
* @return the base path URI
*/
URI getURI();
/**
* Reads an attribute.
*
* @param pathName
* group path
* @param key
* the key
* @param clazz
* attribute class
* @param
* the attribute type
* @return the attribute
* @throws N5Exception
* the exception
*/
T getAttribute(
String pathName,
String key,
Class clazz) throws N5Exception;
/**
* Reads an attribute.
*
* @param pathName
* group path
* @param key
* the key
* @param type
* attribute Type (use this for specifying generic types)
* @param
* the attribute type
* @return the attribute
* @throws N5Exception
* the exception
*/
T getAttribute(
String pathName,
String key,
Type type) throws N5Exception;
/**
* Get mandatory dataset attributes.
*
* @param pathName
* dataset path
* @return dataset attributes or null if either dimensions or dataType are
* not set
* @throws N5Exception
* the exception
*/
DatasetAttributes getDatasetAttributes(String pathName) throws N5Exception;
/**
* Some implementations may need to convert arbitrary DatasetAttributes to their specific equivalent variant.
* Ideally, this method would be `protected`, but that's not valid for the interface. The default implementation
* is the identity (returns the input DatasetAttributes unchanged).
*
* The returned DatasetAttributes is not guaranteed to be a unique instance.
*
* @param attributes
* to convert
* @return the converted attributes
*/
default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes attributes) {
return attributes;
}
/**
* Reads a chunk as a {@link DataBlock}.
*
* @param
* the DataBlock data type
* @param pathName
* dataset path
* @param datasetAttributes
* the dataset attributes
* @param gridPosition
* the grid position
* @return the data block
* @throws N5Exception
* the exception
*/
DataBlock readChunk(
String pathName,
DatasetAttributes datasetAttributes,
long... gridPosition) throws N5Exception;
/**
* Reads multiple chunks as {@link DataBlock}s.
*
* Implementations may optimize / batch read operations when possible, e.g.
* in the case that the datasets are sharded.
*
* @param
* the DataBlock data type
* @param pathName
* dataset path
* @param datasetAttributes
* the dataset attributes
* @param gridPositions
* a list of grid positions
* @return a list of data blocks
* @throws N5Exception
* the exception
*/
default List> readChunks(
final String pathName,
final DatasetAttributes datasetAttributes,
final List gridPositions) throws N5Exception {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
final ArrayList> blocks = new ArrayList<>();
for( final long[] p : gridPositions )
blocks.add(readChunk(pathName, convertedDatasetAttributes, p));
return blocks;
}
/**
* Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded).
*
* A block is the highest (coarsest) level of the dataset's {@link NestedGrid}
* This method's behavior is identical to {@link #readChunk} for un-sharded datasets.
*
* @param
* the DataBlock data type
* @param pathName
* dataset path
* @param datasetAttributes
* the dataset attributes
* @param gridPosition
* the position in the block grid
* @return the data block
* @throws N5Exception
* the exception
*
* @see DatasetAttributes#getNestedBlockGrid()
*/
DataBlock readBlock(
String pathName,
DatasetAttributes datasetAttributes,
long... gridPosition) throws N5Exception;
/**
* Checks if a block exists at the given grid position without reading the data.
*
* A block is the highest (coarsest) level of the dataset's {@link
* NestedGrid}, that is, a shard if the dataset is sharded, or a chunk
* otherwise.
*
* This method only checks for the presence of the key value for the gridPosition, it does not
* read or validate the contents. As a result, this method refers to chunks in un-sharded datasets
* or shards in sharded datasets.
*
* @param pathName
* dataset path
* @param datasetAttributes
* the dataset attributes
* @param gridPosition
* the block grid position
* @return true if the block file exists
* @throws N5Exception
* the exception
*/
boolean blockExists(
String pathName,
DatasetAttributes datasetAttributes,
long... gridPosition) throws N5Exception;
/**
* Load a {@link DataBlock} as a {@link Serializable}. The offset is given
* in
* {@link DataBlock} grid coordinates.
*
* @param dataset
* the dataset path
* @param attributes
* the dataset attributes
* @param
* the data block type
* @param gridPosition
* the grid position
* @return the data block
* @throws N5Exception
* the exception
* @throws ClassNotFoundException
* the class not found exception
*/
default T readSerializedBlock(
final String dataset,
final DatasetAttributes attributes,
final long... gridPosition) throws N5Exception, ClassNotFoundException {
DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(attributes);
final DataBlock block = readChunk(dataset, convertedDatasetAttributes, gridPosition);
if (block == null)
return null;
final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.getData());
try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) {
return (T) in.readObject();
} catch (final IOException | UncheckedIOException e) {
throw new N5Exception.N5IOException(e);
}
}
/**
* Test whether a group or dataset exists at a given path.
*
* @param pathName
* group path
* @return true if the path exists
*/
boolean exists(String pathName);
/**
* Test whether a dataset exists at a given path.
*
* @param pathName
* dataset path
* @return true if a dataset exists
* @throws N5Exception
* an exception is thrown if the existence of a dataset at the given path cannot be determined.
*/
default boolean datasetExists(final String pathName) throws N5Exception {
return exists(pathName) && getDatasetAttributes(pathName) != null;
}
/**
* List all groups (including datasets) in a group.
*
* @param pathName
* group path
* @return list of children
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
String[] list(String pathName) throws N5Exception;
/**
* Recursively list all groups and datasets in the given path.
* Only paths that satisfy the provided filter will be included, but the
* children of paths that were excluded may be included (filter does not
* apply to the subtree).
*
* @param pathName
* base group path
* @param filter
* filter for children to be included
* @return list of child groups and datasets
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
default String[] deepList(
final String pathName,
final Predicate filter) throws N5Exception {
final String groupSeparator = getGroupSeparator();
final String normalPathName = pathName
.replaceAll(
"(^" + groupSeparator + "*)|(" + groupSeparator + "*$)",
"");
final List absolutePaths = deepList(this, normalPathName, false, filter);
return absolutePaths
.stream()
.map(a -> a.replaceFirst(normalPathName + "(" + groupSeparator + "?)", ""))
.filter(a -> !a.isEmpty())
.toArray(String[]::new);
}
/**
* Recursively list all groups and datasets in the given path.
*
* @param pathName
* base group path
* @return list of groups and datasets
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
default String[] deepList(final String pathName) throws N5Exception {
return deepList(pathName, a -> true);
}
/**
* Recursively list all datasets in the given path. Only paths that satisfy the
* provided filter will be included, but the children of paths that were
* excluded may be included (filter does not apply to the subtree).
*
*
* This method delivers the same results as
*
*
*
* {@code
* n5.deepList(prefix, a -> {
* try {
* return n5.datasetExists(a) && filter.test(a);
* } catch (final N5Exception e) {
* return false;
* }
* });
* }
*
*
* but will execute {@link #datasetExists(String)} only once per node. This can
* be relevant for performance on high latency backends such as cloud stores.
*
*
* @param pathName base group path
* @param filter filter for datasets to be included
* @return list of groups
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
default String[] deepListDatasets(
final String pathName,
final Predicate filter) throws N5Exception {
final String groupSeparator = getGroupSeparator();
final String normalPathName = pathName
.replaceAll(
"(^" + groupSeparator + "*)|(" + groupSeparator + "*$)",
"");
final List absolutePaths = deepList(this, normalPathName, true, filter);
return absolutePaths
.stream()
.map(a -> a.replaceFirst(normalPathName + "(" + groupSeparator + "?)", ""))
.filter(a -> !a.isEmpty())
.toArray(String[]::new);
}
/**
* Recursively list all including datasets in the given path.
*
*
* This method delivers the same results as
*
*
*
* {@code
* n5.deepList(prefix, a -> {
* try {
* return n5.datasetExists(a);
* } catch (final N5Exception e) {
* return false;
* }
* });
* }
*
*
* but will execute {@link #datasetExists(String)} only once per node. This
* can
* be relevant for performance on high latency backends such as cloud
* stores.
*
*
* @param pathName
* base group path
* @return list of groups
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
default String[] deepListDatasets(final String pathName) throws N5Exception {
return deepListDatasets(pathName, a -> true);
}
/**
* Helper method to recursively list all groups and datasets. This method is not part of the
* public API and is accessible only because Java 8 does not support private
* interface methods yet.
*
* TODO make private when committing to Java versions newer than 8
*
* @param n5 the n5 reader
* @param pathName the base group path
* @param datasetsOnly true if only dataset paths should be returned
* @param filter a dataset filter
* @return the list of all children
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
*/
static ArrayList deepList(
final N5Reader n5,
final String pathName,
final boolean datasetsOnly,
final Predicate filter) throws N5Exception {
final ArrayList children = new ArrayList<>();
final boolean isDataset = n5.datasetExists(pathName);
final boolean passDatasetTest = datasetsOnly && !isDataset;
if (!passDatasetTest && filter.test(pathName))
children.add(pathName);
if (!isDataset) {
final String groupSeparator = n5.getGroupSeparator();
final String[] baseChildren = n5.list(pathName);
for (final String child : baseChildren)
children.addAll(deepList(n5, pathName + groupSeparator + child, datasetsOnly, filter));
}
return children;
}
/**
* Recursively list all groups (including datasets) in the given group, in
* parallel, using the given {@link ExecutorService}. Only paths that
* satisfy
* the provided filter will be included, but the children of paths that were
* excluded may be included (filter does not apply to the subtree).
*
* @param pathName
* base group path
* @param filter
* filter for children to be included
* @param executor
* executor service
* @return list of datasets
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
* @throws ExecutionException
* the execution exception
* @throws InterruptedException
* the interrupted exception
*/
default String[] deepList(
final String pathName,
final Predicate filter,
final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {
final String groupSeparator = getGroupSeparator();
final String normalPathName = pathName.replaceAll("(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", "");
final ArrayList results = new ArrayList();
final LinkedBlockingQueue> datasetFutures = new LinkedBlockingQueue<>();
deepListHelper(this, normalPathName, false, filter, executor, datasetFutures);
while (!datasetFutures.isEmpty()) {
final String result = datasetFutures.poll().get();
if (result != null && !result.equals(normalPathName))
results.add(result.substring(normalPathName.length() + groupSeparator.length()));
}
return results.toArray(new String[0]);
}
/**
* Recursively list all groups (including datasets) in the given group, in
* parallel, using the given {@link ExecutorService}.
*
* @param pathName
* base group path
* @param executor
* executor service
* @return list of groups
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
* @throws ExecutionException
* the execution exception
* @throws InterruptedException
* this exception is thrown if execution is interrupted
*/
default String[] deepList(
final String pathName,
final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {
return deepList(pathName, a -> true, executor);
}
/**
* Recursively list all datasets in the given group, in parallel, using the
* given {@link ExecutorService}. Only paths that satisfy the provided
* filter
* will be included, but the children of paths that were excluded may be
* included (filter does not apply to the subtree).
*
*
* This method delivers the same results as
*
*
*
* {@code
* n5.deepList(prefix, a -> {
* try {
* return n5.datasetExists(a) && filter.test(a);
* } catch (final N5Exception e) {
* return false;
* }
* }, exec);
* }
*
*
* but will execute {@link #datasetExists(String)} only once per node. This
* can
* be relevant for performance on high latency backends such as cloud
* stores.
*
*
* @param pathName
* base group path
* @param filter
* filter for datasets to be included
* @param executor
* executor service
* @return list of datasets
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
* @throws ExecutionException
* the execution exception
* @throws InterruptedException
* this exception is thrown if execution is interrupted
*/
default String[] deepListDatasets(
final String pathName,
final Predicate filter,
final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {
final String groupSeparator = getGroupSeparator();
final String normalPathName = pathName.replaceAll("(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", "");
final ArrayList results = new ArrayList();
final LinkedBlockingQueue> datasetFutures = new LinkedBlockingQueue<>();
deepListHelper(this, normalPathName, true, filter, executor, datasetFutures);
datasetFutures.poll().get(); // skip self
while (!datasetFutures.isEmpty()) {
final String result = datasetFutures.poll().get();
if (result != null)
results.add(result.substring(normalPathName.length() + groupSeparator.length()));
}
return results.toArray(new String[0]);
}
/**
* Recursively list all datasets in the given group, in parallel, using the
* given {@link ExecutorService}.
*
*
* This method delivers the same results as
*
*
*
* {@code
* n5.deepList(prefix, a -> {
* try {
* return n5.datasetExists(a);
* } catch (final N5Exception e) {
* return false;
* }
* }, exec);
* }
*
*
* but will execute {@link #datasetExists(String)} only once per node. This
* can
* be relevant for performance on high latency backends such as cloud
* stores.
*
*
* @param pathName
* base group path
* @param executor
* executor service
* @return list of groups
* @throws N5Exception
* an exception is thrown if pathName is not a valid group
* @throws ExecutionException
* the execution exception
* @throws InterruptedException
* this exception is thrown if execution is interrupted
*/
default String[] deepListDatasets(
final String pathName,
final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException {
return deepListDatasets(pathName, a -> true, executor);
}
/**
* Helper method for parallel deep listing. This method is not part of the
* public API and is accessible only because Java 8 does not support private
* interface methods yet.
*
* TODO make private when committing to Java versions newer than 8
*
* @param n5
* the n5 reader
* @param path
* the base path
* @param datasetsOnly
* true if only dataset paths should be returned
* @param filter
* filter for datasets to be included
* @param executor
* the executor service
* @param datasetFutures
* result futures
*/
static void deepListHelper(
final N5Reader n5,
final String path,
final boolean datasetsOnly,
final Predicate filter,
final ExecutorService executor,
final LinkedBlockingQueue> datasetFutures) {
final String groupSeparator = n5.getGroupSeparator();
datasetFutures.add(executor.submit(() -> {
boolean isDataset = false;
try {
isDataset = n5.datasetExists(path);
} catch (final N5Exception e) {}
if (!isDataset) {
String[] children = null;
try {
children = n5.list(path);
for (final String child : children) {
final String fullChildPath = path + groupSeparator + child;
deepListHelper(n5, fullChildPath, datasetsOnly, filter, executor, datasetFutures);
}
} catch (final N5Exception e) {}
}
final boolean passDatasetTest = datasetsOnly && !isDataset;
return !passDatasetTest && filter.test(path) ? path : null;
}));
}
/**
* List all attributes and their class of a group.
*
* @param pathName
* group path
* @return a map of attribute keys to their inferred class
* @throws N5Exception if an error occurred during listing
*/
Map> listAttributes(String pathName) throws N5Exception;
/**
* Returns the symbol that is used to separate nodes in a group path.
*
* @return the group separator
*/
default String getGroupSeparator() {
return "/";
}
/**
* Creates a group path by concatenating all nodes with the node separator
* defined by {@link #getGroupSeparator()}. The string will not have a
* leading or trailing node separator symbol.
*
* @param nodes a collection of child node names
* @return the full group path
*/
default String groupPath(final String... nodes) {
if (nodes == null || nodes.length == 0)
return "";
final String groupSeparator = getGroupSeparator();
final StringBuilder builder = new StringBuilder(nodes[0]);
for (int i = 1; i < nodes.length; ++i) {
builder.append(groupSeparator);
builder.append(nodes[i]);
}
return builder.toString();
}
/**
* Default implementation of {@link AutoCloseable#close()} for all
* implementations that do not hold any closable resources.
*/
@Override
default void close() {}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5URI.java
================================================
package org.janelia.saalfeldlab.n5;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* A {@link URI} for N5 containers, groups, datasets, and attributes.
*
* Container paths are stored in the URI path. Group / dataset paths are stored in the URI query,
* and attribute paths are stored in the URI fragment.
*/
public class N5URI {
private static final Charset UTF8 = StandardCharsets.UTF_8;
public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]");
final URI uri;
private final String scheme;
private final String container;
private final String group;
private final String attribute;
public N5URI(final String uri) throws URISyntaxException {
this(encodeAsUri(uri));
}
public N5URI(final URI uri) {
this.uri = uri;
scheme = uri.getScheme() == null ? null : uri.getScheme();
final String schemeSpecificPartWithoutQuery = getSchemeSpecificPartWithoutQuery();
if (uri.getScheme() == null) {
container = schemeSpecificPartWithoutQuery.replaceFirst("//", "");
} else {
container = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery;
}
group = uri.getQuery();
attribute = decodeFragment(uri.getRawFragment());
}
/**
* @return the container path
*/
public String getContainerPath() {
return container;
}
public URI getURI() {
return uri;
}
/**
* @return the group path, or root ("/") if none was provided
*/
public String getGroupPath() {
return group != null ? group : "/";
}
/**
* @return the normalized group path
*/
public String normalizeGroupPath() {
return normalizeGroupPath(getGroupPath());
}
/**
* @return the attribute path, or root ("/") if none was provided
*/
public String getAttributePath() {
return attribute != null ? attribute : "/";
}
/**
* @return the normalized attribute path
*/
public String normalizeAttributePath() {
return normalizeAttributePath(getAttributePath());
}
/**
* Parse this {@link N5URI} as a {@link LinkedAttributePathToken}.
*
* @see N5URI#getAttributePathTokens(String)
* @return the linked attribute path token
*/
public LinkedAttributePathToken getAttributePathTokens() {
return getAttributePathTokens(normalizeAttributePath());
}
/**
* Parses the {@link String normalizedAttributePath} to a list of
* {@link LinkedAttributePathToken}.
* This is useful for traversing or constructing a json representation of
* the provided {@link String normalizedAttributePath}.
* Note that {@link String normalizedAttributePath} should be normalized
* prior to generating this list
*
* @param normalizedAttributePath
* to parse into {@link LinkedAttributePathToken}s
* @return the head of the {@link LinkedAttributePathToken}s
*/
public static LinkedAttributePathToken getAttributePathTokens(final String normalizedAttributePath) {
final String[] attributePathParts = normalizedAttributePath.replaceAll("^/", "").split("(?> firstTokenRef = new AtomicReference<>();
final AtomicReference> currentTokenRef = new AtomicReference<>();
final Consumer> updateCurrentToken = (newToken) -> {
if (firstTokenRef.get() == null) {
firstTokenRef.set(newToken);
currentTokenRef.set(firstTokenRef.get());
} else {
final LinkedAttributePathToken currentToken = currentTokenRef.get();
currentToken.childToken = newToken;
currentTokenRef.set(newToken);
}
};
for (final String pathPart : attributePathParts) {
final Matcher matcher = ARRAY_INDEX.matcher(pathPart);
final LinkedAttributePathToken newToken;
if (matcher.matches()) {
final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", ""));
newToken = new LinkedAttributePathToken.ArrayAttributeToken(index);
} else {
final String pathPartUnEscaped = pathPart.replaceAll("\\\\/", "/").replaceAll("\\\\\\[", "[");
newToken = new LinkedAttributePathToken.ObjectAttributeToken(pathPartUnEscaped);
}
updateCurrentToken.accept(newToken);
}
return firstTokenRef.get();
}
private String getSchemePart() {
return scheme == null ? "" : scheme + "://";
}
private String getContainerPart() {
return container;
}
private String getGroupPart() {
return group == null ? "" : "?" + group;
}
private String getAttributePart() {
return attribute == null ? "" : "#" + attribute;
}
@Override
public String toString() {
return getContainerPart() + getGroupPart() + getAttributePart();
}
private String getSchemeSpecificPartWithoutQuery() {
/* Why not substring "?"? */
return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), "");
}
/**
* N5URI is always considered absolute if a scheme is provided.
* If no scheme is provided, the N5URI is absolute if it starts with either
* "/" or "[A-Z]:"
*
* @return if the path for this N5URI is absolute
*/
public boolean isAbsolute() {
if (scheme != null)
return true;
final String path = uri.getPath();
if (!path.isEmpty()) {
final char char0 = path.charAt(0);
return char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z');
}
return false;
}
/**
* Generate a new N5URI which is the result of resolving {@link N5URI
* relativeN5Url} to this {@link N5URI}.
* If relativeN5Url is not relative to this N5URI, then the resulting N5URI
* is equivalent to relativeN5Url.
*
* @param relativeN5Url
* N5URI to resolve against ourselves
* @return the result of the resolution.
* @throws URISyntaxException
* if the uri is malformed
*/
public N5URI resolve(final N5URI relativeN5Url) throws URISyntaxException {
final URI thisUri = uri;
final URI relativeUri = relativeN5Url.uri;
final StringBuilder newUri = new StringBuilder();
if (relativeUri.getScheme() != null) {
return relativeN5Url;
}
final String thisScheme = thisUri.getScheme();
if (thisScheme != null) {
newUri.append(thisScheme).append(":");
}
if (relativeUri.getAuthority() != null) {
newUri
.append(relativeUri.getAuthority())
.append(relativeUri.getPath())
.append(relativeN5Url.getGroupPart())
.append(relativeN5Url.getAttributePart());
return new N5URI(newUri.toString());
}
final String thisAuthority = thisUri.getAuthority();
if (thisAuthority != null) {
newUri.append("//").append(thisAuthority);
}
final String path = relativeUri.getPath();
if (!path.isEmpty()) {
if (!relativeN5Url.isAbsolute()) {
newUri.append(thisUri.getPath()).append('/');
}
newUri
.append(path)
.append(relativeN5Url.getGroupPart())
.append(relativeN5Url.getAttributePart());
return new N5URI(newUri.toString());
}
newUri.append(thisUri.getPath());
final String query = relativeUri.getQuery();
if (query != null) {
if (query.charAt(0) != '/' && thisUri.getQuery() != null) {
newUri.append(this.getGroupPart()).append('/');
newUri.append(relativeUri.getQuery());
} else {
newUri.append(relativeN5Url.getGroupPart());
}
newUri.append(relativeN5Url.getAttributePart());
return new N5URI(newUri.toString());
}
newUri.append(this.getGroupPart());
final String fragment = relativeUri.getFragment();
if (fragment != null) {
if (fragment.charAt(0) != '/' && thisUri.getFragment() != null) {
newUri.append(this.getAttributePart()).append('/');
} else {
newUri.append(relativeN5Url.getAttributePart());
}
return new N5URI(newUri.toString());
}
newUri.append(this.getAttributePart());
return new N5URI(newUri.toString());
}
/**
* Generate a new N5URI which is the result of resolving {@link URI
* relativeUri} to this {@link N5URI}.
* If relativeUri is not relative to this N5URI, then the resulting N5URI is
* equivalent to relativeUri.
*
* @param relativeUri
* URI to resolve against ourselves
* @return the result of the resolution.
* @throws URISyntaxException
* if the uri is malformed
*/
public N5URI resolve(final URI relativeUri) throws URISyntaxException {
return resolve(new N5URI(relativeUri));
}
/**
* Generate a new N5URI which is the result of resolving {@link String
* relativeString} to this {@link N5URI}
* If relativeString is not relative to this N5URI, then the resulting N5URI
* is equivalent to relativeString.
*
* @param relativeString
* String to resolve against ourselves
* @return the result of the resolution.
* @throws URISyntaxException
* if the uri is malformed
*/
public N5URI resolve(final String relativeString) throws URISyntaxException {
return resolve(new N5URI(relativeString));
}
/**
* Normalize a POSIX path, resulting in removal of redundant "/", "./", and
* resolution of relative "../".
*
* NOTE: currently a private helper method only used by {@link N5URI#normalizeGroupPath(String)}.
* It's safe to do in that case since relative group paths should always be POSIX compliant.
* A new helper method to understand other path types (e.g. Windows) may be necessary eventually.
*
* @param path
* to normalize
* @return the normalized path
*/
private static String normalizePath(String path) {
path = path == null ? "" : path;
final char[] pathChars = path.toCharArray();
final List tokens = new ArrayList<>();
final StringBuilder curToken = new StringBuilder();
boolean escape = false;
for (final char character : pathChars) {
/* Skip if we last saw escape */
if (escape) {
escape = false;
curToken.append(character);
continue;
}
/* Check if we are escape character */
if (character == '\\') {
escape = true;
} else if (character == '/') {
if (tokens.isEmpty() && curToken.length() == 0) {
/* If we are root, and the first token, then add the '/' */
curToken.append(character);
}
/*
* The current token is complete, add it to the list, if it
* isn't empty
*/
final String newToken = curToken.toString();
if (!newToken.isEmpty()) {
/*
* If our token is '..' then remove the last token instead
* of adding a new one
*/
if (newToken.equals("..")) {
tokens.remove(tokens.size() - 1);
} else {
tokens.add(newToken);
}
}
/* reset for the next token */
curToken.setLength(0);
} else {
curToken.append(character);
}
}
final String lastToken = curToken.toString();
if (!lastToken.isEmpty()) {
if (lastToken.equals("..")) {
tokens.remove(tokens.size() - 1);
} else {
tokens.add(lastToken);
}
}
if (tokens.isEmpty())
return "";
String root = "";
if (tokens.get(0).equals("/")) {
tokens.remove(0);
root = "/";
}
return root + tokens
.stream()
.filter(it -> !it.equals("."))
.filter(it -> !it.isEmpty())
.reduce((l, r) -> l + "/" + r)
.orElse("");
}
/**
* Normalize a group path relative to a container's root, resulting in
* removal of redundant "/", "./", resolution of relative "../",
* and removal of leading slashes.
*
* @param path
* to normalize
* @return the normalized path
*/
public static String normalizeGroupPath(final String path) {
/*
* Alternatively, could do something like the below in every
* KeyValueReader implementation
*
* return keyValueAccess.relativize( N5URI.normalizeGroupPath(path),
* basePath);
*
* has to be in the implementations, since KeyValueAccess doesn't have a
* basePath.
*/
return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path);
}
private enum N5UriPattern {
/**
* matches any `/` or `[N]` where `N` is non-negative
*/
MULTI_PART_ATTRIBUTE(Pattern.compile(".*((?\\[[0-9]+])")),
/**
* matches `A[N]` where A is some non-empty preceding path
*/
ATTRIBUTE_ARRAY_EXCEPT_START(Pattern.compile("((?\\[[0-9]+]))")),
/**
* The following Pattern has 4 possible matches.
* It is intended to be used to remove matching portions iteratively until no further matches are found:
*
* The first 3 matches can remove redundant separators of the form:
*
* - (?<=/)/+ : `a///b` -> `a/b`
* - (?<=(/|^))(\./)+ : `a/./b` -> `a/b`
* - ((/|(?<=/))\.)$ : `a/b/` -> `a/b`
*
* The next match avoids removing `/` when it is NOT redundant (e.g. only character, or escaped):
*
* - (? `/ , `/a/b/\\/` -> `/a/b/\\/`
*
* The last match resolves relative paths:
*
* - 5. ((?<=^/)|^|(?<=(/|^))[^/]+(?
*
* - `a/../b` -> `b`
* - `/a/../b` -> `/b`
* - `../a/../b` -> `b`
* - `/../a/../b` -> `/b`
* - `/../a/../../b` -> `/b`
*
*
*
*/
RELATIVE_ATTRIBUTE_PARTS(Pattern.compile( "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(?
* Attribute paths have a few special characters:
*
*
* When normalizing:
*
* - "/" are added before and after any indexing brackets [N]
* - any redundant "/" are removed
* - any relative ".." and "." are resolved
*
*
* Examples of valid attribute paths, and their normalizations
*
* - /a/b/c becomes /a/b/c
* - /a/b/c/ becomes /a/b/c
* - ///a///b///c becomes /a/b/c
* - /a/././b/c becomes /a/b/c
* - /a/b[1]c becomes /a/b/[1]/c
* - /a/b/[1]c becomes /a/b/[1]/c
* - /a/b[1]/c becomes /a/b/[1]/c
* - /a/b[1]/c/.. becomes /a/b/[1]
*
*
* @param attributePath
* to normalize
* @return the normalized attribute path
*/
public static String normalizeAttributePath(final String attributePath) {
/*
* Short circuit if there are no non-escaped `/` or array indices (e.g.
* [N] where N is a non-negative integer)
*/
if (!N5UriPattern.MULTI_PART_ATTRIBUTE.matches(attributePath))
return attributePath;
/* Add separator after arrays at the beginning `[10]b` -> `[10]/b` */
final String attrPathPlusFirstIndexSeparator = N5UriPattern.appendSlashAfterArrayStart(attributePath);
/*
* Add separator before and after arrays not at the beginning `a[10]b`
* -> `a/[10]/b`
*/
final String attrPathPlusIndexSeparators = N5UriPattern.addSlashAroundArrayExceptStart(attrPathPlusFirstIndexSeparator);
/* remove relative path parts like `./`, `../`, `a//b, `a/b/` */
return N5UriPattern.removeRelativePathParts(attrPathPlusIndexSeparators);
}
/**
* If uri is a valid URI, just return it as a URI. Else, encode if possible.
*
* @param uri as String to get as URI
* @return URI from input. Encoded if necessary
*/
public static URI getAsUri(final String uri) throws N5Exception {
try {
return URI.create(uri);
} catch (Exception ignore) {
try {
return N5URI.encodeAsUri(uri);
} catch (URISyntaxException e) {
throw new N5Exception("Could not encode as URI: " + uri, e);
}
}
}
public static URI encodeAsUriPath(final String path) {
try {
return new URI(null, null, path, null);
} catch (Exception e) {
throw new IllegalArgumentException("Could not encode as URI path component: " + path, e);
}
}
/**
* Encode the inpurt {@link String uri} so that illegal characters are
* properly escaped prior to generating the resulting {@link URI}.
*
* @param uri
* to encode
* @return the {@link URI} created from encoding the {@link String uri}
* @throws URISyntaxException
* if the provided String is not a valid URI
*/
public static URI encodeAsUri(final String uri) throws URISyntaxException {
if (uri.trim().length() == 0) {
//TODO Caleb: ???
return new URI(uri);
}
/*
* find last # symbol to split fragment on. If we don't remove it first,
* then it will encode it, and not parse it separately
* after we remove the temporary _N5 scheme
*/
final int fragmentIdx = uri.lastIndexOf('#');
final String uriWithoutFragment;
final String fragment;
if (fragmentIdx >= 0) {
uriWithoutFragment = uri.substring(0, fragmentIdx);
fragment = uri.substring(fragmentIdx + 1);
} else {
uriWithoutFragment = uri;
fragment = null;
}
/* Edge case to handle when uriWithoutFragment is empty */
final URI _n5Uri;
if (uriWithoutFragment.length() == 0 && fragment != null && fragment.length() > 0) {
_n5Uri = new URI("N5Internal", "//STAND_IN", fragment);
} else {
_n5Uri = new URI("N5Internal", uriWithoutFragment, fragment);
}
final URI n5Uri;
if (fragment == null) {
n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart());
} else {
if (Objects.equals(_n5Uri.getPath(), "") && Objects.equals(_n5Uri.getAuthority(), "STAND_IN")) {
n5Uri = new URI("#" + _n5Uri.getRawFragment());
} else {
n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart() + "#" + _n5Uri.getRawFragment());
}
}
return n5Uri;
}
/**
* Generate an {@link N5URI} from a container, group, and attribute
*
* @param container
* of the N5Url
* @param group
* of the N5Url
* @param attribute
* of the N5Url
* @return the {@link N5URI}
* @throws URISyntaxException
* if the uri is malformed
*/
public static N5URI from(
final String container,
final String group,
final String attribute) throws URISyntaxException {
final String containerPart = container != null ? container : "";
final String groupPart = group != null ? "?" + group : "";
final String attributePart = attribute != null ? "#" + attribute : "";
return new N5URI(containerPart + groupPart + attributePart);
}
/**
* Intentionally copied from {@link URI} for internal use
*
* @see URI#decode(char)
*/
@SuppressWarnings("JavadocReference")
private static int decode(final char c) {
if ((c >= '0') && (c <= '9'))
return c - '0';
if ((c >= 'a') && (c <= 'f'))
return c - 'a' + 10;
if ((c >= 'A') && (c <= 'F'))
return c - 'A' + 10;
assert false;
return -1;
}
/**
* Intentionally copied from {@link URI} for internal use
*
* @see URI#decode(char, char)
*/
@SuppressWarnings("JavadocReference")
private static byte decode(final char c1, final char c2) {
return (byte)(((decode(c1) & 0xf) << 4)
| ((decode(c2) & 0xf) << 0));
}
/**
* Modified from {@link URI#decode(String)} to ignore the listed exception,
* where it doesn't decode escape values inside square braces.
*
* As an example of the original implementation, a backslash inside a square
* brace would be encoded to "[%5C]", and
* when calling {@code decode("[%5C]")} it would not decode to "[\]" since
* the encode escape sequence is inside square braces.
*
* We keep all the decoding logic in this modified version, EXCEPT, that we
* don't check for and ignore encoded sequences inside square braces.
*
* Thus, {@code decode("[%5C]")} -> "[\]".
*
* @see URI#decode(char, char)
*/
@SuppressWarnings("JavadocReference")
private static String decodeFragment(final String rawFragment) {
if (rawFragment == null)
return rawFragment;
final int n = rawFragment.length();
if (n == 0)
return rawFragment;
if (rawFragment.indexOf('%') < 0)
return rawFragment;
final StringBuffer sb = new StringBuffer(n);
final ByteBuffer bb = ByteBuffer.allocate(n);
final CharBuffer cb = CharBuffer.allocate(n);
final CharsetDecoder dec = UTF8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
// This is not horribly efficient, but it will do for now
char c = rawFragment.charAt(0);
for (int i = 0; i < n;) {
assert c == rawFragment.charAt(i); // Loop invariant
if (c != '%') {
sb.append(c);
if (++i >= n)
break;
c = rawFragment.charAt(i);
continue;
}
bb.clear();
for (;;) {
assert (n - i >= 2);
bb.put(decode(rawFragment.charAt(++i), rawFragment.charAt(++i)));
if (++i >= n)
break;
c = rawFragment.charAt(i);
if (c != '%')
break;
}
bb.flip();
cb.clear();
dec.reset();
CoderResult cr = dec.decode(bb, cb, true);
assert cr.isUnderflow();
cr = dec.flush(cb);
assert cr.isUnderflow();
sb.append(cb.flip());
}
return sb.toString();
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.io.UncheckedIOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
/**
* A simple structured container API for hierarchies of chunked
* n-dimensional datasets and attributes.
*
* @author Stephan Saalfeld
* @see "https://github.com/axtimwalde/n5"
*/
public interface N5Writer extends N5Reader {
/**
* Sets an attribute.
*
* @param groupPath group path
* @param attributePath the key
* @param attribute the attribute
* @param the attribute type
* @throws N5Exception the exception
*/
default void setAttribute(
final String groupPath,
final String attributePath,
final T attribute) throws N5Exception {
setAttributes(groupPath, Collections.singletonMap(attributePath, attribute));
}
/**
* Sets a map of attributes. The passed attributes are inserted into the
* existing attribute tree. New attributes, including their parent
* objects will be added, existing attributes whose paths are not included
* will remain unchanged, those whose paths are included will be overridden.
*
* @param groupPath group path
* @param attributes the attribute map of attribute paths and values
* @throws N5Exception the exception
*/
void setAttributes(
String groupPath,
Map attributes) throws N5Exception;
/**
* Remove the attribute from group {@code pathName} with key {@code key}.
*
* @param groupPath group path
* @param attributePath of attribute to remove
* @return true if attribute removed, else false
* @throws N5Exception the exception
*/
boolean removeAttribute(String groupPath, String attributePath) throws N5Exception;
/**
* Remove the attribute from group {@code pathName} with key {@code key} and
* type {@code T}.
*
* If an attribute at {@code pathName} and {@code key} exists, but is not of
* type {@code T}, it is not removed.
*
* @param groupPath group path
* @param attributePath of attribute to remove
* @param clazz of the attribute to remove
* @param of the attribute
* @return the removed attribute, as {@code T}, or {@code null} if no
* matching attribute
* @throws N5Exception if removing he attribute failed, parsing the attribute failed, or the attribute cannot be interpreted as T
*/
T removeAttribute(String groupPath, String attributePath, Class clazz) throws N5Exception;
/**
* Remove attributes as provided by {@code attributes}.
*
* If any element of {@code attributes} does not exist, it will be ignored.
* If at least one attribute from {@code attributes} is removed, this will
* return {@code true}.
*
*
* @param groupPath group path
* @param attributePaths to remove
* @return true if any of the listed attributes were removed
* @throws N5Exception the exception
*/
default boolean removeAttributes(final String groupPath, final List attributePaths) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(groupPath);
boolean removed = false;
for (final String attribute : attributePaths) {
removed |= removeAttribute(normalPath, N5URI.normalizeAttributePath(attribute));
}
return removed;
}
/**
* Sets mandatory dataset attributes.
*
* @param datasetPath dataset path
* @param datasetAttributes the dataset attributes
* @throws N5Exception the exception
*/
default void setDatasetAttributes(
final String datasetPath,
final DatasetAttributes datasetAttributes) throws N5Exception {
setAttributes(datasetPath, getConvertedDatasetAttributes(datasetAttributes).asMap());
}
/**
* Set the SemVer version of this container as specified in the
* {@link N5Reader#VERSION_KEY} attribute of the root group. This default
* implementation writes the version only if the current version is not
* equal {@link N5Reader#VERSION}.
*
* @throws N5Exception the exception
*/
default void setVersion() throws N5Exception {
if (!VERSION.equals(getVersion()))
setAttribute("/", VERSION_KEY, VERSION.toString());
}
/**
* Creates a group (directory)
*
* @param groupPath the path
* @throws N5Exception the exception
*/
void createGroup(String groupPath) throws N5Exception;
/**
* Removes a group or dataset (directory and all contained files).
*
*
* {@link #remove(String) remove("")} or
* {@link #remove(String) remove("")} will delete this N5
* container. Please note that no checks for safety will be performed,
* e.g. {@link #remove(String) remove("..")} will try to
* recursively delete the parent directory of this N5 container which
* only fails because it attempts to delete the parent directory before it
* is empty.
*
* @param groupPath group path
* @return true if removal was successful, false otherwise
* @throws N5Exception the exception
*/
boolean remove(String groupPath) throws N5Exception;
/**
* Removes the N5 container.
*
* @return true if removal was successful, false otherwise
* @throws N5Exception the exception
*/
default boolean remove() throws N5Exception {
return remove("/");
}
/**
* Creates a dataset. This does not create any data but the path and
* mandatory attributes only. The returned DatasetAttributes should be used
* for future read/write operations on this dataset. It may not be the same
* DatasetAttributes object that was provided, depending on the implementation.
*
* @param datasetPath dataset path
* @param datasetAttributes the dataset attributes
* @return DatasetAttributes optimal attributes object to be used for read/write operations
* @throws N5Exception
*/
default DatasetAttributes createDataset(
final String datasetPath,
final DatasetAttributes datasetAttributes) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(datasetPath);
createGroup(normalPath);
final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
setDatasetAttributes(normalPath, convertedDatasetAttributes);
return convertedDatasetAttributes;
}
/**
* Creates a dataset. This does not create any data but the path and
* mandatory attributes only. Returns the DatasetAttributes object to be
* used for future read/write operations on this dataset.
*
* @param datasetPath dataset path
* @param dimensions the dataset dimensions
* @param blockSize the block size
* @param dataType the data type
* @param compression the compression
* @return DatasetAttributes optimal attributes object to be used for read/write operations
* @throws N5Exception
*/
default DatasetAttributes createDataset(
final String datasetPath,
final long[] dimensions,
final int[] blockSize,
final DataType dataType,
final Compression compression) throws N5Exception {
return createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));
}
/**
* Writes a chunk represented by a {@link DataBlock}.
*
* @param datasetPath dataset path
* @param datasetAttributes the dataset attributes
* @param chunk the chunk as a DataBlock
* @param the data block data type
* @throws N5Exception the exception
*/
void writeChunk(
String datasetPath,
DatasetAttributes datasetAttributes,
DataBlock chunk) throws N5Exception;
/**
* Write multiple chunks represented by {@link DataBlock}s, useful for aggregation.
*
* @param datasetPath dataset path
* @param datasetAttributes the dataset attributes
* @param chunks the chunks
* @param the data block data type
* @throws N5Exception the exception
*/
default void writeChunks(
final String datasetPath,
final DatasetAttributes datasetAttributes,
final DataBlock... chunks) throws N5Exception {
// default method is naive
DatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes);
for (DataBlock block : chunks) {
writeChunk(datasetPath, convertedAttributes, block);
}
}
/**
* Writes a block stored as a {@link DataBlock}.
*
* A block is the highest (coarsest) level of the dataset's {@link NestedGrid},
* that is, a shard if the dataset is sharded, or a chunk otherwise.
*
* This method's behavior is identical to {@link #writeChunk} for un-sharded datasets.
*
* @param pathName dataset path
* @param datasetAttributes the dataset attributes
* @param dataBlock the data block
* @param the data block data type
* @throws N5Exception the exception
*
* @see DatasetAttributes#getNestedBlockGrid()
*/
void writeBlock(
String pathName,
DatasetAttributes datasetAttributes,
DataBlock dataBlock) throws N5Exception;
@FunctionalInterface
interface DataBlockSupplier {
/**
*
* @param gridPos
* @param existingDataBlock
* existing data to be merged into the new data block (may be {@code null})
*
* @return data block at the given gridPos
*/
DataBlock get(long[] gridPos, DataBlock existingDataBlock);
}
/**
* @param datasetPath the dataset path
* @param datasetAttributes the dataset attributes
* @param min min pixel coordinate of region to write
* @param size size in pixels of region to write
* @param chunkSupplier is asked to create chunks within the given region
* @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything.
* @throws N5Exception the exception
*/
void writeRegion(
String datasetPath,
DatasetAttributes datasetAttributes,
long[] min,
long[] size,
DataBlockSupplier chunkSupplier,
boolean writeFully) throws N5Exception;
/**
* @param datasetPath the dataset path
* @param datasetAttributes the dataset attributes
* @param min min pixel coordinate of region to write
* @param size size in pixels of region to write
* @param chunkSupplier is asked to create chunks within the given region
* @param writeFully if false, merge existing data in shards/chunks that overlap the region boundary. if true, override everything.
* @param exec used to parallelize over blocks (chunks and shards)
* @throws N5Exception the exception
*/
void writeRegion(
String datasetPath,
DatasetAttributes datasetAttributes,
long[] min,
long[] size,
DataBlockSupplier chunkSupplier,
boolean writeFully,
ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException;
/**
* Deletes the block at {@code gridPosition}.
*
* A block is the highest (coarsest) level of the dataset's {@link NestedGrid},
* that is, a shard if the dataset is sharded, or a chunk otherwise.
* Note that {@code gridPosition} is in units of blocks at the highest
* (coarsest) level, too.
*
* This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets.
*
* @param datasetPath dataset path
* @param gridPosition position of block to be deleted
* @throws N5Exception if the block exists but could not be deleted
*
* @return {@code true} if the block at {@code gridPosition} existed and was deleted.
*/
default boolean deleteBlock(
final String datasetPath,
final long... gridPosition) throws N5Exception {
final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath);
return deleteBlock(datasetPath, datasetAttributes, gridPosition);
}
/**
* Deletes the block at {@code gridPosition}.
*
* A block is the highest (coarsest) level of the dataset's {@link NestedGrid},
* that is, a shard if the dataset is sharded, or a chunk otherwise.
* Note that {@code gridPosition} is in units of blocks at the highest
* (coarsest) level, too.
*
* This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets.
*
* @param datasetPath the dataset path
* @param datasetAttributes the dataset attributes
* @param gridPosition position of block to be deleted
* @throws N5Exception if the block exists but could not be deleted
*
* @return {@code true} if the block at {@code gridPosition} existed and was deleted.
*/
boolean deleteBlock(
String datasetPath,
DatasetAttributes datasetAttributes,
long... gridPosition) throws N5Exception;
/**
* Deletes the chunk at {@code gridPosition}.
*
* Note that {@code gridPosition} is in units of chunks.
*
* @param datasetPath dataset path
* @param gridPosition position of chunk to be deleted
* @throws N5Exception if the chunk exists but could not be deleted
*
* @return {@code true} if the chunk at {@code gridPosition} existed and was deleted.
*/
default boolean deleteChunk(
final String datasetPath,
final long... gridPosition) throws N5Exception {
final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath);
return deleteChunk(datasetPath, datasetAttributes, gridPosition);
}
/**
* Deletes the chunk at {@code gridPosition}.
*
* Note that {@code gridPosition} is in units of chunks.
*
* @param datasetPath the dataset path
* @param datasetAttributes the dataset attributes
* @param gridPosition position of chunk to be deleted
* @throws N5Exception if the chunk exists but could not be deleted
*
* @return {@code true} if the chunk at {@code gridPosition} existed and was deleted.
*/
boolean deleteChunk(
String datasetPath,
DatasetAttributes datasetAttributes,
long... gridPosition) throws N5Exception;
/**
* Deletes the chunks at the given {@code gridPositions}.
*
* Note that {@code gridPositions} are in units of chunks.
*
* @param datasetPath dataset path
* @param gridPositions a list of grid positions
* @return {@code true} if any of the specified chunks existed and was deleted
* @throws N5Exception if any of the chunks did exist but could not be deleted
*/
default boolean deleteChunks(
final String datasetPath,
final DatasetAttributes datasetAttributes,
final List gridPositions) throws N5Exception {
boolean deleted = false;
for (long[] pos : gridPositions) {
deleted |= deleteChunk(datasetPath, datasetAttributes, pos);
}
return deleted;
}
/**
* Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset.
* The offset is given in {@link DataBlock} grid coordinates.
*
* @param object the object to serialize
* @param datasetPath the dataset path
* @param datasetAttributes the dataset attributes
* @param gridPosition the grid position
* @throws N5Exception the exception
*/
default void writeSerializedBlock(
final Serializable object,
final String datasetPath,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
try (ObjectOutputStream out = new ObjectOutputStream(byteOutputStream)) {
out.writeObject(object);
} catch (final IOException | UncheckedIOException e) {
throw new N5Exception.N5IOException(e);
}
final byte[] bytes = byteOutputStream.toByteArray();
final DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes);
writeChunk(datasetPath, datasetAttributes, dataBlock);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java
================================================
package org.janelia.saalfeldlab.n5;
import com.google.gson.JsonArray;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import org.janelia.saalfeldlab.n5.serialization.N5Annotations;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
import org.scijava.annotations.Index;
import org.scijava.annotations.IndexItem;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map.Entry;
/**
* T adapter, auto-discovers annotated T implementations in the classpath.
*
* @author Caleb Hulbert
*
* @param
* the class this adapter (de)serializes
*/
public class NameConfigAdapter implements JsonDeserializer, JsonSerializer {
private static HashMap, NameConfigAdapter> adapters = new HashMap<>();
private static void registerAdapter(Class cls) {
adapters.put(cls, new NameConfigAdapter<>(cls));
update(adapters.get(cls));
}
private final HashMap> constructors = new HashMap<>();
private final HashMap> parameters = new HashMap<>();
private final HashMap> parameterNames = new HashMap<>();
private static ArrayList getDeclaredFields(Class clazz) {
final ArrayList fields = new ArrayList<>();
fields.addAll(Arrays.asList(clazz.getDeclaredFields()));
for (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass())
fields.addAll(Arrays.asList(clazz.getDeclaredFields()));
return fields;
}
@SuppressWarnings("unchecked")
public static synchronized void update(final NameConfigAdapter adapter) {
final String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value();
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
final Index annotationIndex = Index.load(NameConfig.Name.class, classLoader);
for (final IndexItem item : annotationIndex) {
Class clazz;
try {
clazz = (Class)Class.forName(item.className());
final NameConfig.Serialize serialize = clazz.getAnnotation(NameConfig.Serialize.class);
if (serialize != null && !serialize.value())
continue;
final String name = clazz.getAnnotation(NameConfig.Name.class).value();
final String type = prefix + "." + name;
final Constructor constructor = clazz.getDeclaredConstructor();
final HashMap parameters = new HashMap<>();
final HashMap parameterNames = new HashMap<>();
final ArrayList fields = getDeclaredFields(clazz);
for (final Field field : fields) {
final NameConfig.Parameter parameter = field.getAnnotation(NameConfig.Parameter.class);
if (parameter != null) {
final String parameterName;
if (parameter.value().equals(""))
parameterName = field.getName();
else
parameterName = parameter.value();
parameterNames.put(field.getName(), parameterName);
parameters.put(field.getName(), field);
}
}
adapter.constructors.put(type, constructor);
adapter.parameters.put(type, parameters);
adapter.parameterNames.put(type, parameterNames);
} catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException
| UnsatisfiedLinkError e) {
System.err.println("T '" + item.className() + "' could not be registered");
e.printStackTrace(System.err);
}
}
}
private final Class type;
public NameConfigAdapter(Class cls) {
this.type = cls;
}
@Override
public JsonElement serialize(
final T object,
final Type typeOfSrc,
final JsonSerializationContext context) {
final Class clazz = (Class)object.getClass();
final String name = clazz.getAnnotation(NameConfig.Name.class).value();
final String prefix = type.getAnnotation(NameConfig.Prefix.class).value();
final String type = prefix + "." + name;
final JsonObject json = new JsonObject();
json.addProperty("name", name);
final JsonObject configuration = new JsonObject();
final HashMap parameterTypes = parameters.get(type);
final HashMap parameterNameMap = parameterNames.get(type);
try {
for (final Entry parameterType : parameterTypes.entrySet()) {
final String fieldName = parameterType.getKey();
final Field field = clazz.getDeclaredField(fieldName);
final boolean isAccessible = field.isAccessible();
field.setAccessible(true);
final Object value = field.get(object);
field.setAccessible(isAccessible);
final JsonElement serialized = context.serialize(value);
if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) {
final JsonArray reversedArray = reverseJsonArray(serialized.getAsJsonArray());
configuration.add(parameterNameMap.get(fieldName), reversedArray);
} else
configuration.add(parameterNameMap.get(fieldName), serialized);
}
if (!configuration.isEmpty())
json.add("configuration", configuration);
} catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {
new RuntimeException("Could not serialize " + clazz.getName(), e).printStackTrace(System.err);
return null;
}
return json;
}
@Override
public T deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {
final String prefix = type.getAnnotation(NameConfig.Prefix.class).value();
final JsonObject objectJson = json.getAsJsonObject();
final String name = objectJson.getAsJsonPrimitive("name").getAsString();
if (name == null) {
return null;
}
final String type = prefix + "." + name;
final JsonObject configuration = objectJson.getAsJsonObject("configuration");
/* It's ok to be null if all parameters are optional.
* Otherwise, return*/
if (configuration == null) {
for (final Field field : parameters.get(type).values()) {
if (!field.getAnnotation(NameConfig.Parameter.class).optional())
return null;
}
}
final Constructor constructor = constructors.get(type);
constructor.setAccessible(true);
final T object;
try {
object = constructor.newInstance();
final HashMap parameterTypes = parameters.get(type);
final HashMap parameterNameMap = parameterNames.get(type);
for (final Entry parameterType : parameterTypes.entrySet()) {
final String fieldName = parameterType.getKey();
final String paramName = parameterNameMap.get(fieldName);
final JsonElement paramJson = configuration == null ? null : configuration.get(paramName);
final Field field = parameterType.getValue();
if (paramJson != null) {
final Object parameter;
if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) {
final JsonArray reversedArray = reverseJsonArray(paramJson);
parameter = context.deserialize(reversedArray, field.getType());
} else
parameter = context.deserialize(paramJson, field.getType());
ReflectionUtils.setFieldValue(object, fieldName, parameter);
} else if (!field.getAnnotation(NameConfig.Parameter.class).optional()) {
/* if param is null, and not optional, return null */
return null;
}
}
} catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException
| SecurityException | NoSuchFieldException e) {
e.printStackTrace(System.err);
return null;
}
return object;
}
private static JsonArray reverseJsonArray(JsonElement paramJson) {
final JsonArray reversedJson = new JsonArray(paramJson.getAsJsonArray().size());
for (int i = paramJson.getAsJsonArray().size() - 1; i >= 0; i--) {
reversedJson.add(paramJson.getAsJsonArray().get(i));
}
return reversedJson;
}
public static NameConfigAdapter getJsonAdapter(Class cls) {
if (adapters.get(cls) == null)
registerAdapter(cls);
return (NameConfigAdapter) adapters.get(cls);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java
================================================
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@CompressionType("raw")
@NameConfig.Name("raw")
public class RawCompression implements Compression, DeterministicSizeDataCodec {
private static final long serialVersionUID = 7526445806847086477L;
@Override
public boolean equals(final Object other) {
return other != null && other.getClass() == RawCompression.class;
}
@Override
public ReadData encode(final ReadData readData) {
return readData;
}
@Override
public ReadData decode(final ReadData readData) {
return readData;
}
@Override
public long encodedSize(final long size) {
return size;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java
================================================
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
class ReflectionUtils {
static void setFieldValue(
final Object object,
final String fieldName,
final T value) throws NoSuchFieldException, IllegalAccessException {
Field modifiersField;
boolean isModifiersAccessible;
try {
modifiersField = Field.class.getDeclaredField("modifiers");
isModifiersAccessible = modifiersField.isAccessible();
modifiersField.setAccessible(true);
} catch (final NoSuchFieldException e) {
// Java 11+ does not allow to access modifiers
modifiersField = null;
isModifiersAccessible = false;
}
final Field field = object.getClass().getDeclaredField(fieldName);
final boolean isFieldAccessible = field.isAccessible();
field.setAccessible(true);
if (modifiersField != null) {
final int modifiers = field.getModifiers();
modifiersField.setInt(field, modifiers & ~Modifier.FINAL);
field.set(object, value);
modifiersField.setInt(field, modifiers);
} else {
field.set(object, value);
}
field.setAccessible(isFieldAccessible);
if (modifiersField != null) {
modifiersField.setAccessible(isModifiersAccessible);
}
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
public class ShortArrayDataBlock extends AbstractDataBlock {
public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final short[] data) {
super(size, gridPosition, data, a -> a.length);
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
================================================
package org.janelia.saalfeldlab.n5;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
public class StringDataBlock extends AbstractDataBlock {
protected static final Charset ENCODING = StandardCharsets.UTF_8;
protected static final String NULLCHAR = "\0";
protected byte[] serializedData = null;
protected String[] actualData = null;
public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) {
super(size, gridPosition, new String[0], a -> a.length);
actualData = data;
}
public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) {
super(size, gridPosition, new String[0], a -> a.length);
serializedData = data;
}
public void readData(final ByteBuffer buffer) {
if (buffer.hasArray()) {
if (buffer.array() != serializedData)
buffer.get(serializedData);
actualData = deserialize(buffer.array());
} else
actualData = ENCODING.decode(buffer).toString().split(NULLCHAR);
}
protected byte[] serialize(String[] strings) {
final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR;
return flattenedArray.getBytes(ENCODING);
}
protected String[] deserialize(byte[] rawBytes) {
final String rawChars = new String(rawBytes, ENCODING);
return rawChars.split(NULLCHAR);
}
@Override
public int getNumElements() {
if (serializedData == null)
serializedData = serialize(actualData);
return serializedData.length;
}
@Override
public String[] getData() {
if (actualData == null)
actualData = deserialize(serializedData);
return actualData;
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java
================================================
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@CompressionType("xz")
@NameConfig.Name("xz")
public class XzCompression implements Compression {
private static final long serialVersionUID = -7272153943564743774L;
@CompressionParameter
@NameConfig.Parameter
private final int preset;
public XzCompression(final int preset) {
this.preset = preset;
}
public XzCompression() {
this(6);
}
@Override
public boolean equals(final Object other) {
if (other == null || other.getClass() != XzCompression.class)
return false;
else
return preset == ((XzCompression)other).preset;
}
@Override
public ReadData decode(final ReadData readData) throws N5IOException {
try {
return ReadData.from(new XZCompressorInputStream(readData.inputStream()));
} catch (IOException e) {
throw new N5IOException(e);
}
}
@Override
public ReadData encode(final ReadData readData) {
return readData.encode(out -> new XZCompressorOutputStream(out, preset));
}
}
================================================
FILE: src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java
================================================
package org.janelia.saalfeldlab.n5.cache;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import org.janelia.saalfeldlab.n5.N5Exception;
import com.google.gson.JsonElement;
/*
* A cache containing JSON attributes and children for groups and
* datasets stored in N5 containers. Used by {@link CachedGsonKeyValueN5Reader}
* and {@link CachedGsonKeyValueN5Writer}.
*
*/
public class N5JsonCache {
public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo();
public static final EmptyJson emptyJson = new EmptyJson();
protected final N5JsonCacheableContainer container;
/**
* Data object for caching meta data. Elements that are null are not yet
* cached.
*/
protected static class N5CacheInfo {
protected final HashMap attributesCache = new HashMap<>();
protected LinkedHashSet children = null;
protected boolean isDataset = false;
protected boolean isGroup = false;
public JsonElement getCache(final String normalCacheKey) {
// synchronize the method instead?
synchronized (attributesCache) {
return attributesCache.get(normalCacheKey);
}
}
public boolean containsKey(final String normalCacheKey) {
// synchronize the method instead?
synchronized (attributesCache) {
return attributesCache.containsKey(normalCacheKey);
}
}
public boolean isDataset() {
return isDataset;
}
public boolean isGroup() {
return isGroup;
}
}
@SuppressWarnings("deprecation")
protected static class EmptyJson extends JsonElement {
@Override
public JsonElement deepCopy() {
throw new N5Exception("Do not copy EmptyJson, you naughty person");
}
}
private final HashMap