Extend node variants, rework provider structure

core/src/main/java/io/xpipe/core/data/node/ImmutableValueNode.java

@@ -1,18 +1,10 @@
 package io.xpipe.core.data.node;
 
-public class ImmutableValueNode extends ValueNode {
+public abstract class ImmutableValueNode extends ValueNode {
-
-    private final byte[] data;
-    private final boolean textual;
-
-    ImmutableValueNode(byte[] data, boolean textual) {
-        this.data = data;
-        this.textual = textual;
-    }
 
     @Override
     public String toString(int indent) {
-        return (textual ? "\"" : "") + new String(data) + (textual ? "\"" : "") + " (I)";
+        return (isTextual() ? "\"" : "") + asString() + (isTextual() ? "\"" : "") + " (I)";
     }
 
     @Override
@@ -25,16 +17,6 @@ public class ImmutableValueNode extends ValueNode {
         return this;
     }
 
-    @Override
-    public ValueNode mutableCopy() {
-        return ValueNode.mutable(data, textual);
-    }
-
-    @Override
-    public boolean isTextual() {
-        return textual;
-    }
-
     @Override
     public DataStructureNode setRaw(byte[] data) {
         throw new UnsupportedOperationException("Value node is immutable");
@@ -49,8 +31,4 @@ public class ImmutableValueNode extends ValueNode {
     public DataStructureNode set(Object newValue, boolean textual) {
         throw new UnsupportedOperationException("Value node is immutable");
     }
-
-    public byte[] getRawData() {
-        return data;
-    }
 }

core/src/main/java/io/xpipe/core/data/node/LinkedTupleNode.java

@@ -0,0 +1,189 @@
+package io.xpipe.core.data.node;
+
+import io.xpipe.core.data.type.DataType;
+import io.xpipe.core.data.type.TupleType;
+
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+public class LinkedTupleNode extends TupleNode {
+
+    private final List<TupleNode> tupleNodes;
+    private List<KeyValue> joined;
+
+    public LinkedTupleNode(List<TupleNode> tupleNodes) {
+        this.tupleNodes = new ArrayList<>(tupleNodes);
+    }
+
+    @Override
+    public String keyNameAt(int index) {
+        int list = getTupleNodeForIndex(index);
+        return tupleNodes.get(list).keyNameAt(getLocalIndex(list, index));
+    }
+
+    @Override
+    public List<KeyValue> getKeyValuePairs() {
+        // Lazy initialize joined list
+        if (joined == null) {
+            this.joined = new ArrayList<>();
+            for (var n : tupleNodes) {
+                joined.addAll(n.getKeyValuePairs());
+            }
+            this.joined = Collections.unmodifiableList(joined);
+        }
+
+        return joined;
+    }
+
+    @Override
+    public List<String> getKeyNames() {
+        return getKeyValuePairs().stream().map(KeyValue::key).toList();
+    }
+
+    @Override
+    public List<DataStructureNode> getNodes() {
+        return getKeyValuePairs().stream().map(KeyValue::value).toList();
+    }
+
+    @Override
+    protected String getName() {
+        return "linked tuple node";
+    }
+
+    @Override
+    public boolean isMutable() {
+        return tupleNodes.stream().allMatch(DataStructureNode::isMutable);
+    }
+
+    @Override
+    public DataStructureNode clear() {
+        tupleNodes.forEach(DataStructureNode::clear);
+        return this;
+    }
+
+    @Override
+    public DataStructureNode set(int index, DataStructureNode node) {
+        return super.set(index, node);
+    }
+
+    @Override
+    public DataStructureNode put(String keyName, DataStructureNode node) {
+        return super.put(keyName, node);
+    }
+
+    @Override
+    public DataStructureNode remove(int index) {
+        return super.remove(index);
+    }
+
+    @Override
+    public DataStructureNode remove(String keyName) {
+        return super.remove(keyName);
+    }
+
+    @Override
+    public DataType determineDataType() {
+        return TupleType.of(getKeyNames(), getNodes().stream().map(DataStructureNode::determineDataType).toList());
+    }
+
+    @Override
+    public DataStructureNode at(int index) {
+        int list = getTupleNodeForIndex(index);
+        return tupleNodes.get(list).at(getLocalIndex(list, index));
+    }
+
+    @Override
+    public DataStructureNode forKey(String name) {
+        for (var ar : tupleNodes) {
+            var r = ar.forKeyIfPresent(name);
+            if (r.isPresent()) {
+                return r.get();
+            }
+        }
+        throw new IllegalArgumentException("Invalid key " + name);
+    }
+
+    @Override
+    public Optional<DataStructureNode> forKeyIfPresent(String name) {
+        for (var ar : tupleNodes) {
+            var r = ar.forKeyIfPresent(name);
+            if (r.isPresent()) {
+                return r;
+            }
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public Stream<DataStructureNode> stream() {
+        return getNodes().stream();
+    }
+
+    @Override
+    public void forEach(Consumer<? super DataStructureNode> action) {
+        for (var ar : tupleNodes) {
+            ar.forEach(action);
+        }
+    }
+
+    @Override
+    public Spliterator<DataStructureNode> spliterator() {
+        return stream().spliterator();
+    }
+
+    @Override
+    public Iterator<DataStructureNode> iterator() {
+        return stream().iterator();
+    }
+
+    @Override
+    public String toString() {
+        return "LinkedTupleNode(" + size() + ")";
+    }
+
+
+    @Override
+    public int size() {
+        return tupleNodes.stream().mapToInt(TupleNode::size).sum();
+    }
+
+    private int getLocalIndex(int listIndex, int absIndex) {
+        int current = 0;
+        for (int i = 0; i < listIndex; i++) {
+            current += tupleNodes.get(i).size();
+        }
+        return absIndex - current;
+    }
+
+    private int getTupleNodeForIndex(int index) {
+        int current = 0;
+        for (var a : tupleNodes) {
+            if (index < current + a.size()) {
+                return tupleNodes.indexOf(a);
+            } else {
+                current += a.size();
+            }
+        }
+        throw new IllegalArgumentException();
+    }
+
+    @Override
+    protected String getIdentifier() {
+        return "linked tuple node";
+    }
+
+    @Override
+    public TupleNode mutableCopy() {
+        if (isMutable()) {
+            return this;
+        }
+
+        return new LinkedTupleNode(tupleNodes.stream().map(n -> n.isMutable() ? n : n.mutableCopy()).toList());
+    }
+
+    @Override
+    public TupleNode immutableView() {
+        return this;
+    }
+}

core/src/main/java/io/xpipe/core/data/node/MutableValueNode.java

@@ -31,7 +31,7 @@ public class MutableValueNode extends ValueNode {
 
     @Override
     public ValueNode immutableView() {
-        return new ImmutableValueNode(data, textual);
+        return new SimpleImmutableValueNode(data, textual);
     }
 
     @Override

core/src/main/java/io/xpipe/core/data/node/SimpleImmutableValueNode.java

@@ -0,0 +1,31 @@
+package io.xpipe.core.data.node;
+
+public class SimpleImmutableValueNode extends ImmutableValueNode {
+
+    private final byte[] data;
+    private final boolean textual;
+
+    SimpleImmutableValueNode(byte[] data, boolean textual) {
+        this.data = data;
+        this.textual = textual;
+    }
+
+    @Override
+    public ValueNode mutableCopy() {
+        return ValueNode.mutable(data, textual);
+    }
+
+    @Override
+    public boolean isTextual() {
+        return textual;
+    }
+
+    public byte[] getRawData() {
+        return data;
+    }
+
+    @Override
+    public final String asString() {
+        return new String(getRawData());
+    }
+}

core/src/main/java/io/xpipe/core/data/node/ValueNode.java

@@ -30,7 +30,7 @@ public abstract class ValueNode extends DataStructureNode {
     public abstract ValueNode mutableCopy();
 
     public static ValueNode immutable(byte[] data, boolean textual) {
-        return new ImmutableValueNode(data, textual);
+        return new SimpleImmutableValueNode(data, textual);
     }
 
     public static ValueNode immutable(Object o, boolean textual) {
@@ -86,11 +86,6 @@ public abstract class ValueNode extends DataStructureNode {
         return Integer.parseInt(asString());
     }
 
-    @Override
-    public final String asString() {
-        return new String(getRawData());
-    }
-
     @Override
     public final boolean isValue() {
         return true;

core/src/main/java/io/xpipe/core/source/CollectionDataSourceDescriptor.java

@@ -2,29 +2,49 @@ package io.xpipe.core.source;
 
 import io.xpipe.core.store.DataStore;
 
-public interface CollectionDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS> {
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class CollectionDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS> {
+
+    private final Map<String, String> preferredProviders;
+
+    public CollectionDataSourceDescriptor(DS store) {
+        super(store);
+        this.preferredProviders = new HashMap<>();
+    }
+
+    public CollectionDataSourceDescriptor<DS> annotate(String file, String provider) {
+        preferredProviders.put(file, provider);
+        return this;
+    }
+
+    public CollectionDataSourceDescriptor<DS> annotate(Map<String, String> preferredProviders) {
+        this.preferredProviders.putAll(preferredProviders);
+        return this;
+    }
 
     @Override
-    default DataSourceInfo determineInfo(DS store) throws Exception {
+    public final DataSourceInfo determineInfo() throws Exception {
-        try (var con = openReadConnection(store)) {
+        try (var con = openReadConnection()) {
             var c = (int) con.listEntries().count();
-            return new DataSourceInfo.Structure(c);
+            return new DataSourceInfo.Collection(c);
         }
     }
 
-    default CollectionReadConnection openReadConnection(DS store) throws Exception {
+    public final CollectionReadConnection openReadConnection() throws Exception {
-        var con = newReadConnection(store);
+        var con = newReadConnection();
         con.init();
         return con;
     }
 
-    default CollectionWriteConnection openWriteConnection(DS store) throws Exception {
+    public final CollectionWriteConnection openWriteConnection() throws Exception {
-        var con = newWriteConnection(store);
+        var con = newWriteConnection();
         con.init();
         return con;
     }
 
-    CollectionWriteConnection newWriteConnection(DS store);
+    protected abstract CollectionWriteConnection newWriteConnection();
 
-    CollectionReadConnection newReadConnection(DS store);
+    protected abstract CollectionReadConnection newReadConnection();
 }

core/src/main/java/io/xpipe/core/source/CollectionReadConnection.java

@@ -1,23 +1,22 @@
 package io.xpipe.core.source;
 
+import io.xpipe.core.store.CollectionEntryDataStore;
 import lombok.SneakyThrows;
 
 import java.util.stream.Stream;
 
 public interface CollectionReadConnection extends DataSourceReadConnection {
 
-    <T extends DataSourceReadConnection> T open(String entry) throws Exception;
+    Stream<CollectionEntryDataStore> listEntries() throws Exception;
-
-    Stream<String> listEntries() throws Exception;
 
     @SneakyThrows
     default void forward(DataSourceConnection con) throws Exception {
         try (var tCon = (CollectionWriteConnection) con) {
             tCon.init();
             listEntries().forEach(s -> {
-                try (var subCon = open(s)) {
+//                try (var subCon = open(s)) {
-                    ((CollectionWriteConnection) con).write(s, subCon);
+//                    ((CollectionWriteConnection) con).write(s, subCon);
-                }
+//                }
             });
         }
     }

core/src/main/java/io/xpipe/core/source/CollectionWriteConnection.java

@@ -1,6 +1,8 @@
 package io.xpipe.core.source;
 
+import java.io.InputStream;
+
 public interface CollectionWriteConnection extends DataSourceConnection {
 
-    void write(String entry, DataSourceReadConnection con) throws Exception;
+    void write(String entry, InputStream content) throws Exception;
 }

core/src/main/java/io/xpipe/core/source/DataSourceConnection.java

@@ -9,5 +9,7 @@ public interface DataSourceConnection extends AutoCloseable {
      * Initializes this connection. Required to be called
      * exactly once prior to attempting to use this connection.
      */
-    void init() throws Exception;
+    default void init() throws Exception {}
+
+    default void close() throws Exception {}
 }

core/src/main/java/io/xpipe/core/source/DataSourceDescriptor.java

@@ -10,13 +10,31 @@ import java.util.Optional;
  *
  * This instance is only valid in combination with its associated data store instance.
  */
-public interface DataSourceDescriptor<DS extends DataStore> {
+public abstract class DataSourceDescriptor<DS extends DataStore> {
+
+    protected DS store;
+
+    public DataSourceDescriptor(DS store) {
+        this.store = store;
+    }
+
+    public DataSourceDescriptor<DS> withStore(DS newStore) {
+        return null;
+    }
+
+    /**
+     * Casts this instance to the required type without checking whether a cast is possible.
+     */
+    @SuppressWarnings("unchecked")
+    public final <DSD extends DataSourceDescriptor<?>> DSD asNeeded() {
+        return (DSD) this;
+    }
 
     /**
      * Determines on optional default name for this data store that is
      * used when determining a suitable default name for a data source.
      */
-    default Optional<String> determineDefaultName(DS store) {
+    public Optional<String> determineDefaultName() {
         return Optional.empty();
     }
 
@@ -25,9 +43,13 @@ public interface DataSourceDescriptor<DS extends DataStore> {
      * This is usually called only once on data source
      * creation as this process might be expensive.
      */
-    DataSourceInfo determineInfo(DS store) throws Exception;
+    public abstract DataSourceInfo determineInfo() throws Exception;
 
-    DataSourceReadConnection openReadConnection(DS store) throws Exception;
+    public abstract DataSourceReadConnection openReadConnection() throws Exception;
 
-    DataSourceConnection openWriteConnection(DS store) throws Exception;
+    public abstract DataSourceConnection openWriteConnection() throws Exception;
+
+    public DS getStore() {
+        return store;
+    }
 }

core/src/main/java/io/xpipe/core/source/DataSourceInfo.java

@@ -117,24 +117,6 @@ public abstract class DataSourceInfo {
         }
     }
 
-
-    @EqualsAndHashCode(callSuper = false)
-    @Value
-    @JsonTypeName("archive")
-    public static class Archive extends DataSourceInfo {
-        int contentCount;
-
-        @JsonCreator
-        public Archive(int contentCount) {
-            this.contentCount = contentCount;
-        }
-
-        @Override
-        public DataSourceType getType() {
-            return null;
-        }
-    }
-
     /**
      * Casts this instance to a table info.
      */
@@ -178,4 +160,15 @@ public abstract class DataSourceInfo {
 
         return (Raw) this;
     }
+
+    /**
+     * Casts this instance to a collection info.
+     */
+    public Collection asCollection() {
+        if (!getType().equals(DataSourceType.COLLECTION)) {
+            throw new IllegalStateException("Not a collection");
+        }
+
+        return (Collection) this;
+    }
 }

core/src/main/java/io/xpipe/core/source/RawDataSourceDescriptor.java

@@ -2,13 +2,17 @@ package io.xpipe.core.source;
 
 import io.xpipe.core.store.DataStore;
 
-public abstract class RawDataSourceDescriptor <DS extends DataStore> implements DataSourceDescriptor<DS>  {
+public abstract class RawDataSourceDescriptor <DS extends DataStore> extends DataSourceDescriptor<DS>  {
 
     private static final int MAX_BYTES_READ = 100000;
 
+    public RawDataSourceDescriptor(DS store) {
+        super(store);
+    }
+
     @Override
-    public DataSourceInfo determineInfo(DS store) throws Exception {
+    public final DataSourceInfo determineInfo() throws Exception {
-        try (var con = openReadConnection(store)) {
+        try (var con = openReadConnection()) {
             var b = con.readBytes(MAX_BYTES_READ);
             int usedCount = b.length == MAX_BYTES_READ ? -1 : b.length;
             return new DataSourceInfo.Raw(usedCount);
@@ -16,20 +20,20 @@ public abstract class RawDataSourceDescriptor <DS extends DataStore> implements
     }
 
     @Override
-    public RawReadConnection openReadConnection(DS store) throws Exception {
+    public final RawReadConnection openReadConnection() throws Exception {
-        var con = newReadConnection(store);
+        var con = newReadConnection();
         con.init();
         return con;
     }
 
     @Override
-    public RawWriteConnection openWriteConnection(DS store) throws Exception {
+    public final RawWriteConnection openWriteConnection() throws Exception {
-        var con = newWriteConnection(store);
+        var con = newWriteConnection();
         con.init();
         return con;
     }
 
-    protected abstract RawWriteConnection newWriteConnection(DS store);
+    protected abstract RawWriteConnection newWriteConnection();
 
-    protected abstract RawReadConnection newReadConnection(DS store);
+    protected abstract RawReadConnection newReadConnection();
 }

core/src/main/java/io/xpipe/core/source/StructureDataSourceDescriptor.java

@@ -3,7 +3,11 @@ package io.xpipe.core.source;
 import io.xpipe.core.data.node.DataStructureNode;
 import io.xpipe.core.store.DataStore;
 
-public interface StructureDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS> {
+public abstract class StructureDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS> {
+
+    public StructureDataSourceDescriptor(DS store) {
+        super(store);
+    }
 
     private int countEntries(DataStructureNode n) {
         if (n.isValue()) {
@@ -18,26 +22,27 @@ public interface StructureDataSourceDescriptor<DS extends DataStore> extends Dat
     }
 
     @Override
-    default DataSourceInfo determineInfo(DS store) throws Exception {
+    public final DataSourceInfo determineInfo() throws Exception {
-        try (var con = openReadConnection(store)) {
+        try (var con = openReadConnection()) {
             var n = con.read();
             var c = countEntries(n);
             return new DataSourceInfo.Structure(c);
         }
     }
 
-    default StructureReadConnection openReadConnection(DS store) throws Exception {
+    public final StructureReadConnection openReadConnection() throws Exception {
-        var con = newReadConnection(store);
+        var con = newReadConnection();
         con.init();
         return con;
     }
 
-    default StructureWriteConnection openWriteConnection(DS store) throws Exception {
+    public final StructureWriteConnection openWriteConnection() throws Exception {
-        var con = newWriteConnection(store);
+        var con = newWriteConnection();
         con.init();
         return con;
     }
-    StructureWriteConnection newWriteConnection(DS store);
 
-    StructureReadConnection newReadConnection(DS store);
+    protected abstract StructureWriteConnection newWriteConnection();
+
+    protected abstract StructureReadConnection newReadConnection();
 }

core/src/main/java/io/xpipe/core/source/TableDataSourceDescriptor.java

@@ -2,21 +2,34 @@ package io.xpipe.core.source;
 
 import io.xpipe.core.store.DataStore;
 
-public abstract class TableDataSourceDescriptor<DS extends DataStore> implements DataSourceDescriptor<DS> {
+public abstract class TableDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS> {
 
-    public final TableReadConnection openReadConnection(DS store) throws Exception {
+    public TableDataSourceDescriptor(DS store) {
-        var con = newReadConnection(store);
+        super(store);
+    }
+
+    @Override
+    public final DataSourceInfo determineInfo() throws Exception {
+        try (var con = openReadConnection()) {
+            var dataType = con.getDataType();
+            var rowCount = con.getRowCount();
+            return new DataSourceInfo.Table(dataType, rowCount);
+        }
+    }
+
+    public final TableReadConnection openReadConnection() throws Exception {
+        var con = newReadConnection();
         con.init();
         return con;
     }
 
-    public final TableWriteConnection openWriteConnection(DS store) throws Exception {
+    public final TableWriteConnection openWriteConnection() throws Exception {
-        var con = newWriteConnection(store);
+        var con = newWriteConnection();
         con.init();
         return con;
     }
 
-    protected abstract TableWriteConnection newWriteConnection(DS store);
+    protected abstract TableWriteConnection newWriteConnection();
 
-    protected abstract TableReadConnection newReadConnection(DS store);
+    protected abstract TableReadConnection newReadConnection();
 }

core/src/main/java/io/xpipe/core/source/TextDataSourceDescriptor.java

@@ -2,13 +2,17 @@ package io.xpipe.core.source;
 
 import io.xpipe.core.store.DataStore;
 
-public abstract class TextDataSourceDescriptor<DS extends DataStore> implements DataSourceDescriptor<DS>  {
+public abstract class TextDataSourceDescriptor<DS extends DataStore> extends DataSourceDescriptor<DS>  {
 
     private static final int MAX_LINE_READ = 1000;
 
+    public TextDataSourceDescriptor(DS store) {
+        super(store);
+    }
+
     @Override
-    public DataSourceInfo determineInfo(DS store) throws Exception {
+    public final DataSourceInfo determineInfo() throws Exception {
-        try (var con = openReadConnection(store)) {
+        try (var con = openReadConnection()) {
             int count = (int) con.lines().limit(MAX_LINE_READ).count();
             int usedCount = count == MAX_LINE_READ ? -1 : count;
             return new DataSourceInfo.Text(usedCount);
@@ -16,20 +20,20 @@ public abstract class TextDataSourceDescriptor<DS extends DataStore> implements
     }
 
     @Override
-    public TextReadConnection openReadConnection(DS store) throws Exception {
+    public final TextReadConnection openReadConnection() throws Exception {
-        var con = newReadConnection(store);
+        var con = newReadConnection();
         con.init();
         return con;
     }
 
     @Override
-    public TextWriteConnection openWriteConnection(DS store) throws Exception {
+    public final TextWriteConnection openWriteConnection() throws Exception {
-        var con = newWriteConnection(store);
+        var con = newWriteConnection();
         con.init();
         return con;
     }
 
-    protected abstract TextWriteConnection newWriteConnection(DS store);
+    protected abstract TextWriteConnection newWriteConnection();
 
-    protected abstract TextReadConnection newReadConnection(DS store);
+    protected abstract TextReadConnection newReadConnection();
 }

core/src/main/java/io/xpipe/core/store/CollectionEntryDataStore.java

@@ -0,0 +1,36 @@
+package io.xpipe.core.store;
+
+public abstract class CollectionEntryDataStore implements FileDataStore {
+
+    private final boolean directory;
+    private final String name;
+    private final DataStore collectionStore;
+
+
+    public CollectionEntryDataStore(boolean directory, String name, DataStore collectionStore) {
+        this.directory = directory;
+        this.name = name;
+        this.collectionStore = collectionStore;
+    }
+
+    @Override
+    public String getFileName() {
+        return name;
+    }
+
+    public boolean isDirectory() {
+        return directory;
+    }
+
+    public DataStore getCollectionStore() {
+        return collectionStore;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getPreferredProvider() {
+        return null;
+    }
+}

core/src/main/java/io/xpipe/core/store/FileDataStore.java

@@ -1,6 +1,14 @@
 package io.xpipe.core.store;
 
+import java.util.Optional;
+
 public interface FileDataStore extends StreamDataStore {
 
+    @Override
+    default Optional<String> determineDefaultName() {
+        var n = getFileName();
+        var i = n.lastIndexOf('.');
+        return Optional.of(i != -1 ? n.substring(0, i) : n);
+    }
     String getFileName();
 }

core/src/main/java/io/xpipe/core/store/LocalDirectoryDataStore.java

@@ -0,0 +1,43 @@
+package io.xpipe.core.store;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import lombok.EqualsAndHashCode;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Optional;
+
+@JsonTypeName("localDir")
+@EqualsAndHashCode
+public class LocalDirectoryDataStore implements DataStore {
+
+    private final Path file;
+
+    @JsonCreator
+    public LocalDirectoryDataStore(Path file) {
+        this.file = file;
+    }
+
+    @Override
+    public Optional<String> determineDefaultName() {
+        return Optional.of(file.getFileName().toString());
+    }
+
+    @Override
+    public Optional<Instant> determineLastModified() {
+        try {
+            var l = Files.getLastModifiedTime(file);
+            return Optional.of(l.toInstant());
+        } catch (IOException e) {
+            return Optional.empty();
+        }
+    }
+
+    public Path getPath() {
+        return file;
+    }
+
+}

core/src/main/java/io/xpipe/core/store/LocalFileDataStore.java

@@ -24,9 +24,8 @@ public class LocalFileDataStore implements FileDataStore {
         this.file = file;
     }
 
-    @Override
+    public String toString() {
-    public Optional<String> determineDefaultName() {
+        return getFileName();
-        return Optional.of(file.getFileName().toString());
     }
 
     @Override

core/src/main/java/io/xpipe/core/store/StreamDataStore.java

@@ -34,12 +34,16 @@ public interface StreamDataStore extends DataStore {
     /**
      * Opens an input stream. This input stream does not necessarily have to be a new instance.
      */
-    InputStream openInput() throws Exception;
+    default InputStream openInput() throws Exception {
+        throw new UnsupportedOperationException("Can't open store input");
+    }
 
     /**
      * Opens an output stream. This output stream does not necessarily have to be a new instance.
      */
-    OutputStream openOutput() throws Exception;
+    default OutputStream openOutput() throws Exception {
+        throw new UnsupportedOperationException("Can't open store output");
+    }
 
     boolean exists();
 }

core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java

@@ -17,6 +17,8 @@ import io.xpipe.core.data.type.ValueType;
 import io.xpipe.core.data.type.WildcardType;
 import io.xpipe.core.source.DataSourceInfo;
 import io.xpipe.core.source.DataSourceReference;
+import io.xpipe.core.store.CollectionEntryDataStore;
+import io.xpipe.core.store.LocalDirectoryDataStore;
 import io.xpipe.core.store.LocalFileDataStore;
 
 import java.io.IOException;
@@ -29,6 +31,8 @@ public class CoreJacksonModule extends SimpleModule {
     public void setupModule(SetupContext context) {
         context.registerSubtypes(
                 new NamedType(LocalFileDataStore.class),
+                new NamedType(LocalDirectoryDataStore.class),
+                new NamedType(CollectionEntryDataStore.class),
                 new NamedType(ValueType.class),
                 new NamedType(TupleType.class),
                 new NamedType(ArrayType.class),
@@ -36,6 +40,7 @@ public class CoreJacksonModule extends SimpleModule {
                 new NamedType(DataSourceInfo.Table.class),
                 new NamedType(DataSourceInfo.Structure.class),
                 new NamedType(DataSourceInfo.Text.class),
+                new NamedType(DataSourceInfo.Collection.class),
                 new NamedType(DataSourceInfo.Raw.class)
         );
 

extension/src/main/java/io/xpipe/extension/DataSourceProvider.java

@@ -3,20 +3,42 @@ package io.xpipe.extension;
 import io.xpipe.core.config.ConfigOption;
 import io.xpipe.core.config.ConfigOptionSet;
 import io.xpipe.core.source.DataSourceDescriptor;
-import io.xpipe.core.source.DataSourceInfo;
 import io.xpipe.core.source.DataSourceType;
 import io.xpipe.core.store.DataStore;
 import javafx.beans.property.Property;
 import javafx.scene.layout.Region;
 
 import java.nio.charset.Charset;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Function;
-import java.util.function.Supplier;
 
-public interface DataSourceProvider {
+public interface DataSourceProvider<T extends DataSourceDescriptor<?>> {
+
+    static enum GeneralType {
+        FILE,
+        DATABASE;
+    }
+
+    default GeneralType getGeneralType() {
+        if (getFileProvider() != null) {
+            return GeneralType.FILE;
+        }
+
+        throw new ExtensionException("Provider has no general type");
+    }
+
+    default boolean supportsConversion(T in, DataSourceType t) {
+        return false;
+    }
+
+    default DataSourceDescriptor<?> convert(T in, DataSourceType t) throws Exception {
+        throw new ExtensionException();
+    }
+
+    default void init() {
+    }
 
     default String i18n(String key) {
         return I18n.get(getId() + "." + key);
@@ -34,12 +56,16 @@ public interface DataSourceProvider {
         return i18n("displayName");
     }
 
-    default String getDisplayImageFile() {
+    default String getDisplayDescription() {
-        return "logo.png";
+        return i18n("displayDescription");
     }
 
-    default String getDescription(DataSourceDescriptor<?> source) {
+    default String getDisplayIconFileName() {
-        return i18n("description");
+        return getId() + ":icon.png";
+    }
+
+    default String getSourceDescription(T source) {
+        return getDisplayName();
     }
 
     interface FileProvider {
@@ -49,22 +75,22 @@ public interface DataSourceProvider {
         Map<String, List<String>> getFileExtensions();
     }
 
-    interface ConfigProvider {
+    interface ConfigProvider<T extends DataSourceDescriptor<?>> {
 
-        static ConfigProvider empty(List<String> names, Supplier<DataSourceDescriptor<?>> supplier) {
+        static <T extends DataSourceDescriptor<?>> ConfigProvider<T> empty(List<String> names, Function<DataStore, T> func) {
-            return new ConfigProvider() {
+            return new ConfigProvider<>() {
                 @Override
                 public ConfigOptionSet getConfig() {
                     return ConfigOptionSet.empty();
                 }
 
                 @Override
-                public DataSourceDescriptor<?> toDescriptor(Map<String, String> values) {
+                public T toDescriptor(DataStore store, Map<String, String> values) {
-                    return supplier.get();
+                    return func.apply(store);
                 }
 
                 @Override
-                public Map<String, String> toConfigOptions(DataSourceDescriptor<?> desc) {
+                public Map<String, String> toConfigOptions(T source) {
                     return Map.of();
                 }
 
@@ -121,9 +147,9 @@ public interface DataSourceProvider {
 
         ConfigOptionSet getConfig();
 
-        DataSourceDescriptor<?> toDescriptor(Map<String, String> values);
+        T toDescriptor(DataStore store, Map<String, String> values);
 
-        Map<String, String> toConfigOptions(DataSourceDescriptor<?> desc);
+        Map<String, String> toConfigOptions(T desc);
 
         Map<ConfigOption, Function<String, ?>> getConverters();
 
@@ -160,9 +186,15 @@ public interface DataSourceProvider {
         return false;
     }
 
-    FileProvider getFileProvider();
+    default FileProvider getFileProvider() {
+        return null;
+    }
 
-    ConfigProvider getConfigProvider();
+    default boolean hasDirectoryProvider() {
+        return false;
+    }
+
+    ConfigProvider<T> getConfigProvider();
 
     default String getId() {
         var n = getClass().getPackageName();
@@ -170,17 +202,20 @@ public interface DataSourceProvider {
         return i != -1 ? n.substring(i + 1) : n;
     }
 
-    DataSourceDescriptor<?> createDefaultDescriptor();
-
     /**
      * Attempt to create a useful data source descriptor from a data store.
      * The result does not need to be always right, it should only reflect the best effort.
      */
-    DataSourceDescriptor<?> createDefaultDescriptor(DataStore input) throws Exception;
+    T createDefaultDescriptor(DataStore input) throws Exception;
 
-    DataSourceDescriptor<?> createDefaultWriteDescriptor(DataStore input, DataSourceInfo info) throws Exception;
+    default T createDefaultWriteDescriptor(DataStore input) throws Exception {
-
+        return createDefaultDescriptor(input);
-    Class<? extends DataSourceDescriptor<?>> getDescriptorClass();
+    }
-
+
-    Optional<String> determineDefaultName(DataStore store);
+    @SuppressWarnings("unchecked")
+    default Class<T> getDescriptorClass() {
+        return (Class<T>) Arrays.stream(getClass().getDeclaredClasses())
+                .filter(c -> c.getName().endsWith("Descriptor")).findFirst()
+                .orElseThrow(() -> new AssertionError("Descriptor class not found"));
+    }
 }

extension/src/main/java/io/xpipe/extension/DataSourceProviders.java

@@ -1,13 +1,10 @@
 package io.xpipe.extension;
 
-import io.xpipe.core.data.type.TupleType;
 import io.xpipe.core.source.*;
 import io.xpipe.core.store.DataStore;
 import io.xpipe.core.store.LocalFileDataStore;
-import io.xpipe.core.store.StreamDataStore;
 import lombok.SneakyThrows;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
@@ -15,27 +12,25 @@ import java.util.stream.Collectors;
 
 public class DataSourceProviders {
 
-    private static Set<DataSourceProvider> ALL;
+    private static Set<DataSourceProvider<?>> ALL;
 
     public static void init(ModuleLayer layer) {
         if (ALL == null) {
             ALL = ServiceLoader.load(layer, DataSourceProvider.class).stream()
-                    .map(ServiceLoader.Provider::get).collect(Collectors.toSet());
+                    .map(p -> (DataSourceProvider<?>) p.get()).collect(Collectors.toSet());
+            ALL.forEach(DataSourceProvider::init);
         }
     }
 
-    @SuppressWarnings("unchecked")
+    public static DataSourceProvider<?> getNativeDataSourceDescriptorForType(DataSourceType t) {
-    public static DataSourceDescriptor<StreamDataStore> getNativeDataSourceDescriptorForType(DataSourceType t) {
         try {
             return switch (t) {
-                case TABLE -> (DataSourceDescriptor<StreamDataStore>) DataSourceProviders.byId("xpbt").orElseThrow()
+                case TABLE -> DataSourceProviders.byId("xpbt");
-                        .getDescriptorClass().getConstructors()[0].newInstance();
+                case STRUCTURE -> DataSourceProviders.byId("xpbs");
-                case STRUCTURE -> (DataSourceDescriptor<StreamDataStore>) DataSourceProviders.byId("xpbs").orElseThrow()
+                case TEXT -> DataSourceProviders.byId("text");
-                        .getDescriptorClass().getConstructors()[0].newInstance();
+                case RAW -> DataSourceProviders.byId("binary");
-                case TEXT -> (DataSourceDescriptor<StreamDataStore>) DataSourceProviders.byId("text").orElseThrow()
+                //TODO
-                        .getDescriptorClass().getConstructors()[0].newInstance(StandardCharsets.UTF_8);
+                case COLLECTION -> null;
-                case RAW -> (DataSourceDescriptor<StreamDataStore>) DataSourceProviders.byId("xpbr").orElseThrow()
-                        .getDescriptorClass().getConstructors()[0].newInstance();
             };
         } catch (Exception ex) {
             throw new AssertionError(ex);
@@ -44,53 +39,66 @@ public class DataSourceProviders {
 
     @SuppressWarnings("unchecked")
     @SneakyThrows
-    public static StructureDataSourceDescriptor<LocalFileDataStore> createLocalStructureDescriptor() {
+    public static StructureDataSourceDescriptor<LocalFileDataStore> createLocalStructureDescriptor(DataStore store) {
         return (StructureDataSourceDescriptor<LocalFileDataStore>)
-                DataSourceProviders.byId("json").orElseThrow().getDescriptorClass()
+                DataSourceProviders.byId("xpbs").getDescriptorClass()
-                        .getDeclaredConstructors()[0].newInstance();
+                        .getDeclaredConstructors()[0].newInstance(store);
     }
 
     @SuppressWarnings("unchecked")
     @SneakyThrows
-    public static RawDataSourceDescriptor<LocalFileDataStore> createLocalRawDescriptor() {
+    public static RawDataSourceDescriptor<LocalFileDataStore> createLocalRawDescriptor(DataStore store) {
         return (RawDataSourceDescriptor<LocalFileDataStore>)
-                DataSourceProviders.byId("binary").orElseThrow().getDescriptorClass()
+                DataSourceProviders.byId("binary").getDescriptorClass()
-                        .getDeclaredConstructors()[0].newInstance();
+                        .getDeclaredConstructors()[0].newInstance(store);
     }
 
     @SuppressWarnings("unchecked")
     @SneakyThrows
-    public static TextDataSourceDescriptor<LocalFileDataStore> createLocalTextDescriptor() {
+    public static RawDataSourceDescriptor<LocalFileDataStore> createLocalCollectionDescriptor(DataStore store) {
+        return (RawDataSourceDescriptor<LocalFileDataStore>)
+                DataSourceProviders.byId("br").getDescriptorClass()
+                        .getDeclaredConstructors()[0].newInstance(store);
+    }
+
+    @SuppressWarnings("unchecked")
+    @SneakyThrows
+    public static TextDataSourceDescriptor<LocalFileDataStore> createLocalTextDescriptor(DataStore store) {
         return (TextDataSourceDescriptor<LocalFileDataStore>)
-                DataSourceProviders.byId("text").orElseThrow().getDescriptorClass()
+                DataSourceProviders.byId("text").getDescriptorClass()
-                        .getDeclaredConstructors()[0].newInstance();
+                        .getDeclaredConstructors()[0].newInstance(store);
     }
 
     @SuppressWarnings("unchecked")
     @SneakyThrows
-    public static TableDataSourceDescriptor<LocalFileDataStore> createLocalTableDescriptor(TupleType type) {
+    public static TableDataSourceDescriptor<LocalFileDataStore> createLocalTableDescriptor(DataStore store) {
         return (TableDataSourceDescriptor<LocalFileDataStore>)
-                DataSourceProviders.byId("xpbt").orElseThrow().getDescriptorClass()
+                DataSourceProviders.byId("xpbt").getDescriptorClass()
-                        .getDeclaredConstructors()[0].newInstance(type);
+                        .getDeclaredConstructors()[0].newInstance(store);
     }
 
-    public static Optional<DataSourceProvider> byDescriptorClass(Class<?> clazz) {
+    @SuppressWarnings("unchecked")
+    public static <T extends DataSourceProvider<?>> T byId(String name) {
         if (ALL == null) {
             throw new IllegalStateException("Not initialized");
         }
 
-        return ALL.stream().filter(d -> d.getDescriptorClass().equals(clazz)).findAny();
+        return (T) ALL.stream().filter(d -> d.getId().equals(name)).findAny()
+                .orElseThrow(() -> new IllegalArgumentException("Provider " + name + " not found"));
     }
 
-    public static Optional<DataSourceProvider> byId(String name) {
+
+    @SuppressWarnings("unchecked")
+    public static <C extends DataSourceDescriptor<?>, T extends DataSourceProvider<C>> T byDataSourceClass(Class<C> c) {
         if (ALL == null) {
             throw new IllegalStateException("Not initialized");
         }
 
-        return ALL.stream().filter(d -> d.getId().equals(name)).findAny();
+        return (T) ALL.stream().filter(d -> d.getDescriptorClass().equals(c)).findAny()
+                .orElseThrow(() -> new IllegalArgumentException("Provider for " + c.getSimpleName() + " not found"));
     }
 
-    public static Optional<DataSourceProvider> byName(String name) {
+    public static Optional<DataSourceProvider<?>> byName(String name) {
         if (ALL == null) {
             throw new IllegalStateException("Not initialized");
         }
@@ -99,7 +107,7 @@ public class DataSourceProviders {
                 .anyMatch(s -> s.equalsIgnoreCase(name))).findAny();
     }
 
-    public static Optional<DataSourceProvider> byStore(DataStore store) {
+    public static Optional<DataSourceProvider<?>> byStore(DataStore store) {
         if (ALL == null) {
             throw new IllegalStateException("Not initialized");
         }
@@ -108,7 +116,7 @@ public class DataSourceProviders {
                 .filter(d -> d.couldSupportStore(store)).findAny();
     }
 
-    public static Set<DataSourceProvider> getAll() {
+    public static Set<DataSourceProvider<?>> getAll() {
         return ALL;
     }
 }

extension/src/main/java/io/xpipe/extension/ExtensionException.java

@@ -0,0 +1,23 @@
+package io.xpipe.extension;
+
+public class ExtensionException extends RuntimeException {
+
+    public ExtensionException() {
+    }
+
+    public ExtensionException(String message) {
+        super(message);
+    }
+
+    public ExtensionException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ExtensionException(Throwable cause) {
+        super(cause);
+    }
+
+    public ExtensionException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

extension/src/main/java/io/xpipe/extension/SimpleFileDataSourceProvider.java

@@ -1,5 +1,6 @@
 package io.xpipe.extension;
 
+import io.xpipe.core.source.DataSourceDescriptor;
 import io.xpipe.core.store.DataStore;
 import io.xpipe.core.store.FileDataStore;
 import io.xpipe.core.store.StreamDataStore;
@@ -7,20 +8,8 @@ import io.xpipe.core.store.StreamDataStore;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 
-public interface SimpleFileDataSourceProvider extends DataSourceProvider {
+public interface SimpleFileDataSourceProvider<T extends DataSourceDescriptor<?>> extends DataSourceProvider<T> {
-
-    @Override
-    default Optional<String> determineDefaultName(DataStore store) {
-        if (store instanceof FileDataStore l) {
-            var n = l.getFileName();
-            var i = n.lastIndexOf('.');
-            return Optional.of(i != -1 ? n.substring(0, i) : n);
-        }
-
-        return Optional.empty();
-    }
 
     @Override
     default boolean prefersStore(DataStore store) {

extension/src/main/java/io/xpipe/extension/UniformDataSourceProvider.java

@@ -1,35 +1,25 @@
 package io.xpipe.extension;
 
 import io.xpipe.core.source.DataSourceDescriptor;
-import io.xpipe.core.source.DataSourceInfo;
 import io.xpipe.core.store.DataStore;
 
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
-public interface UniformDataSourceProvider extends DataSourceProvider {
+public interface UniformDataSourceProvider<T extends DataSourceDescriptor<?>> extends DataSourceProvider<T> {
 
     @Override
-    default ConfigProvider getConfigProvider() {
+    default ConfigProvider<T> getConfigProvider() {
         return ConfigProvider.empty(List.of(getId()), this::createDefaultDescriptor);
     }
 
     @Override
-    default DataSourceDescriptor<?> createDefaultDescriptor() {
+    @SuppressWarnings("unchecked")
+    default T createDefaultDescriptor(DataStore input) {
         try {
-            return (DataSourceDescriptor<?>) getDescriptorClass().getDeclaredConstructors()[0].newInstance();
+            return (T) getDescriptorClass().getDeclaredConstructors()[0].newInstance(input);
         } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
             throw new AssertionError(e);
         }
     }
-
-    @Override
-    default DataSourceDescriptor<?> createDefaultDescriptor(DataStore input) throws Exception {
-        return createDefaultDescriptor();
-    }
-
-    @Override
-    default DataSourceDescriptor<?> createDefaultWriteDescriptor(DataStore input, DataSourceInfo info) throws Exception {
-        return createDefaultDescriptor();
-    }
 }

extension/src/main/java/io/xpipe/extension/comp/DynamicOptionsBuilder.java

@@ -0,0 +1,33 @@
+package io.xpipe.extension.comp;
+
+import io.xpipe.core.source.DataSourceDescriptor;
+import io.xpipe.fxcomps.Comp;
+import javafx.beans.Observable;
+import javafx.beans.binding.Bindings;
+import javafx.beans.property.Property;
+import javafx.beans.value.ObservableValue;
+import javafx.scene.control.TextField;
+import javafx.scene.layout.Region;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class DynamicOptionsBuilder<T extends DataSourceDescriptor<?>> {
+
+    private final  List<DynamicOptionsComp.Entry> entries = new ArrayList<>();
+    private final List<Property<?>> props = new ArrayList<>();
+
+    public DynamicOptionsBuilder<T> addText(ObservableValue<String> name, Property<String> prop) {
+        var comp = new TextField();
+        comp.textProperty().bindBidirectional(prop);
+        entries.add(new DynamicOptionsComp.Entry(name, Comp.of(() -> comp)));
+        return this;
+    }
+
+    public Region build(Supplier<T> creator, Property<T> toBind) {
+        var bind = Bindings.createObjectBinding(() -> creator.get(), props.toArray(Observable[]::new));
+        toBind.bind(bind);
+        return new DynamicOptionsComp(entries).createRegion();
+    }
+}

extension/src/main/java/io/xpipe/extension/comp/DynamicOptionsComp.java

@@ -4,6 +4,7 @@ import io.xpipe.fxcomps.Comp;
 import io.xpipe.fxcomps.CompStructure;
 import javafx.beans.Observable;
 import javafx.beans.binding.Bindings;
+import javafx.beans.value.ObservableValue;
 import javafx.geometry.Orientation;
 import javafx.geometry.Pos;
 import javafx.scene.control.Label;
@@ -13,7 +14,6 @@ import javafx.scene.layout.Region;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.Supplier;
 
 public class DynamicOptionsComp extends Comp<CompStructure<FlowPane>> {
 
@@ -37,7 +37,8 @@ public class DynamicOptionsComp extends Comp<CompStructure<FlowPane>> {
             var line = new HBox();
             line.setSpacing(5);
 
-            var name = new Label(entry.name().get());
+            var name = new Label();
+            name.textProperty().bind(entry.name());
             name.prefHeightProperty().bind(line.heightProperty());
             name.setMinWidth(Region.USE_PREF_SIZE);
             name.setAlignment(Pos.CENTER_LEFT);
@@ -78,7 +79,7 @@ public class DynamicOptionsComp extends Comp<CompStructure<FlowPane>> {
         return entries;
     }
 
-    public static record Entry(Supplier<String> name, Comp<?> comp) {
+    public static record Entry(ObservableValue<String> name, Comp<?> comp) {
 
     }
 }

samples/sample_extensions/file_data_source_sample/build.gradle

@@ -3,8 +3,6 @@ plugins {
     id "org.moditect.gradleplugin" version "1.0.0-rc3"
 }
 
-apply from: "$rootDir/deps/java.gradle"
-apply from: "$rootDir/deps/javafx.gradle"
 apply from: "$rootDir/deps/lombok.gradle"
 apply from: "$rootDir/deps/extension.gradle"