More beacon additions and small fixes

beacon/src/main/java/io/xpipe/beacon/BeaconClient.java

@@ -168,6 +168,10 @@ public class BeaconClient implements AutoCloseable {
             System.out.println(read.toPrettyString());
         }
 
+        if (read.isMissingNode()) {
+            throw new ConnectorException("Received unexpected EOF");
+        }
+
         var se = parseServerError(read);
         if (se.isPresent()) {
             se.get().throwError();

beacon/src/main/java/io/xpipe/beacon/BeaconServer.java

@@ -75,12 +75,17 @@ public class BeaconServer {
     public static boolean tryStart() throws Exception {
         var custom = BeaconConfig.getCustomExecCommand();
         if (custom != null) {
+            System.out.println("Starting fork: " + custom);
             startFork(custom);
             return true;
         }
 
         var daemonExecutable = getDaemonExecutable();
         if (daemonExecutable.isPresent()) {
+            if (BeaconConfig.debugEnabled()) {
+                System.out.println("Starting daemon executable: " + daemonExecutable.get());
+            }
+
             // Tell daemon that we launched from an external tool
             new ProcessBuilder(daemonExecutable.get().toString(), "--external")
                     .redirectErrorStream(true)

beacon/src/main/java/io/xpipe/beacon/exchange/EditExecuteExchange.java

@@ -3,6 +3,7 @@ package io.xpipe.beacon.exchange;
 import io.xpipe.beacon.message.RequestMessage;
 import io.xpipe.beacon.message.ResponseMessage;
 import io.xpipe.core.source.DataSourceConfigInstance;
+import io.xpipe.core.source.DataSourceId;
 import io.xpipe.core.source.DataSourceReference;
 import lombok.Builder;
 import lombok.NonNull;
@@ -19,21 +20,13 @@ public class EditExecuteExchange implements MessageExchange<EditExecuteExchange.
         return "editExecute";
     }
 
-    @Override
-    public Class<EditExecuteExchange.Request> getRequestClass() {
-        return EditExecuteExchange.Request.class;
-    }
-
-    @Override
-    public Class<EditExecuteExchange.Response> getResponseClass() {
-        return EditExecuteExchange.Response.class;
-    }
-
     @Jacksonized
     @Builder
     @Value
     public static class Request implements RequestMessage {
-        @NonNull DataSourceReference ref;
+        @NonNull
+        DataSourceReference ref;
+
         @NonNull
         DataSourceConfigInstance config;
     }
@@ -42,5 +35,6 @@ public class EditExecuteExchange implements MessageExchange<EditExecuteExchange.
     @Builder
     @Value
     public static class Response implements ResponseMessage {
+        DataSourceId id;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/MessageExchange.java

@@ -20,12 +20,19 @@ public interface MessageExchange<RQ extends RequestMessage, RP extends ResponseM
     @SneakyThrows
     @SuppressWarnings("unchecked")
     default Class<RQ> getRequestClass() {
-        var name = getClass().getName() + "$Request";
+        var c = getClass().getSuperclass();
+        var name = (MessageExchange.class.isAssignableFrom(c) ? c : getClass()).getName() + "$Request";
         return (Class<RQ>) Class.forName(name);
     }
 
     /**
      * Returns the response class, needed for serialization.
      */
-    Class<RP> getResponseClass();
+    @SneakyThrows
+    @SuppressWarnings("unchecked")
+    default Class<RP> getResponseClass() {
+        var c = getClass().getSuperclass();
+        var name = (MessageExchange.class.isAssignableFrom(c) ? c : getClass()).getName() + "$Response";
+        return (Class<RP>) Class.forName(name);
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/RemoveCollectionExchange.java

@@ -0,0 +1,30 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class RemoveCollectionExchange implements MessageExchange<RemoveCollectionExchange.Request, RemoveCollectionExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "removeCollection";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        String collectionName;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/RemoveEntryExchange.java

@@ -0,0 +1,34 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import io.xpipe.core.source.DataSourceId;
+import io.xpipe.core.source.DataSourceReference;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class RemoveEntryExchange implements MessageExchange<RemoveEntryExchange.Request, RemoveEntryExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "removeEntry";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        DataSourceReference ref;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+        @NonNull
+        DataSourceId id;
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/RenameCollectionExchange.java

@@ -0,0 +1,32 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class RenameCollectionExchange implements MessageExchange<RenameCollectionExchange.Request, RenameCollectionExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "renameCollection";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        String collectionName;
+        @NonNull
+        String newName;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/RenameEntryExchange.java

@@ -0,0 +1,36 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import io.xpipe.core.source.DataSourceId;
+import io.xpipe.core.source.DataSourceReference;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class RenameEntryExchange implements MessageExchange<RenameEntryExchange.Request, RenameEntryExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "renameEntry";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        DataSourceReference ref;
+
+        @NonNull
+        String newName;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+        @NonNull DataSourceId newId;
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/api/QueryRawDataExchange.java

@@ -0,0 +1,34 @@
+package io.xpipe.beacon.exchange.api;
+
+import io.xpipe.beacon.exchange.MessageExchange;
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import io.xpipe.core.source.DataSourceReference;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class QueryRawDataExchange implements MessageExchange<QueryRawDataExchange.Request, QueryRawDataExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "queryRawData";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        DataSourceReference ref;
+
+        int maxBytes;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/api/QueryTextDataExchange.java

@@ -16,16 +16,6 @@ public class QueryTextDataExchange implements MessageExchange<QueryTextDataExcha
         return "queryTextData";
     }
 
-    @Override
-    public Class<QueryTextDataExchange.Request> getRequestClass() {
-        return QueryTextDataExchange.Request.class;
-    }
-
-    @Override
-    public Class<QueryTextDataExchange.Response> getResponseClass() {
-        return QueryTextDataExchange.Response.class;
-    }
-
     @Jacksonized
     @Builder
     @Value

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ConvertExchange.java

@@ -6,6 +6,7 @@ import io.xpipe.beacon.message.ResponseMessage;
 import io.xpipe.core.source.DataSourceConfigInstance;
 import io.xpipe.core.source.DataSourceId;
 import io.xpipe.core.source.DataSourceReference;
+import io.xpipe.core.source.DataSourceType;
 import lombok.Builder;
 import lombok.NonNull;
 import lombok.Value;
@@ -18,16 +19,6 @@ public class ConvertExchange implements MessageExchange<ConvertExchange.Request,
         return "convert";
     }
 
-    @Override
-    public Class<ConvertExchange.Request> getRequestClass() {
-        return ConvertExchange.Request.class;
-    }
-
-    @Override
-    public Class<ConvertExchange.Response> getResponseClass() {
-        return ConvertExchange.Response.class;
-    }
-
     @Jacksonized
     @Builder
     @Value
@@ -35,15 +26,18 @@ public class ConvertExchange implements MessageExchange<ConvertExchange.Request,
         @NonNull
         DataSourceReference ref;
 
-        @NonNull DataSourceId copyId;
+        String newProvider;
 
-        @NonNull
+        DataSourceType newType;
-        DataSourceConfigInstance config;
+
+        DataSourceId copyId;
     }
 
     @Jacksonized
     @Builder
     @Value
     public static class Response implements ResponseMessage {
+        @NonNull
+        DataSourceConfigInstance config;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ProviderListExchange.java

@@ -0,0 +1,35 @@
+package io.xpipe.beacon.exchange.cli;
+
+import io.xpipe.beacon.exchange.MessageExchange;
+import io.xpipe.beacon.exchange.data.ProviderEntry;
+import io.xpipe.beacon.message.RequestMessage;
+import io.xpipe.beacon.message.ResponseMessage;
+import io.xpipe.core.source.DataSourceType;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+import java.util.List;
+import java.util.Map;
+
+public class ProviderListExchange implements MessageExchange<ProviderListExchange.Request, ProviderListExchange.Response> {
+
+    @Override
+    public String getId() {
+        return "providerList";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+        @NonNull Map<DataSourceType, List<ProviderEntry>> entries;
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/cli/WriteExecuteExchange.java

@@ -5,7 +5,6 @@ import io.xpipe.beacon.message.RequestMessage;
 import io.xpipe.beacon.message.ResponseMessage;
 import io.xpipe.core.source.DataSourceConfigInstance;
 import io.xpipe.core.source.DataSourceReference;
-import io.xpipe.core.store.DataStore;
 import lombok.Builder;
 import lombok.NonNull;
 import lombok.Value;
@@ -21,16 +20,6 @@ public class WriteExecuteExchange implements MessageExchange<WriteExecuteExchang
         return "writeExecute";
     }
 
-    @Override
-    public Class<WriteExecuteExchange.Request> getRequestClass() {
-        return WriteExecuteExchange.Request.class;
-    }
-
-    @Override
-    public Class<WriteExecuteExchange.Response> getResponseClass() {
-        return WriteExecuteExchange.Response.class;
-    }
-
     @Jacksonized
     @Builder
     @Value
@@ -38,7 +27,6 @@ public class WriteExecuteExchange implements MessageExchange<WriteExecuteExchang
         @NonNull
         DataSourceReference ref;
 
-        DataStore dataStore;
         @NonNull
         DataSourceConfigInstance config;
     }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/WritePreparationExchange.java

@@ -5,7 +5,7 @@ import io.xpipe.beacon.message.RequestMessage;
 import io.xpipe.beacon.message.ResponseMessage;
 import io.xpipe.core.source.DataSourceConfigInstance;
 import io.xpipe.core.source.DataSourceReference;
-import io.xpipe.core.store.DataStore;
+import io.xpipe.core.store.StreamDataStore;
 import lombok.Builder;
 import lombok.NonNull;
 import lombok.Value;
@@ -21,22 +21,13 @@ public class WritePreparationExchange implements MessageExchange<WritePreparatio
         return "writePreparation";
     }
 
-    @Override
-    public Class<WritePreparationExchange.Request> getRequestClass() {
-        return WritePreparationExchange.Request.class;
-    }
-
-    @Override
-    public Class<WritePreparationExchange.Response> getResponseClass() {
-        return WritePreparationExchange.Response.class;
-    }
-
     @Jacksonized
     @Builder
     @Value
     public static class Request implements RequestMessage {
         String type;
-        String output;
+        @NonNull
+        StreamDataStore output;
         @NonNull
         DataSourceReference ref;
     }
@@ -45,8 +36,6 @@ public class WritePreparationExchange implements MessageExchange<WritePreparatio
     @Builder
     @Value
     public static class Response implements ResponseMessage {
-        DataStore store;
-
         @NonNull
         DataSourceConfigInstance config;
     }

beacon/src/main/java/io/xpipe/beacon/exchange/data/ProviderEntry.java

@@ -0,0 +1,14 @@
+package io.xpipe.beacon.exchange.data;
+
+import lombok.Builder;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+@Value
+@Jacksonized
+@Builder
+public class ProviderEntry {
+    String id;
+    String description;
+    boolean hidden;
+}

beacon/src/main/java/module-info.java

@@ -40,7 +40,13 @@ module io.xpipe.beacon {
             StoreStreamExchange,
             EditPreparationExchange,
             EditExecuteExchange,
+            RemoveEntryExchange,
+            RemoveCollectionExchange,
+            RenameCollectionExchange,
+            RenameEntryExchange,
+            ProviderListExchange,
             ConvertExchange,
+            QueryRawDataExchange,
             QueryTableDataExchange,
             VersionExchange;
 }

core/src/main/java/io/xpipe/core/data/node/MutableValueNode.java

@@ -14,6 +14,11 @@ public class MutableValueNode extends ValueNode {
         this.textual = textual;
     }
 
+    @Override
+    public String asString() {
+        return new String(data);
+    }
+
     @Override
     public String toString(int indent) {
         return (textual ? "\"" : "") + new String(data) + (textual ? "\"" : "") +  " (M)";

core/src/main/java/io/xpipe/core/source/DataSource.java

@@ -1,6 +1,11 @@
 package io.xpipe.core.source;
 
+import com.fasterxml.jackson.databind.util.TokenBuffer;
 import io.xpipe.core.store.DataStore;
+import io.xpipe.core.util.JacksonHelper;
+import lombok.AllArgsConstructor;
+import lombok.NonNull;
+import lombok.SneakyThrows;
 
 import java.util.Optional;
 
@@ -10,16 +15,25 @@ import java.util.Optional;
  *
  * This instance is only valid in combination with its associated data store instance.
  */
+@AllArgsConstructor
 public abstract class DataSource<DS extends DataStore> {
 
+    @NonNull
     protected DS store;
 
-    public DataSource(DS store) {
+    @SneakyThrows
-        this.store = store;
+    @SuppressWarnings("unchecked")
+    public DataSource<DS> copy() {
+        var mapper = JacksonHelper.newMapper();
+        TokenBuffer tb = new TokenBuffer(mapper, false);
+        mapper.writeValue(tb, this);
+        return mapper.readValue(tb.asParser(), getClass());
     }
 
-    public DataSource<DS> withStore(DS newStore) {
+    public DataSource<DS> withStore(DS store) {
-        return null;
+        var c = copy();
+        c.store = store;
+        return c;
     }
 
     /**

core/src/main/java/io/xpipe/core/source/DataSourceInfo.java

@@ -86,10 +86,12 @@ public abstract class DataSourceInfo {
     @Value
     @JsonTypeName("text")
     public static class Text extends DataSourceInfo {
+        int characters;
         int lineCount;
 
         @JsonCreator
-        public Text(int lineCount) {
+        public Text(int characters, int lineCount) {
+            this.characters = characters;
             this.lineCount = lineCount;
         }
 

core/src/main/java/io/xpipe/core/source/RawReadConnection.java

@@ -1,11 +1,21 @@
 package io.xpipe.core.source;
 
+import java.io.OutputStream;
+
 public interface RawReadConnection extends DataSourceReadConnection {
 
     byte[] readBytes(int max) throws Exception;
 
     int BUFFER_SIZE = 8192;
 
+    default void forwardBytes(OutputStream out, int maxBytes) throws Exception {
+        if (maxBytes == 0) {
+            return;
+        }
+
+        out.write(readBytes(maxBytes));
+    }
+
     default void forward(DataSourceConnection con) throws Exception {
         try (var tCon = (RawWriteConnection) con) {
             tCon.init();

core/src/main/java/io/xpipe/core/source/TextDataSource.java

@@ -2,6 +2,8 @@ package io.xpipe.core.source;
 
 import io.xpipe.core.store.DataStore;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 public abstract class TextDataSource<DS extends DataStore> extends DataSource<DS> {
 
     private static final int MAX_LINE_READ = 1000;
@@ -13,9 +15,14 @@ public abstract class TextDataSource<DS extends DataStore> extends DataSource<DS
     @Override
     public final DataSourceInfo determineInfo() throws Exception {
         try (var con = openReadConnection()) {
-            int count = (int) con.lines().limit(MAX_LINE_READ).count();
+            AtomicInteger lineCount = new AtomicInteger();
-            int usedCount = count == MAX_LINE_READ ? -1 : count;
+            AtomicInteger charCount = new AtomicInteger();
-            return new DataSourceInfo.Text(usedCount);
+            con.lines().limit(MAX_LINE_READ).forEach(s -> {
+                lineCount.getAndIncrement();
+                charCount.addAndGet(s.length());
+            });
+            boolean limitHit = lineCount.get() == MAX_LINE_READ;
+            return new DataSourceInfo.Text(limitHit ? -1 : charCount.get(), limitHit ? -1 : lineCount.get());
         }
     }
 

core/src/main/java/io/xpipe/core/store/StdinDataStore.java

@@ -0,0 +1,93 @@
+package io.xpipe.core.store;
+
+import lombok.EqualsAndHashCode;
+import lombok.Value;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+@EqualsAndHashCode
+@Value
+public class StdinDataStore implements StreamDataStore {
+
+    @Override
+    public InputStream openInput() throws Exception {
+        var in = System.in;
+        return new InputStream() {
+            @Override
+            public int read() throws IOException {
+                return in.read();
+            }
+
+            @Override
+            public int read(byte[] b) throws IOException {
+                return in.read(b);
+            }
+
+            @Override
+            public int read(byte[] b, int off, int len) throws IOException {
+                return in.read(b, off, len);
+            }
+
+            @Override
+            public byte[] readAllBytes() throws IOException {
+                return in.readAllBytes();
+            }
+
+            @Override
+            public byte[] readNBytes(int len) throws IOException {
+                return in.readNBytes(len);
+            }
+
+            @Override
+            public int readNBytes(byte[] b, int off, int len) throws IOException {
+                return in.readNBytes(b, off, len);
+            }
+
+            @Override
+            public long skip(long n) throws IOException {
+                return in.skip(n);
+            }
+
+            @Override
+            public void skipNBytes(long n) throws IOException {
+                in.skipNBytes(n);
+            }
+
+            @Override
+            public int available() throws IOException {
+                return in.available();
+            }
+
+            @Override
+            public void close() throws IOException {
+            }
+
+            @Override
+            public synchronized void mark(int readlimit) {
+                in.mark(readlimit);
+            }
+
+            @Override
+            public synchronized void reset() throws IOException {
+                in.reset();
+            }
+
+            @Override
+            public boolean markSupported() {
+                return in.markSupported();
+            }
+
+            @Override
+            public long transferTo(OutputStream out) throws IOException {
+                return in.transferTo(out);
+            }
+        };
+    }
+
+    @Override
+    public boolean exists() {
+        return false;
+    }
+}

core/src/main/java/io/xpipe/core/store/StdoutDataStore.java

@@ -0,0 +1,46 @@
+package io.xpipe.core.store;
+
+import lombok.EqualsAndHashCode;
+import lombok.Value;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+@EqualsAndHashCode
+@Value
+public class StdoutDataStore implements StreamDataStore {
+
+    @Override
+    public OutputStream openOutput() throws Exception {
+        return new OutputStream() {
+            @Override
+            public void write(int b) throws IOException {
+                System.out.write(b);
+            }
+
+            @Override
+            public void write(byte[] b) throws IOException {
+                System.out.write(b);
+            }
+
+            @Override
+            public void write(byte[] b, int off, int len) throws IOException {
+                System.out.write(b, off, len);
+            }
+
+            @Override
+            public void flush() throws IOException {
+                System.out.flush();
+            }
+
+            @Override
+            public void close() throws IOException {
+            }
+        };
+    }
+
+    @Override
+    public boolean exists() {
+        return false;
+    }
+}

extension/src/main/java/io/xpipe/extension/DataSourceProviders.java

@@ -107,13 +107,13 @@ public class DataSourceProviders {
                 .anyMatch(s -> s.equalsIgnoreCase(name))).findAny();
     }
 
-    public static Optional<DataSourceProvider<?>> byStore(DataStore store) {
+    public static Optional<DataSourceProvider<?>> byPreferredStore(DataStore store) {
         if (ALL == null) {
             throw new IllegalStateException("Not initialized");
         }
 
         return ALL.stream().filter(d -> d.getFileProvider() != null)
-                .filter(d -> d.couldSupportStore(store)).findAny();
+                .filter(d -> d.prefersStore(store)).findAny();
     }
 
     public static Set<DataSourceProvider<?>> getAll() {

extension/src/main/java/io/xpipe/extension/SimpleFileDataSourceProvider.java

@@ -18,8 +18,14 @@ public interface SimpleFileDataSourceProvider<T extends DataSource<?>> extends D
                 continue;
             }
 
+            for (var ext : e.getValue()) {
+                if (ext == null) {
+                    continue;
+                }
+
                 if (store instanceof FileDataStore l) {
-                return l.getFileName().matches("\\." + e.getValue() + "$");
+                    return l.getFileName().endsWith("." + ext);
+                }
             }
         }
         return false;