Many small fixes

.github/workflows/build.yml

@@ -1,6 +1,9 @@
 name: Build
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
 
 jobs:
   build:
@@ -14,8 +17,9 @@ jobs:
       - name: Set up GraalVM
         uses: graalvm/setup-graalvm@v1
         with:
-          version: '21.3.0'
-          java-version: '17'
+          version: '22.3.0'
+          java-version: '19'
+          github-token: ${{ secrets.XPIPE_GITHUB_TOKEN }}
 
       - name: Verify Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1

.github/workflows/publish.yml

@@ -17,8 +17,9 @@ jobs:
       - name: Set up GraalVM
         uses: graalvm/setup-graalvm@v1
         with:
-          version: '21.3.0'
-          java-version: '17'
+          version: '22.3.0'
+          java-version: '19'
+          github-token: ${{ secrets.XPIPE_GITHUB_TOKEN }}
 
       - name: Verify Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1

.gitignore

@@ -1,6 +1,9 @@
 .gradle/
 build/
 .idea
+local/
+local_test/
+local_stage/
 dev.properties
 extensions.txt
 local/

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "deps"]
-	path = deps
-	url = https://github.com/xpipe-io/xpipe_java_deps

api/build.gradle

@@ -5,11 +5,11 @@ plugins {
     id "org.moditect.gradleplugin" version "1.0.0-rc3"
 }
 
-apply from: "$projectDir/../deps/java.gradle"
-apply from: "$projectDir/../deps/junit.gradle"
+apply from: "$projectDir/../gradle_scripts/java.gradle"
+apply from: "$projectDir/../gradle_scripts/junit.gradle"
 
 System.setProperty('excludeExtensionLibrary', 'true')
-apply from: "$projectDir/../deps/extension_test.gradle"
+apply from: "$projectDir/../gradle_scripts/extension_test.gradle"
 
 version = file('../misc/version').text
 group = 'io.xpipe'
@@ -35,4 +35,4 @@ configurations {
 
 
 apply from: 'publish.gradle'
-apply from: "$projectDir/../deps/publish-base.gradle"
+apply from: "$projectDir/../gradle_scripts/publish-base.gradle"

api/src/main/java/io/xpipe/api/DataSource.java

@@ -15,10 +15,10 @@ import java.nio.file.Path;
 
 /**
  * Represents a reference to a data source that is managed by X-Pipe.
- *
+ * <p>
  * The actual data is only queried when required and is not cached.
  * Therefore, the queried data is always up-to-date at the point of calling a method that queries the data.
- *
+ * <p>
  * As soon a data source reference is created, the data source is locked
  * within X-Pipe to prevent concurrent modification and the problems that can arise from it.
  * By default, the lock is held until the calling program terminates and prevents
@@ -29,20 +29,29 @@ public interface DataSource {
 
     /**
      * NOT YET IMPLEMENTED!
-     *
+     * <p>
      * Creates a new supplier data source that will be interpreted as the generated data source.
      * In case this program should be a data source generator, this method has to be called at
      * least once to register that it actually generates a data source.
-     *
+     * <p>
      * All content that is written to this data source until the generator program terminates is
      * will be available later on when the data source is used as a supplier later on.
-     *
+     * <p>
      * In case this method is called multiple times, the same data source is returned.
      *
      * @return the generator data source
      */
-    @Deprecated
-    static DataSource supplySource() {
+    static DataSource drain() {
+        return null;
+    }
+
+    /**
+     * NOT YET IMPLEMENTED!
+     * <p>
+     * Creates a data source sink that will block with any read operations
+     * until an external data producer routes the output into this sink.
+     */
+    static DataSource sink() {
         return null;
     }
 
@@ -132,9 +141,9 @@ public interface DataSource {
     /**
      * Creates a new data source from an input stream.
      *
-     * @param id the data source id
+     * @param id   the data source id
      * @param type the data source type
-     * @param in the input stream to read
+     * @param in   the input stream to read
      * @return a {@link DataSource} instances that can be used to access the underlying data
      */
     public static DataSource create(DataSourceId id, String type, InputStream in) {
@@ -153,10 +162,11 @@ public interface DataSource {
 
     /**
      * Creates a new data source from an input stream.
-     *1
-     * @param id the data source id
+     * 1
+     *
+     * @param id   the data source id
      * @param type the data source type
-     * @param in the data store to add
+     * @param in   the data store to add
      * @return a {@link DataSource} instances that can be used to access the underlying data
      */
     public static DataSource create(DataSourceId id, String type, DataStore in) {

api/src/main/java/io/xpipe/api/DataStores.java

@@ -1,8 +1,8 @@
 package io.xpipe.api;
 
-import io.xpipe.api.connector.XPipeConnection;
-import io.xpipe.api.util.QuietDialogHandler;
+import io.xpipe.api.connector.XPipeApiConnection;
 import io.xpipe.beacon.exchange.cli.StoreAddExchange;
+import io.xpipe.beacon.util.QuietDialogHandler;
 import io.xpipe.core.store.DataStore;
 
 import java.util.Map;
@@ -10,7 +10,7 @@ import java.util.Map;
 public class DataStores {
 
     public static void addNamedStore(DataStore store, String name) {
-        XPipeConnection.execute(con -> {
+        XPipeApiConnection.execute(con -> {
             var req = StoreAddExchange.Request.builder()
                     .storeInput(store)
                     .name(name)

api/src/main/java/io/xpipe/api/DataTableAccumulator.java

@@ -8,7 +8,7 @@ import io.xpipe.core.source.DataSourceId;
 
 /**
  * An accumulator for table data.
- *
+ * <p>
  * This class can be used to construct new table data sources by
  * accumulating the rows using {@link #add(DataStructureNode)} or {@link #acceptor()} and then calling
  * {@link #finish(DataSourceId)} to complete the construction process and create a new data source.

api/src/main/java/io/xpipe/api/connector/XPipeApiConnection.java

@@ -1,23 +1,29 @@
 package io.xpipe.api.connector;
 
-import io.xpipe.beacon.*;
+import io.xpipe.beacon.BeaconClient;
+import io.xpipe.beacon.BeaconConnection;
+import io.xpipe.beacon.BeaconException;
+import io.xpipe.beacon.BeaconServer;
 import io.xpipe.beacon.exchange.cli.DialogExchange;
 import io.xpipe.core.dialog.DialogReference;
+import io.xpipe.core.util.XPipeDaemonMode;
+import io.xpipe.core.util.XPipeInstallation;
 
 import java.util.Optional;
 
-public final class XPipeConnection extends BeaconConnection {
+public final class XPipeApiConnection extends BeaconConnection {
 
-    private XPipeConnection() {}
+    private XPipeApiConnection() {
+    }
 
-    public static XPipeConnection open() {
-        var con = new XPipeConnection();
+    public static XPipeApiConnection open() {
+        var con = new XPipeApiConnection();
         con.constructSocket();
         return con;
     }
 
     public static void finishDialog(DialogReference reference) {
-        try (var con = new XPipeConnection()) {
+        try (var con = new XPipeApiConnection()) {
             con.constructSocket();
             var element = reference.getStart();
             while (true) {
@@ -26,8 +32,8 @@ public final class XPipeConnection extends BeaconConnection {
                 }
 
                 DialogExchange.Response response = con.performSimpleExchange(DialogExchange.Request.builder()
-                        .dialogKey(reference.getDialogId())
-                        .build());
+                                                                                     .dialogKey(reference.getDialogId())
+                                                                                     .build());
                 element = response.getElement();
                 if (response.getElement() == null) {
                     break;
@@ -41,7 +47,7 @@ public final class XPipeConnection extends BeaconConnection {
     }
 
     public static void execute(Handler handler) {
-        try (var con = new XPipeConnection()) {
+        try (var con = new XPipeApiConnection()) {
             con.constructSocket();
             handler.handle(con);
         } catch (BeaconException e) {
@@ -52,7 +58,7 @@ public final class XPipeConnection extends BeaconConnection {
     }
 
     public static <T> T execute(Mapper<T> mapper) {
-        try (var con = new XPipeConnection()) {
+        try (var con = new XPipeApiConnection()) {
             con.constructSocket();
             return mapper.handle(con);
         } catch (BeaconException e) {
@@ -73,7 +79,10 @@ public final class XPipeConnection extends BeaconConnection {
             } catch (InterruptedException ignored) {
             }
 
-            var s = BeaconClient.tryConnect(BeaconClient.ApiClientInformation.builder().version("?").language("Java").build());
+            var s = BeaconClient.tryConnect(BeaconClient.ApiClientInformation.builder()
+                                                    .version("?")
+                                                    .language("Java")
+                                                    .build());
             if (s.isPresent()) {
                 return s;
             }
@@ -114,28 +123,29 @@ public final class XPipeConnection extends BeaconConnection {
         }
 
         try {
-            beaconClient = BeaconClient.connect(BeaconClient.ApiClientInformation.builder().version("?").language("Java").build());
+            beaconClient = BeaconClient.connect(BeaconClient.ApiClientInformation.builder()
+                                                        .version("?")
+                                                        .language("Java")
+                                                        .build());
         } catch (Exception ex) {
             throw new BeaconException("Unable to connect to running xpipe daemon", ex);
         }
     }
 
     private void start() throws Exception {
-        if (BeaconServer.tryStart() == null) {
-            throw new UnsupportedOperationException("Unable to determine xpipe daemon launch command");
-        }
-        ;
+        var installation = XPipeInstallation.getLocalDefaultInstallationBasePath(true);
+        BeaconServer.start(installation, XPipeDaemonMode.BACKGROUND);
     }
 
     @FunctionalInterface
     public static interface Handler {
 
-        void handle(BeaconConnection con) throws ClientException, ServerException, ConnectorException;
+        void handle(BeaconConnection con) throws Exception;
     }
 
     @FunctionalInterface
     public static interface Mapper<T> {
 
-        T handle(BeaconConnection con) throws ClientException, ServerException, ConnectorException;
+        T handle(BeaconConnection con) throws Exception;
     }
 }

api/src/main/java/io/xpipe/api/impl/DataRawImpl.java

@@ -12,7 +12,8 @@ public class DataRawImpl extends DataSourceImpl implements DataRaw {
     public DataRawImpl(
             DataSourceId sourceId,
             DataSourceConfig sourceConfig,
-            io.xpipe.core.source.DataSource<?> internalSource) {
+            io.xpipe.core.source.DataSource<?> internalSource
+    ) {
         super(sourceId, sourceConfig, internalSource);
     }
 

api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java

@@ -2,7 +2,7 @@ package io.xpipe.api.impl;
 
 import io.xpipe.api.DataSource;
 import io.xpipe.api.DataSourceConfig;
-import io.xpipe.api.connector.XPipeConnection;
+import io.xpipe.api.connector.XPipeApiConnection;
 import io.xpipe.beacon.exchange.*;
 import io.xpipe.core.source.DataSourceId;
 import io.xpipe.core.source.DataSourceReference;
@@ -18,14 +18,15 @@ public abstract class DataSourceImpl implements DataSource {
     private final io.xpipe.core.source.DataSource<?> internalSource;
 
     public DataSourceImpl(
-            DataSourceId sourceId, DataSourceConfig config, io.xpipe.core.source.DataSource<?> internalSource) {
+            DataSourceId sourceId, DataSourceConfig config, io.xpipe.core.source.DataSource<?> internalSource
+    ) {
         this.sourceId = sourceId;
         this.config = config;
         this.internalSource = internalSource;
     }
 
     public static DataSource get(DataSourceReference ds) {
-        return XPipeConnection.execute(con -> {
+        return XPipeApiConnection.execute(con -> {
             var req = QueryDataSourceExchange.Request.builder().ref(ds).build();
             QueryDataSourceExchange.Response res = con.performSimpleExchange(req);
             var config = new DataSourceConfig(res.getProvider(), res.getConfig());
@@ -42,10 +43,8 @@ public abstract class DataSourceImpl implements DataSource {
                 case RAW -> {
                     yield new DataRawImpl(res.getId(), config, res.getInternalSource());
                 }
-                case COLLECTION -> throw new UnsupportedOperationException(
-                        "Unimplemented case: " + res.getType());
-                default -> throw new IllegalArgumentException(
-                        "Unexpected value: " + res.getType());
+                case COLLECTION -> throw new UnsupportedOperationException("Unimplemented case: " + res.getType());
+                default -> throw new IllegalArgumentException("Unexpected value: " + res.getType());
             };
         });
     }
@@ -53,7 +52,7 @@ public abstract class DataSourceImpl implements DataSource {
     public static DataSource create(DataSourceId id, io.xpipe.core.source.DataSource<?> source) {
         var startReq =
                 AddSourceExchange.Request.builder().source(source).target(id).build();
-        var returnedId = XPipeConnection.execute(con -> {
+        var returnedId = XPipeApiConnection.execute(con -> {
             AddSourceExchange.Response r = con.performSimpleExchange(startReq);
             return r.getId();
         });
@@ -64,17 +63,18 @@ public abstract class DataSourceImpl implements DataSource {
 
     public static DataSource create(DataSourceId id, String type, DataStore store) {
         if (store instanceof StreamDataStore s && s.isContentExclusivelyAccessible()) {
-            var res = XPipeConnection.execute(con -> {
-                var req = StoreStreamExchange.Request.builder().build();
-                StoreStreamExchange.Response r = con.performOutputExchange(req, out -> {
+            store = XPipeApiConnection.execute(con -> {
+                var internal = con.createInternalStreamStore();
+                var req = WriteStreamExchange.Request.builder()
+                        .name(internal.getUuid().toString())
+                        .build();
+                con.performOutputExchange(req, out -> {
                     try (InputStream inputStream = s.openInput()) {
                         inputStream.transferTo(out);
                     }
                 });
-                return r;
+                return internal;
             });
-
-            store = res.getStore();
         }
 
         var startReq = ReadExchange.Request.builder()
@@ -83,40 +83,43 @@ public abstract class DataSourceImpl implements DataSource {
                 .target(id)
                 .configureAll(false)
                 .build();
-        var startRes = XPipeConnection.execute(con -> {
+        var startRes = XPipeApiConnection.execute(con -> {
             ReadExchange.Response r = con.performSimpleExchange(startReq);
             return r;
         });
 
         var configInstance = startRes.getConfig();
-        XPipeConnection.finishDialog(configInstance);
+        XPipeApiConnection.finishDialog(configInstance);
 
         var ref = id != null ? DataSourceReference.id(id) : DataSourceReference.latest();
         return get(ref);
     }
 
     public static DataSource create(DataSourceId id, String type, InputStream in) {
-        var res = XPipeConnection.execute(con -> {
-            var req = StoreStreamExchange.Request.builder().build();
-            StoreStreamExchange.Response r = con.performOutputExchange(req, out -> in.transferTo(out));
-            return r;
+        var store = XPipeApiConnection.execute(con -> {
+            var internal = con.createInternalStreamStore();
+            var req = WriteStreamExchange.Request.builder()
+                    .name(internal.getUuid().toString())
+                    .build();
+            con.performOutputExchange(req, out -> {
+                in.transferTo(out);
+            });
+            return internal;
         });
 
-        var store = res.getStore();
-
         var startReq = ReadExchange.Request.builder()
                 .provider(type)
                 .store(store)
                 .target(id)
                 .configureAll(false)
                 .build();
-        var startRes = XPipeConnection.execute(con -> {
+        var startRes = XPipeApiConnection.execute(con -> {
             ReadExchange.Response r = con.performSimpleExchange(startReq);
             return r;
         });
 
         var configInstance = startRes.getConfig();
-        XPipeConnection.finishDialog(configInstance);
+        XPipeApiConnection.finishDialog(configInstance);
 
         var ref = id != null ? DataSourceReference.id(id) : DataSourceReference.latest();
         return get(ref);
@@ -124,7 +127,7 @@ public abstract class DataSourceImpl implements DataSource {
 
     @Override
     public void forwardTo(DataSource target) {
-        XPipeConnection.execute(con -> {
+        XPipeApiConnection.execute(con -> {
             var req = ForwardExchange.Request.builder()
                     .source(DataSourceReference.id(sourceId))
                     .target(DataSourceReference.id(target.getId()))
@@ -135,7 +138,7 @@ public abstract class DataSourceImpl implements DataSource {
 
     @Override
     public void appendTo(DataSource target) {
-        XPipeConnection.execute(con -> {
+        XPipeApiConnection.execute(con -> {
             var req = ForwardExchange.Request.builder()
                     .source(DataSourceReference.id(sourceId))
                     .target(DataSourceReference.id(target.getId()))

api/src/main/java/io/xpipe/api/impl/DataStructureImpl.java

@@ -11,7 +11,8 @@ public class DataStructureImpl extends DataSourceImpl implements DataStructure {
     DataStructureImpl(
             DataSourceId sourceId,
             DataSourceConfig sourceConfig,
-            io.xpipe.core.source.DataSource<?> internalSource) {
+            io.xpipe.core.source.DataSource<?> internalSource
+    ) {
         super(sourceId, sourceConfig, internalSource);
     }
 

api/src/main/java/io/xpipe/api/impl/DataTableAccumulatorImpl.java

@@ -3,16 +3,19 @@ package io.xpipe.api.impl;
 import io.xpipe.api.DataSource;
 import io.xpipe.api.DataTable;
 import io.xpipe.api.DataTableAccumulator;
-import io.xpipe.api.connector.XPipeConnection;
+import io.xpipe.api.connector.XPipeApiConnection;
 import io.xpipe.api.util.TypeDescriptor;
 import io.xpipe.beacon.BeaconException;
 import io.xpipe.beacon.exchange.ReadExchange;
-import io.xpipe.beacon.exchange.StoreStreamExchange;
+import io.xpipe.beacon.exchange.WriteStreamExchange;
+import io.xpipe.beacon.exchange.cli.StoreAddExchange;
+import io.xpipe.beacon.util.QuietDialogHandler;
 import io.xpipe.core.data.node.DataStructureNode;
 import io.xpipe.core.data.node.DataStructureNodeAcceptor;
 import io.xpipe.core.data.node.TupleNode;
 import io.xpipe.core.data.type.TupleType;
 import io.xpipe.core.data.typed.TypedDataStreamWriter;
+import io.xpipe.core.impl.InternalStreamStore;
 import io.xpipe.core.source.DataSourceId;
 import io.xpipe.core.source.DataSourceReference;
 
@@ -22,16 +25,23 @@ import java.nio.charset.StandardCharsets;
 
 public class DataTableAccumulatorImpl implements DataTableAccumulator {
 
-    private final XPipeConnection connection;
+    private final XPipeApiConnection connection;
     private final TupleType type;
     private int rows;
+    private InternalStreamStore store;
     private TupleType writtenDescriptor;
     private OutputStream bodyOutput;
 
     public DataTableAccumulatorImpl(TupleType type) {
         this.type = type;
-        connection = XPipeConnection.open();
-        connection.sendRequest(StoreStreamExchange.Request.builder().build());
+        connection = XPipeApiConnection.open();
+
+        store = new InternalStreamStore();
+        var addReq = StoreAddExchange.Request.builder().storeInput(store).name(store.getUuid().toString()).build();
+        StoreAddExchange.Response addRes = connection.performSimpleExchange(addReq);
+        QuietDialogHandler.handle(addRes.getConfig(), connection);
+
+        connection.sendRequest(WriteStreamExchange.Request.builder().name(store.getUuid().toString()).build());
         bodyOutput = connection.sendBody();
     }
 
@@ -43,21 +53,21 @@ public class DataTableAccumulatorImpl implements DataTableAccumulator {
             throw new BeaconException(e);
         }
 
-        StoreStreamExchange.Response res = connection.receiveResponse();
+        WriteStreamExchange.Response res = connection.receiveResponse();
         connection.close();
 
         var req = ReadExchange.Request.builder()
                 .target(id)
-                .store(res.getStore())
+                .store(store)
                 .provider("xpbt")
                 .configureAll(false)
                 .build();
-        ReadExchange.Response response = XPipeConnection.execute(con -> {
+        ReadExchange.Response response = XPipeApiConnection.execute(con -> {
             return con.performSimpleExchange(req);
         });
 
         var configInstance = response.getConfig();
-        XPipeConnection.finishDialog(configInstance);
+        XPipeApiConnection.finishDialog(configInstance);
 
         return DataSource.get(DataSourceReference.id(id)).asTable();
     }

api/src/main/java/io/xpipe/api/impl/DataTableImpl.java

@@ -2,7 +2,7 @@ package io.xpipe.api.impl;
 
 import io.xpipe.api.DataSourceConfig;
 import io.xpipe.api.DataTable;
-import io.xpipe.api.connector.XPipeConnection;
+import io.xpipe.api.connector.XPipeApiConnection;
 import io.xpipe.beacon.BeaconConnection;
 import io.xpipe.beacon.BeaconException;
 import io.xpipe.beacon.exchange.api.QueryTableDataExchange;
@@ -27,7 +27,8 @@ public class DataTableImpl extends DataSourceImpl implements DataTable {
     DataTableImpl(
             DataSourceId id,
             DataSourceConfig sourceConfig,
-            io.xpipe.core.source.DataSource<?> internalSource) {
+            io.xpipe.core.source.DataSource<?> internalSource
+    ) {
         super(id, sourceConfig, internalSource);
     }
 
@@ -55,7 +56,7 @@ public class DataTableImpl extends DataSourceImpl implements DataTable {
     @Override
     public ArrayNode read(int maxRows) {
         List<DataStructureNode> nodes = new ArrayList<>();
-        XPipeConnection.execute(con -> {
+        XPipeApiConnection.execute(con -> {
             var req = QueryTableDataExchange.Request.builder()
                     .ref(DataSourceReference.id(getId()))
                     .maxRows(maxRows)
@@ -73,6 +74,7 @@ public class DataTableImpl extends DataSourceImpl implements DataTable {
     public Iterator<TupleNode> iterator() {
         return new TableIterator();
     }
+
     ;
 
     private class TableIterator implements Iterator<TupleNode> {
@@ -83,7 +85,7 @@ public class DataTableImpl extends DataSourceImpl implements DataTable {
         private TupleNode node;
 
         {
-            connection = XPipeConnection.open();
+            connection = XPipeApiConnection.open();
             var req = QueryTableDataExchange.Request.builder()
                     .ref(DataSourceReference.id(getId()))
                     .maxRows(Integer.MAX_VALUE)

api/src/main/java/io/xpipe/api/impl/DataTextImpl.java

@@ -2,7 +2,7 @@ package io.xpipe.api.impl;
 
 import io.xpipe.api.DataSourceConfig;
 import io.xpipe.api.DataText;
-import io.xpipe.api.connector.XPipeConnection;
+import io.xpipe.api.connector.XPipeApiConnection;
 import io.xpipe.beacon.BeaconConnection;
 import io.xpipe.beacon.BeaconException;
 import io.xpipe.beacon.exchange.api.QueryTextDataExchange;
@@ -24,10 +24,11 @@ import java.util.stream.StreamSupport;
 
 public class DataTextImpl extends DataSourceImpl implements DataText {
 
-     DataTextImpl(
+    DataTextImpl(
             DataSourceId sourceId,
             DataSourceConfig sourceConfig,
-            io.xpipe.core.source.DataSource<?> internalSource) {
+            io.xpipe.core.source.DataSource<?> internalSource
+    ) {
         super(sourceId, sourceConfig, internalSource);
     }
 
@@ -62,7 +63,7 @@ public class DataTextImpl extends DataSourceImpl implements DataText {
             private String nextValue;
 
             {
-                connection = XPipeConnection.open();
+                connection = XPipeApiConnection.open();
                 var req = QueryTextDataExchange.Request.builder()
                         .ref(DataSourceReference.id(getId()))
                         .maxLines(-1)

api/src/main/java/io/xpipe/api/util/XPipeDaemonController.java

@@ -1,46 +0,0 @@
-package io.xpipe.api.util;
-
-import io.xpipe.api.connector.XPipeConnection;
-import io.xpipe.beacon.BeaconClient;
-import io.xpipe.beacon.BeaconServer;
-
-public class XPipeDaemonController {
-
-    private static boolean alreadyStarted;
-
-    public static void start() throws Exception {
-        if (BeaconServer.isRunning()) {
-            alreadyStarted = true;
-            return;
-        }
-
-        Process process = null;
-        if ((process = BeaconServer.tryStartCustom()) != null) {
-        } else {
-            if ((process = BeaconServer.tryStart()) == null) {
-                throw new AssertionError();
-            }
-        }
-
-        XPipeConnection.waitForStartup(process).orElseThrow();
-        if (!BeaconServer.isRunning()) {
-            throw new AssertionError();
-        }
-    }
-
-    public static void stop() throws Exception {
-        if (alreadyStarted) {
-            return;
-        }
-
-        if (!BeaconServer.isRunning()) {
-            return;
-        }
-
-        var client = BeaconClient.connect(BeaconClient.ApiClientInformation.builder().version("?").language("Java API Test").build());
-        if (!BeaconServer.tryStop(client)) {
-            throw new AssertionError();
-        }
-        XPipeConnection.waitForShutdown();
-    }
-}

api/src/test/java/io/xpipe/api/test/ApiTest.java

@@ -1,6 +1,7 @@
 package io.xpipe.api.test;
 
-import io.xpipe.api.util.XPipeDaemonController;
+import io.xpipe.beacon.BeaconDaemonController;
+import io.xpipe.core.util.XPipeDaemonMode;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 
@@ -8,11 +9,11 @@ public class ApiTest {
 
     @BeforeAll
     public static void setup() throws Exception {
-        XPipeDaemonController.start();
+        BeaconDaemonController.start(XPipeDaemonMode.TRAY);
     }
 
     @AfterAll
     public static void teardown() throws Exception {
-        XPipeDaemonController.stop();
+        BeaconDaemonController.stop();
     }
 }

api/src/test/java/io/xpipe/api/test/DataTableAccumulatorTest.java

@@ -5,11 +5,9 @@ import io.xpipe.core.data.node.TupleNode;
 import io.xpipe.core.data.node.ValueNode;
 import io.xpipe.core.data.type.TupleType;
 import io.xpipe.core.data.type.ValueType;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
-import java.util.OptionalInt;
 
 public class DataTableAccumulatorTest extends ApiTest {
 
@@ -23,8 +21,8 @@ public class DataTableAccumulatorTest extends ApiTest {
         acc.add(val);
         var table = acc.finish(":test");
 
-        Assertions.assertEquals(table.getInfo().getDataType(), TupleType.tableType(List.of("col1", "col2")));
-        Assertions.assertEquals(table.getInfo().getRowCountIfPresent(), OptionalInt.empty());
+        // Assertions.assertEquals(table.getInfo().getDataType(), TupleType.tableType(List.of("col1", "col2")));
+        // Assertions.assertEquals(table.getInfo().getRowCountIfPresent(), OptionalInt.empty());
         // var read = table.read(1).at(0);
         // Assertions.assertEquals(val, read);
     }

beacon/build.gradle

@@ -5,8 +5,8 @@ plugins {
     id "org.moditect.gradleplugin" version "1.0.0-rc3"
 }
 
-apply from: "$projectDir/../deps/java.gradle"
-apply from: "$projectDir/../deps/lombok.gradle"
+apply from: "$projectDir/../gradle_scripts/java.gradle"
+apply from: "$projectDir/../gradle_scripts/lombok.gradle"
 
 dependencies {
 }
@@ -24,4 +24,4 @@ dependencies {
 }
 
 apply from: 'publish.gradle'
-apply from: "$projectDir/../deps/publish-base.gradle"
+apply from: "$projectDir/../gradle_scripts/publish-base.gradle"

beacon/src/main/java/io/xpipe/beacon/BeaconClient.java

@@ -11,10 +11,13 @@ import com.fasterxml.jackson.databind.node.TextNode;
 import io.xpipe.beacon.exchange.MessageExchanges;
 import io.xpipe.beacon.exchange.data.ClientErrorMessage;
 import io.xpipe.beacon.exchange.data.ServerErrorMessage;
-import io.xpipe.core.store.ProcessControl;
+import io.xpipe.core.store.ShellStore;
+import io.xpipe.core.util.Deobfuscator;
 import io.xpipe.core.util.JacksonMapper;
+import io.xpipe.core.util.ProxyManagerProvider;
 import lombok.Builder;
 import lombok.EqualsAndHashCode;
+import lombok.Getter;
 import lombok.Value;
 import lombok.extern.jackson.Jacksonized;
 
@@ -30,69 +33,13 @@ import static io.xpipe.beacon.BeaconConfig.BODY_SEPARATOR;
 
 public class BeaconClient implements AutoCloseable {
 
-    @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
-    public static abstract class ClientInformation {
-
-        public final CliClientInformation cli() {
-            return (CliClientInformation) this;
-        }
-
-        public abstract String toDisplayString();
-    }
-
-    @JsonTypeName("cli")
-    @Value
-    @Builder
-    @Jacksonized
-    @EqualsAndHashCode(callSuper = false)
-    public static class CliClientInformation extends ClientInformation {
-
-        String version;
-        int consoleWidth;
-
-        @Override
-        public String toDisplayString() {
-            return "X-Pipe CLI " + version;
-        }
-    }
-
-    @JsonTypeName("gateway")
-    @Value
-    @Builder
-    @Jacksonized
-    @EqualsAndHashCode(callSuper = false)
-    public static class GatewayClientInformation extends ClientInformation {
-
-        String version;
-
-        @Override
-        public String toDisplayString() {
-            return "X-Pipe Gateway " + version;
-        }
-    }
-
-    @JsonTypeName("api")
-    @Value
-    @Builder
-    @Jacksonized
-    @EqualsAndHashCode(callSuper = false)
-    public static class ApiClientInformation extends ClientInformation {
-
-        String version;
-        String language;
-
-        @Override
-        public String toDisplayString() {
-            return String.format("X-Pipe %s API v%s", language, version);
-        }
-    }
-
-    private final Closeable closeable;
+    @Getter
+    private final Closeable base;
     private final InputStream in;
     private final OutputStream out;
 
-    private BeaconClient(Closeable closeable, InputStream in, OutputStream out) {
-        this.closeable = closeable;
+    private BeaconClient(Closeable base, InputStream in, OutputStream out) {
+        this.base = base;
         this.in = in;
         this.out = out;
     }
@@ -104,10 +51,49 @@ public class BeaconClient implements AutoCloseable {
         return client;
     }
 
-    public static BeaconClient connectGateway(ProcessControl control, GatewayClientInformation information) throws Exception {
-        var client = new BeaconClient(() -> {}, control.getStdout(), control.getStdin());
-        client.sendObject(JacksonMapper.newMapper().valueToTree(information));
-        return client;
+    public static BeaconClient connectProxy(ShellStore proxy) throws Exception {
+        var control = proxy.create().start();
+        if (!ProxyManagerProvider.get().setup(control)) {
+            throw new IOException("X-Pipe connector required to perform operation");
+        }
+        var command = control.command("xpipe beacon --raw").start();
+        command.discardErr();
+        return new BeaconClient(command, command.getStdout(), command.getStdin()) {
+
+            //            {
+            //                new Thread(() -> {
+            //                    while (true) {
+            //                        if (!control.isRunning()) {
+            //                            close();
+            //                        }
+            //                    }
+            //                })
+            //            }
+
+            @Override
+            public <T extends ResponseMessage> T receiveResponse()
+                    throws ConnectorException, ClientException, ServerException {
+                try {
+                    sendEOF();
+                    getRawOutputStream().close();
+                } catch (IOException ex) {
+                    throw new ConnectorException(ex);
+                }
+
+                return super.receiveResponse();
+            }
+
+            @Override
+            public void close() throws ConnectorException {
+                try {
+                    getRawInputStream().readAllBytes();
+                } catch (IOException ex) {
+                    throw new ConnectorException(ex);
+                }
+
+                super.close();
+            }
+        };
     }
 
     public static Optional<BeaconClient> tryConnect(ClientInformation information) {
@@ -120,7 +106,7 @@ public class BeaconClient implements AutoCloseable {
 
     public void close() throws ConnectorException {
         try {
-            closeable.close();
+            base.close();
         } catch (IOException ex) {
             throw new ConnectorException("Couldn't close client", ex);
         }
@@ -148,7 +134,7 @@ public class BeaconClient implements AutoCloseable {
     }
 
     public <T extends RequestMessage> void sendRequest(T req) throws ClientException, ConnectorException {
-        ObjectNode json = JacksonMapper.newMapper().valueToTree(req);
+        ObjectNode json = JacksonMapper.getDefault().valueToTree(req);
         var prov = MessageExchanges.byRequest(req);
         if (prov.isEmpty()) {
             throw new ClientException("Unknown request class " + req.getClass());
@@ -168,6 +154,13 @@ public class BeaconClient implements AutoCloseable {
         sendObject(msg);
     }
 
+    public void sendEOF() throws ConnectorException {
+        try (OutputStream ignored = BeaconFormat.writeBlocks(out)) {
+        } catch (IOException ex) {
+            throw new ConnectorException("Couldn't write to socket", ex);
+        }
+    }
+
     public void sendObject(JsonNode node) throws ConnectorException {
         var writer = new StringWriter();
         var mapper = JacksonMapper.newMapper();
@@ -229,8 +222,8 @@ public class BeaconClient implements AutoCloseable {
         }
 
         try {
-            var reader = JacksonMapper.newMapper().readerFor(ClientErrorMessage.class);
-            return Optional.of(reader.readValue(content));
+            var message = JacksonMapper.getDefault().treeToValue(content, ClientErrorMessage.class);
+            return Optional.of(message);
         } catch (IOException ex) {
             throw new ConnectorException("Couldn't parse client error message", ex);
         }
@@ -243,8 +236,9 @@ public class BeaconClient implements AutoCloseable {
         }
 
         try {
-            var reader = JacksonMapper.newMapper().readerFor(ServerErrorMessage.class);
-            return Optional.of(reader.readValue(content));
+            var message = JacksonMapper.getDefault().treeToValue(content, ServerErrorMessage.class);
+            Deobfuscator.deobfuscate(message.getError());
+            return Optional.of(message);
         } catch (IOException ex) {
             throw new ConnectorException("Couldn't parse server error message", ex);
         }
@@ -301,4 +295,89 @@ public class BeaconClient implements AutoCloseable {
 
         void run() throws E;
     }
+
+    @JsonTypeInfo(
+            use = JsonTypeInfo.Id.NAME,
+            property = "type"
+    )
+    public abstract static class ClientInformation {
+
+        public final CliClientInformation cli() {
+            return (CliClientInformation) this;
+        }
+
+        public abstract String toDisplayString();
+    }
+
+    @JsonTypeName("cli")
+    @Value
+    @Builder
+    @Jacksonized
+    @EqualsAndHashCode(callSuper = false)
+    public static class CliClientInformation extends ClientInformation {
+
+        int consoleWidth;
+
+        @Override
+        public String toDisplayString() {
+            return "X-Pipe CLI";
+        }
+    }
+
+    @JsonTypeName("reachableCheck")
+    @Value
+    @Builder
+    @Jacksonized
+    @EqualsAndHashCode(callSuper = false)
+    public static class ReachableCheckInformation extends ClientInformation {
+
+        @Override
+        public String toDisplayString() {
+            return "Reachable check";
+        }
+    }
+
+    @JsonTypeName("daemon")
+    @Value
+    @Builder
+    @Jacksonized
+    @EqualsAndHashCode(callSuper = false)
+    public static class DaemonInformation extends ClientInformation {
+
+        @Override
+        public String toDisplayString() {
+            return "Daemon";
+        }
+    }
+
+    @JsonTypeName("gateway")
+    @Value
+    @Builder
+    @Jacksonized
+    @EqualsAndHashCode(callSuper = false)
+    public static class GatewayClientInformation extends ClientInformation {
+
+        String version;
+
+        @Override
+        public String toDisplayString() {
+            return "X-Pipe Gateway " + version;
+        }
+    }
+
+    @JsonTypeName("api")
+    @Value
+    @Builder
+    @Jacksonized
+    @EqualsAndHashCode(callSuper = false)
+    public static class ApiClientInformation extends ClientInformation {
+
+        String version;
+        String language;
+
+        @Override
+        public String toDisplayString() {
+            return String.format("X-Pipe %s API v%s", language, version);
+        }
+    }
 }

beacon/src/main/java/io/xpipe/beacon/BeaconConfig.java

@@ -15,7 +15,15 @@ public class BeaconConfig {
     private static final String ATTACH_DEBUGGER_PROP = "io.xpipe.beacon.attachDebuggerToDaemon";
     private static final String EXEC_DEBUG_PROP = "io.xpipe.beacon.printDaemonOutput";
     private static final String EXEC_PROCESS_PROP = "io.xpipe.beacon.customDaemonCommand";
-    private static final String DAEMON_ARGUMENTS_PROP = "io.xpipe.beacon.daemonArgs";
+    public static final String DAEMON_ARGUMENTS_PROP = "io.xpipe.beacon.daemonArgs";
+    private static final String LOCAL_PROXY_PROP = "io.xpipe.beacon.localProxy";
+
+    public static boolean localProxy() {
+        if (System.getProperty(LOCAL_PROXY_PROP) != null) {
+            return Boolean.parseBoolean(System.getProperty(LOCAL_PROXY_PROP));
+        }
+        return false;
+    }
 
     public static boolean printMessages() {
         if (System.getProperty(PRINT_MESSAGES_PROPERTY) != null) {

beacon/src/main/java/io/xpipe/beacon/BeaconConnection.java

@@ -1,5 +1,10 @@
 package io.xpipe.beacon;
 
+import io.xpipe.beacon.exchange.WriteStreamExchange;
+import io.xpipe.beacon.exchange.cli.StoreAddExchange;
+import io.xpipe.beacon.util.QuietDialogHandler;
+import io.xpipe.core.impl.InternalStreamStore;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -73,7 +78,8 @@ public abstract class BeaconConnection implements AutoCloseable {
     }
 
     public <REQ extends RequestMessage, RES extends ResponseMessage> void performInputExchange(
-            REQ req, BeaconClient.FailableBiConsumer<RES, InputStream, Exception> responseConsumer) {
+            REQ req, BeaconClient.FailableBiConsumer<RES, InputStream, Exception> responseConsumer
+    ) {
         checkClosed();
 
         performInputOutputExchange(req, null, responseConsumer);
@@ -82,7 +88,8 @@ public abstract class BeaconConnection implements AutoCloseable {
     public <REQ extends RequestMessage, RES extends ResponseMessage> void performInputOutputExchange(
             REQ req,
             BeaconClient.FailableConsumer<OutputStream, IOException> reqWriter,
-            BeaconClient.FailableBiConsumer<RES, InputStream, Exception> responseConsumer) {
+            BeaconClient.FailableBiConsumer<RES, InputStream, Exception> responseConsumer
+    ) {
         checkClosed();
 
         try {
@@ -144,7 +151,8 @@ public abstract class BeaconConnection implements AutoCloseable {
     }
 
     public <REQ extends RequestMessage, RES extends ResponseMessage> RES performOutputExchange(
-            REQ req, BeaconClient.FailableConsumer<OutputStream, Exception> reqWriter) {
+            REQ req, BeaconClient.FailableConsumer<OutputStream, Exception> reqWriter
+    ) {
         checkClosed();
 
         try {
@@ -169,6 +177,27 @@ public abstract class BeaconConnection implements AutoCloseable {
         }
     }
 
+    public InternalStreamStore createInternalStreamStore() {
+        return createInternalStreamStore(null);
+    }
+
+    public InternalStreamStore createInternalStreamStore(String name) {
+        var store = new InternalStreamStore();
+        var addReq = StoreAddExchange.Request.builder().storeInput(store).name(name != null ? name : store.getUuid().toString()).build();
+        StoreAddExchange.Response addRes = performSimpleExchange(addReq);
+        QuietDialogHandler.handle(addRes.getConfig(), this);
+        return store;
+    }
+
+    public void writeStream(InternalStreamStore s, InputStream in) {
+        writeStream(s.getUuid().toString(), in);
+    }
+
+    public void writeStream(String name, InputStream in) {
+        performOutputExchange(
+                WriteStreamExchange.Request.builder().name(name).build(), in::transferTo);
+    }
+
     private BeaconException unwrapException(Exception exception) {
         if (exception instanceof ServerException s) {
             return new BeaconException("An internal server error occurred", s);

beacon/src/main/java/io/xpipe/beacon/BeaconDaemonController.java

@@ -0,0 +1,89 @@
+package io.xpipe.beacon;
+
+import io.xpipe.core.util.XPipeDaemonMode;
+import io.xpipe.core.util.XPipeInstallation;
+
+import java.io.IOException;
+
+public class BeaconDaemonController {
+
+    private static boolean alreadyStarted;
+
+    public static void start(XPipeDaemonMode mode) throws Exception {
+        if (BeaconServer.isRunning()) {
+            alreadyStarted = true;
+            return;
+        }
+
+        var custom = false;
+        Process process;
+        if ((process = BeaconServer.tryStartCustom()) != null) {
+            custom = true;
+        } else {
+            var defaultBase = XPipeInstallation.getLocalDefaultInstallationBasePath(true);
+            process = BeaconServer.start(defaultBase, mode);
+        }
+
+        waitForStartup(process, custom);
+        if (!BeaconServer.isRunning()) {
+            throw new AssertionError();
+        }
+    }
+
+    public static void stop() throws Exception {
+        if (alreadyStarted) {
+            return;
+        }
+
+        if (!BeaconServer.isRunning()) {
+            return;
+        }
+
+        var client = BeaconClient.connect(BeaconClient.ApiClientInformation.builder().version("?").language("Java API Test").build());
+        if (!BeaconServer.tryStop(client)) {
+            throw new AssertionError();
+        }
+        waitForShutdown();
+    }
+
+    private static void waitForStartup(Process process, boolean custom) throws IOException {
+        for (int i = 0; i < 160; i++) {
+            if (process != null && !custom && !process.isAlive()) {
+                throw new IOException("Daemon start failed");
+            }
+
+            if (process != null && custom && !process.isAlive() && process.exitValue() != 0) {
+                throw new IOException("Custom launch command failed");
+            }
+
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException ignored) {
+            }
+
+            var s = BeaconClient.tryConnect(BeaconClient.ApiClientInformation.builder()
+                                                    .version("?")
+                                                    .language("Java")
+                                                    .build());
+            if (s.isPresent()) {
+                return;
+            }
+        }
+
+        throw new IOException("Wait for daemon start up timed out");
+    }
+
+    private static void waitForShutdown() {
+        for (int i = 0; i < 40; i++) {
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException ignored) {
+            }
+
+            var r = BeaconServer.isRunning();
+            if (!r) {
+                return;
+            }
+        }
+    }
+}

beacon/src/main/java/io/xpipe/beacon/BeaconException.java

@@ -5,7 +5,8 @@ package io.xpipe.beacon;
  */
 public class BeaconException extends RuntimeException {
 
-    public BeaconException() {}
+    public BeaconException() {
+    }
 
     public BeaconException(String message) {
         super(message);

beacon/src/main/java/io/xpipe/beacon/BeaconJacksonModule.java

@@ -10,6 +10,8 @@ public class BeaconJacksonModule extends SimpleModule {
         context.registerSubtypes(
                 new NamedType(BeaconClient.ApiClientInformation.class),
                 new NamedType(BeaconClient.CliClientInformation.class),
-                new NamedType(BeaconClient.GatewayClientInformation.class));
+                new NamedType(BeaconClient.DaemonInformation.class),
+                new NamedType(BeaconClient.ReachableCheckInformation.class)
+        );
     }
 }

beacon/src/main/java/io/xpipe/beacon/BeaconProxyImpl.java

@@ -0,0 +1,147 @@
+package io.xpipe.beacon;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import io.xpipe.beacon.exchange.ProxyFunctionExchange;
+import io.xpipe.beacon.exchange.ProxyReadConnectionExchange;
+import io.xpipe.beacon.exchange.ProxyWriteConnectionExchange;
+import io.xpipe.core.impl.InputStreamStore;
+import io.xpipe.core.impl.OutputStreamStore;
+import io.xpipe.core.source.DataSource;
+import io.xpipe.core.source.DataSourceConnection;
+import io.xpipe.core.source.DataSourceReadConnection;
+import io.xpipe.core.source.WriteMode;
+import io.xpipe.core.store.ShellStore;
+import io.xpipe.core.util.JacksonMapper;
+import io.xpipe.core.util.ProxyFunction;
+import io.xpipe.core.util.ProxyProvider;
+import io.xpipe.core.util.Proxyable;
+import lombok.SneakyThrows;
+
+import java.io.FilterInputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.function.Function;
+
+public class BeaconProxyImpl extends ProxyProvider {
+
+    private static JsonNode replace(JsonNode node, Function<JsonNode, Optional<JsonNode>> function) {
+        var value = function.apply(node);
+        if (value.isPresent()) {
+            return value.get();
+        }
+
+        if (!node.isObject()) {
+            return node;
+        }
+
+        var replacement = JsonNodeFactory.instance.objectNode();
+        var iterator = node.fields();
+        while (iterator.hasNext()) {
+            var stringJsonNodeEntry = iterator.next();
+            var resolved = function.apply(stringJsonNodeEntry.getValue()).orElse(stringJsonNodeEntry.getValue());
+            replacement.set(stringJsonNodeEntry.getKey(), resolved);
+        }
+        return replacement;
+    }
+
+    @Override
+    @SneakyThrows
+    @SuppressWarnings("unchecked")
+    public <T> T downstreamTransform(T object, ShellStore proxy) {
+        var proxyNode = JacksonMapper.getDefault().valueToTree(proxy);
+        var inputNode = JacksonMapper.getDefault().valueToTree(object);
+        var localNode = JacksonMapper.getDefault().valueToTree(ShellStore.local());
+        var result = replace(inputNode, node -> node.equals(proxyNode) ? Optional.of(localNode) : Optional.empty());
+        return (T) JacksonMapper.getDefault().treeToValue(result, object.getClass());
+    }
+
+    @Override
+    public ShellStore getProxy(Object base) {
+        var proxy = base instanceof Proxyable p ? p.getProxy() : null;
+        return ShellStore.isLocal(proxy) ? (BeaconConfig.localProxy() ? proxy : null) : proxy;
+    }
+
+    @Override
+    public boolean isRemote(Object base) {
+        if (base == null) {
+            throw new IllegalArgumentException("Proxy base is null");
+        }
+
+        return getProxy(base) != null;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public <T extends DataSourceReadConnection> T createRemoteReadConnection(DataSource<?> source, ShellStore proxy) throws Exception {
+        var downstream = downstreamTransform(source, proxy);
+
+        BeaconClient client = null;
+        try {
+            client = BeaconClient.connectProxy(proxy);
+            client.sendRequest(ProxyReadConnectionExchange.Request.builder()
+                                       .source(downstream)
+                                       .build());
+            client.receiveResponse();
+            BeaconClient finalClient = client;
+            var inputStream = new FilterInputStream(finalClient.receiveBody()) {
+                @Override
+                @SneakyThrows
+                public void close() throws IOException {
+                    super.close();
+                    finalClient.close();
+                }
+            };
+            var inputSource = DataSource.createInternalDataSource(source.getType(), new InputStreamStore(inputStream));
+            return (T) inputSource.openReadConnection();
+        } catch (Exception ex) {
+            if (client != null) {
+                client.close();
+            }
+            throw ex;
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public <T extends DataSourceConnection> T createRemoteWriteConnection(DataSource<?> source, WriteMode mode, ShellStore proxy) throws Exception {
+        var downstream = downstreamTransform(source, proxy);
+
+        BeaconClient client = null;
+        try {
+            client = BeaconClient.connectProxy(proxy);
+            client.sendRequest(ProxyWriteConnectionExchange.Request.builder()
+                                       .source(downstream)
+                                       .build());
+            BeaconClient finalClient = client;
+            var outputStream = new FilterOutputStream(client.sendBody()) {
+                @Override
+                @SneakyThrows
+                public void close() throws IOException {
+                    super.close();
+                    finalClient.receiveResponse();
+                    finalClient.close();
+                }
+            };
+            var outputSource = DataSource.createInternalDataSource(source.getType(), new OutputStreamStore(outputStream));
+            return (T) outputSource.openWriteConnection(mode);
+        } catch (Exception ex) {
+            if (client != null) {
+                client.close();
+            }
+            throw ex;
+        }
+    }
+
+    @Override
+    @SneakyThrows
+    public ProxyFunction call(ProxyFunction func, ShellStore proxy) {
+        try (var client = BeaconClient.connectProxy(proxy)) {
+            client.sendRequest(
+                    ProxyFunctionExchange.Request.builder().function(func).build());
+            ProxyFunctionExchange.Response response = client.receiveResponse();
+            return response.getFunction();
+        }
+    }
+}

beacon/src/main/java/io/xpipe/beacon/BeaconServer.java

@@ -1,30 +1,24 @@
 package io.xpipe.beacon;
 
 import io.xpipe.beacon.exchange.StopExchange;
-import lombok.experimental.UtilityClass;
+import io.xpipe.core.impl.FileNames;
+import io.xpipe.core.process.OsType;
+import io.xpipe.core.process.ShellTypes;
+import io.xpipe.core.util.XPipeDaemonMode;
+import io.xpipe.core.util.XPipeInstallation;
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Optional;
+import java.util.List;
 
 /**
  * Contains basic functionality to start, communicate, and stop a remote beacon server.
  */
-@UtilityClass
 public class BeaconServer {
 
-    public static void main(String[] args) throws Exception {
-        if (tryStartCustom() == null) {
-            if (tryStart() == null) {
-                System.exit(1);
-            }
-        }
-    }
-
     public static boolean isRunning() {
-        try (var socket = BeaconClient.connect(null)) {
+        try (var ignored = BeaconClient.connect(
+                BeaconClient.ReachableCheckInformation.builder().build())) {
             return true;
         } catch (Exception e) {
             return false;
@@ -34,72 +28,78 @@ public class BeaconServer {
     public static Process tryStartCustom() throws Exception {
         var custom = BeaconConfig.getCustomDaemonCommand();
         if (custom != null) {
-            var command =
-                    custom + " " + (BeaconConfig.getDaemonArguments() != null ? BeaconConfig.getDaemonArguments() : "");
-            Process process = Runtime.getRuntime().exec(command);
+            var command = ShellTypes.getPlatformDefault()
+                    .executeCommandListWithShell(custom
+                                                         + (BeaconConfig.getDaemonArguments() != null
+                            ? " " + BeaconConfig.getDaemonArguments()
+                            : ""));
+            Process process = Runtime.getRuntime().exec(command.toArray(String[]::new));
             printDaemonOutput(process, command);
             return process;
         }
         return null;
     }
 
-    public static Process tryStart() throws Exception {
-        var daemonExecutable = getDaemonExecutable();
-        if (daemonExecutable.isPresent()) {
-            var command = "\"" + daemonExecutable.get() + "\" --external "
-                    + (BeaconConfig.getDaemonArguments() != null ? BeaconConfig.getDaemonArguments() : "");
-            // Tell daemon that we launched from an external tool
-            Process process = Runtime.getRuntime().exec(command);
-            printDaemonOutput(process, command);
-            return process;
+    public static Process start(String installationBase, XPipeDaemonMode mode) throws Exception {
+        String command;
+        if (!BeaconConfig.launchDaemonInDebugMode()) {
+            command = XPipeInstallation.createExternalAsyncLaunchCommand(installationBase, mode, BeaconConfig.getDaemonArguments());
+        } else {
+            command = XPipeInstallation.createExternalLaunchCommand(
+                    getDaemonDebugExecutable(installationBase), BeaconConfig.getDaemonArguments(), mode);
         }
 
-        return null;
+        var fullCommand = ShellTypes.getPlatformDefault().executeCommandListWithShell(command);
+        Process process = new ProcessBuilder(fullCommand).start();
+        printDaemonOutput(process, fullCommand);
+        return process;
     }
 
-    private static void printDaemonOutput(Process proc, String command) {
+    private static void printDaemonOutput(Process proc, List<String> command) {
         boolean print = BeaconConfig.printDaemonOutput();
         if (print) {
             System.out.println("Starting daemon: " + command);
         }
 
         var out = new Thread(
-                        null,
-                        () -> {
-                            try {
-                                InputStreamReader isr = new InputStreamReader(proc.getInputStream());
-                                BufferedReader br = new BufferedReader(isr);
-                                String line;
-                                while ((line = br.readLine()) != null) {
-                                    if (print) {
-                                        System.out.println("[xpiped] " + line);
-                                    }
-                                }
-                            } catch (Exception ioe) {
-                                ioe.printStackTrace();
+                null,
+                () -> {
+                    try {
+                        InputStreamReader isr = new InputStreamReader(proc.getInputStream());
+                        BufferedReader br = new BufferedReader(isr);
+                        String line;
+                        while ((line = br.readLine()) != null) {
+                            if (print) {
+                                System.out.println("[xpiped] " + line);
                             }
-                        },
-                        "daemon sysout");
+                        }
+                    } catch (Exception ioe) {
+                        ioe.printStackTrace();
+                    }
+                },
+                "daemon sysout"
+        );
         out.setDaemon(true);
         out.start();
 
         var err = new Thread(
-                        null,
-                        () -> {
-                            try {
-                                InputStreamReader isr = new InputStreamReader(proc.getErrorStream());
-                                BufferedReader br = new BufferedReader(isr);
-                                String line;
-                                while ((line = br.readLine()) != null) {
-                                    if (print) {
-                                        System.err.println("[xpiped] " + line);
-                                    }
-                                }
-                            } catch (Exception ioe) {
-                                ioe.printStackTrace();
+                null,
+                () -> {
+                    try {
+                        InputStreamReader isr = new InputStreamReader(proc.getErrorStream());
+                        BufferedReader br = new BufferedReader(isr);
+                        String line;
+                        while ((line = br.readLine()) != null) {
+                            if (print) {
+                                System.err.println("[xpiped] " + line);
                             }
-                        },
-                        "daemon syserr");
+                        }
+                    } catch (Exception ioe) {
+                        ioe.printStackTrace();
+                    }
+                },
+                "daemon syserr"
+        );
         err.setDaemon(true);
         err.start();
     }
@@ -110,62 +110,18 @@ public class BeaconServer {
         return res.isSuccess();
     }
 
-    private static Optional<Path> getDaemonBasePath() {
-        Path base = null;
-        // Prepare for invalid XPIPE_HOME path value
-        try {
-            var environmentVariable = System.getenv("XPIPE_HOME");
-            base = environmentVariable != null ? Path.of(environmentVariable) : null;
-        } catch (Exception ex) {
-        }
-
-        if (base == null) {
-            if (System.getProperty("os.name").startsWith("Windows")) {
-                base = Path.of(System.getenv("LOCALAPPDATA"), "X-Pipe");
-            } else {
-                base = Path.of("/opt/xpipe/");
-            }
-            if (!Files.exists(base)) {
-                base = null;
-            }
-        }
-
-        return Optional.ofNullable(base);
-    }
-
-    public static Optional<Path> getDaemonExecutable() {
-        var base = getDaemonBasePath().orElseThrow();
+    public static String getDaemonDebugExecutable(String installationBase) throws Exception {
+        var osType = OsType.getLocal();
         var debug = BeaconConfig.launchDaemonInDebugMode();
-        Path executable = null;
         if (!debug) {
-            if (System.getProperty("os.name").startsWith("Windows")) {
-                executable = Path.of("app", "runtime", "bin", "xpiped.bat");
-            } else {
-                executable = Path.of("app/bin/xpiped");
-            }
-
+            throw new IllegalStateException();
         } else {
-            String scriptName = null;
             if (BeaconConfig.attachDebuggerToDaemon()) {
-                scriptName = "xpiped_debug_attach";
+                return FileNames.join(
+                        installationBase, XPipeInstallation.getDaemonDebugAttachScriptPath(osType));
             } else {
-                scriptName = "xpiped_debug";
+                return FileNames.join(installationBase, XPipeInstallation.getDaemonDebugScriptPath(osType));
             }
-
-            if (System.getProperty("os.name").startsWith("Windows")) {
-                scriptName = scriptName + ".bat";
-            } else {
-                scriptName = scriptName + ".sh";
-            }
-
-            executable = Path.of("app", "scripts", scriptName);
-        }
-
-        Path file = base.resolve(executable);
-        if (Files.exists(file)) {
-            return Optional.of(file);
-        } else {
-            return Optional.empty();
         }
     }
 }

beacon/src/main/java/io/xpipe/beacon/ClientException.java

@@ -5,7 +5,8 @@ package io.xpipe.beacon;
  */
 public class ClientException extends Exception {
 
-    public ClientException() {}
+    public ClientException() {
+    }
 
     public ClientException(String message) {
         super(message);

beacon/src/main/java/io/xpipe/beacon/ConnectorException.java

@@ -5,7 +5,8 @@ package io.xpipe.beacon;
  */
 public class ConnectorException extends Exception {
 
-    public ConnectorException() {}
+    public ConnectorException() {
+    }
 
     public ConnectorException(String message) {
         super(message);

beacon/src/main/java/io/xpipe/beacon/RequestMessage.java

@@ -1,3 +1,4 @@
 package io.xpipe.beacon;
 
-public interface RequestMessage {}
+public interface RequestMessage {
+}

beacon/src/main/java/io/xpipe/beacon/ResponseMessage.java

@@ -1,3 +1,4 @@
 package io.xpipe.beacon;
 
-public interface ResponseMessage {}
+public interface ResponseMessage {
+}

beacon/src/main/java/io/xpipe/beacon/SecretProviderImpl.java

@@ -0,0 +1,73 @@
+package io.xpipe.beacon;
+
+import io.xpipe.core.util.SecretProvider;
+import lombok.SneakyThrows;
+
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.KeySpec;
+import java.util.Random;
+
+public class SecretProviderImpl extends SecretProvider {
+
+    private static final String ENCRYPT_ALGO = "AES/GCM/NoPadding";
+    private static final int TAG_LENGTH_BIT = 128;
+    private static final int IV_LENGTH_BYTE = 12;
+    private static final int AES_KEY_BIT = 128;
+    private static final byte[] IV = getFixedNonce(IV_LENGTH_BYTE);
+
+    private static byte[] getFixedNonce(int numBytes) {
+        byte[] nonce = new byte[numBytes];
+        new SecureRandom(new byte[] {1, -28, 123}).nextBytes(nonce);
+        return nonce;
+    }
+
+
+    private static SecretKey getAESKey(int keysize) throws NoSuchAlgorithmException, InvalidKeySpecException {
+        SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256");
+        var salt = new byte[16];
+        new Random(keysize).nextBytes(salt);
+        KeySpec spec = new PBEKeySpec(new char[] {'X', 'P', 'E' << 1}, salt, 65536, keysize);
+        SecretKey secret = new SecretKeySpec(factory.generateSecret(spec).getEncoded(), "AES");
+        return secret;
+    }
+
+    @Override
+    @SneakyThrows
+    public byte[] encrypt(byte[] c) {
+        SecretKey secretKey = getAESKey(AES_KEY_BIT);
+        Cipher cipher = Cipher.getInstance(ENCRYPT_ALGO);
+        cipher.init(Cipher.ENCRYPT_MODE, secretKey, new GCMParameterSpec(TAG_LENGTH_BIT, IV));
+        var bytes = cipher.doFinal(c);
+        bytes = ByteBuffer.allocate(IV.length + bytes.length)
+                .order(ByteOrder.LITTLE_ENDIAN)
+                .put(IV)
+                .put(bytes)
+                .array();
+        return bytes;
+    }
+
+    @Override
+    @SneakyThrows
+    public byte[] decrypt(byte[] c) {
+        ByteBuffer bb = ByteBuffer.wrap(c).order(ByteOrder.LITTLE_ENDIAN);
+        byte[] iv = new byte[IV_LENGTH_BYTE];
+        bb.get(iv);
+        byte[] cipherText = new byte[bb.remaining()];
+        bb.get(cipherText);
+
+        SecretKey secretKey = getAESKey(AES_KEY_BIT);
+        Cipher cipher = Cipher.getInstance(ENCRYPT_ALGO);
+        cipher.init(Cipher.DECRYPT_MODE, secretKey, new GCMParameterSpec(TAG_LENGTH_BIT, iv));
+        return cipher.doFinal(cipherText);
+    }
+}

beacon/src/main/java/io/xpipe/beacon/ServerException.java

@@ -5,7 +5,8 @@ package io.xpipe.beacon;
  */
 public class ServerException extends Exception {
 
-    public ServerException() {}
+    public ServerException() {
+    }
 
     public ServerException(String message) {
         super(message);

beacon/src/main/java/io/xpipe/beacon/exchange/FocusExchange.java

@@ -2,21 +2,17 @@ package io.xpipe.beacon.exchange;
 
 import io.xpipe.beacon.RequestMessage;
 import io.xpipe.beacon.ResponseMessage;
-import io.xpipe.core.source.DataSourceId;
-import io.xpipe.core.store.DataStore;
+import io.xpipe.core.util.XPipeDaemonMode;
 import lombok.Builder;
 import lombok.NonNull;
 import lombok.Value;
 import lombok.extern.jackson.Jacksonized;
 
-/**
- * Sends stream-based data to a daemon.
- */
-public class ReadExecuteExchange implements MessageExchange {
+public class FocusExchange implements MessageExchange {
 
     @Override
     public String getId() {
-        return "readExecute";
+        return "focus";
     }
 
     @Jacksonized
@@ -24,13 +20,12 @@ public class ReadExecuteExchange implements MessageExchange {
     @Value
     public static class Request implements RequestMessage {
         @NonNull
-        DataStore dataStore;
-
-        DataSourceId target;
+        XPipeDaemonMode mode;
     }
 
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/ForwardExchange.java

@@ -31,5 +31,6 @@ public class ForwardExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/MessageExchanges.java

@@ -12,10 +12,13 @@ public class MessageExchanges {
 
     private static Set<MessageExchange> ALL;
 
-    private static void loadAll() {
+    public static void loadAll() {
         if (ALL == null) {
             ALL = ServiceLoader.load(MessageExchange.class).stream()
-                    .map(ServiceLoader.Provider::get)
+                    .map(s -> {
+                        var ex = (MessageExchange) s.get();
+                        return ex;
+                    })
                     .collect(Collectors.toSet());
         }
     }

beacon/src/main/java/io/xpipe/beacon/exchange/OpenExchange.java

@@ -0,0 +1,31 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+import java.util.List;
+
+public class OpenExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "open";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull List<String> arguments;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/ProxyFunctionExchange.java

@@ -0,0 +1,50 @@
+package io.xpipe.beacon.exchange;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import io.xpipe.core.util.ProxyFunction;
+import lombok.Builder;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class ProxyFunctionExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "proxyFunction";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+
+        @JsonSerialize(
+                using = ProxyFunction.Serializer.class,
+                as = ProxyFunction.class
+        )
+        @JsonDeserialize(
+                using = ProxyFunction.Deserializer.class,
+                as = ProxyFunction.class
+        )
+        ProxyFunction function;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+
+        @JsonSerialize(
+                using = ProxyFunction.Serializer.class,
+                as = ProxyFunction.class
+        )
+        @JsonDeserialize(
+                using = ProxyFunction.Deserializer.class,
+                as = ProxyFunction.class
+        )
+        ProxyFunction function;
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/ProxyReadConnectionExchange.java

@@ -0,0 +1,30 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import io.xpipe.core.source.DataSource;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class ProxyReadConnectionExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "proxyReadConnection";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull DataSource<?> source;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/ProxyWriteConnectionExchange.java

@@ -0,0 +1,32 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import io.xpipe.core.source.DataSource;
+import io.xpipe.core.source.WriteMode;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+public class ProxyWriteConnectionExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "proxyWriteConnection";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull DataSource<?> source;
+        @NonNull WriteMode mode;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/QueryStoreExchange.java

@@ -0,0 +1,50 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import io.xpipe.core.store.DataStore;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+import java.util.LinkedHashMap;
+
+/**
+ * Queries general information about a data source.
+ */
+public class QueryStoreExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "queryStore";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull
+        String name;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+        @NonNull
+        String name;
+
+        String information;
+
+        String summary;
+
+        @NonNull
+        String provider;
+
+        @NonNull
+        LinkedHashMap<String, String> config;
+
+        DataStore internalStore;
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/ReadStreamExchange.java

@@ -2,30 +2,31 @@ package io.xpipe.beacon.exchange;
 
 import io.xpipe.beacon.RequestMessage;
 import io.xpipe.beacon.ResponseMessage;
-import io.xpipe.core.store.FileStore;
 import lombok.Builder;
+import lombok.NonNull;
 import lombok.Value;
 import lombok.extern.jackson.Jacksonized;
 
 /**
  * Stores a stream of data in a storage.
  */
-public class StoreStreamExchange implements MessageExchange {
+public class ReadStreamExchange implements MessageExchange {
 
     @Override
     public String getId() {
-        return "storeStream";
+        return "readStream";
     }
 
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+        @NonNull String name;
+    }
 
     @Jacksonized
     @Builder
     @Value
     public static class Response implements ResponseMessage {
-        FileStore store;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/StopExchange.java

@@ -19,7 +19,8 @@ public class StopExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/WriteStreamExchange.java

@@ -0,0 +1,32 @@
+package io.xpipe.beacon.exchange;
+
+import io.xpipe.beacon.RequestMessage;
+import io.xpipe.beacon.ResponseMessage;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+/**
+ * Stores a stream of data in a storage.
+ */
+public class WriteStreamExchange implements MessageExchange {
+
+    @Override
+    public String getId() {
+        return "writeStream";
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Request implements RequestMessage {
+        @NonNull String name;
+    }
+
+    @Jacksonized
+    @Builder
+    @Value
+    public static class Response implements ResponseMessage {
+    }
+}

beacon/src/main/java/io/xpipe/beacon/exchange/api/QueryRawDataExchange.java

@@ -29,5 +29,6 @@ public class QueryRawDataExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/api/QueryTextDataExchange.java

@@ -30,5 +30,6 @@ public class QueryTextDataExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ConvertExchange.java

@@ -37,7 +37,6 @@ public class ConvertExchange implements MessageExchange {
     @Builder
     @Value
     public static class Response implements ResponseMessage {
-        @NonNull
         DialogReference config;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/InstanceExchange.java

@@ -19,7 +19,8 @@ public class InstanceExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ListCollectionsExchange.java

@@ -20,7 +20,8 @@ public class ListCollectionsExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ListStoresExchange.java

@@ -20,7 +20,8 @@ public class ListStoresExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ModeExchange.java

@@ -3,6 +3,7 @@ package io.xpipe.beacon.exchange.cli;
 import io.xpipe.beacon.RequestMessage;
 import io.xpipe.beacon.ResponseMessage;
 import io.xpipe.beacon.exchange.MessageExchange;
+import io.xpipe.core.util.XPipeDaemonMode;
 import lombok.Builder;
 import lombok.NonNull;
 import lombok.Value;
@@ -20,11 +21,14 @@ public class ModeExchange implements MessageExchange {
     @Value
     public static class Request implements RequestMessage {
         @NonNull
-        String modeId;
+        XPipeDaemonMode mode;
     }
 
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+        @NonNull
+        XPipeDaemonMode usedMode;
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/ReadDrainExchange.java

@@ -30,5 +30,6 @@ public class ReadDrainExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/RemoveCollectionExchange.java

@@ -26,5 +26,6 @@ public class RemoveCollectionExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/RemoveStoreExchange.java

@@ -21,12 +21,11 @@ public class RemoveStoreExchange implements MessageExchange {
     public static class Request implements RequestMessage {
         @NonNull
         String storeName;
-
-        boolean removeUnderlying;
     }
 
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/RenameCollectionExchange.java

@@ -29,5 +29,6 @@ public class RenameCollectionExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/RenameEntryExchange.java

@@ -25,14 +25,12 @@ public class RenameEntryExchange implements MessageExchange {
         DataSourceReference ref;
 
         @NonNull
-        String newName;
+        DataSourceId newId;
     }
 
     @Jacksonized
     @Builder
     @Value
     public static class Response implements ResponseMessage {
-        @NonNull
-        DataSourceId newId;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/RenameStoreExchange.java

@@ -29,5 +29,6 @@ public class RenameStoreExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/SelectExchange.java

@@ -27,5 +27,6 @@ public class SelectExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/SourceProviderListExchange.java

@@ -23,7 +23,8 @@ public class SourceProviderListExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/StatusExchange.java

@@ -17,7 +17,8 @@ public class StatusExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/StoreAddExchange.java

@@ -20,7 +20,6 @@ public class StoreAddExchange implements MessageExchange {
     @Builder
     @Value
     public static class Request implements RequestMessage {
-        String input;
         DataStore storeInput;
 
         String type;

beacon/src/main/java/io/xpipe/beacon/exchange/cli/StoreProviderListExchange.java

@@ -10,6 +10,7 @@ import lombok.Value;
 import lombok.extern.jackson.Jacksonized;
 
 import java.util.List;
+import java.util.Map;
 
 public class StoreProviderListExchange implements MessageExchange {
 
@@ -21,13 +22,14 @@ public class StoreProviderListExchange implements MessageExchange {
     @Jacksonized
     @Builder
     @Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder
     @Value
     public static class Response implements ResponseMessage {
         @NonNull
-        List<ProviderEntry> entries;
+        Map<String, List<ProviderEntry>> entries;
     }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/VersionExchange.java

@@ -17,7 +17,8 @@ public class VersionExchange implements MessageExchange {
     @lombok.extern.jackson.Jacksonized
     @lombok.Builder
     @lombok.Value
-    public static class Request implements RequestMessage {}
+    public static class Request implements RequestMessage {
+    }
 
     @Jacksonized
     @Builder

beacon/src/main/java/io/xpipe/beacon/exchange/cli/WriteExecuteExchange.java

@@ -30,10 +30,14 @@ public class WriteExecuteExchange implements MessageExchange {
 
         @NonNull
         UUID id;
+
+        String mode;
     }
 
     @Jacksonized
     @Builder
     @Value
-    public static class Response implements ResponseMessage {}
+    public static class Response implements ResponseMessage {
+        boolean hasBody;
+    }
 }

beacon/src/main/java/io/xpipe/beacon/exchange/cli/WritePreparationExchange.java

@@ -27,8 +27,9 @@ public class WritePreparationExchange implements MessageExchange {
     public static class Request implements RequestMessage {
         String type;
 
-        @NonNull
-        DataStore output;
+        DataStore outputStore;
+
+        DataSourceReference outputSource;
 
         @NonNull
         DataSourceReference source;
@@ -38,8 +39,6 @@ public class WritePreparationExchange implements MessageExchange {
     @Builder
     @Value
     public static class Response implements ResponseMessage {
-        boolean hasBody;
-
         @NonNull
         DialogReference config;
     }

beacon/src/main/java/io/xpipe/beacon/util/QuietDialogHandler.java

@@ -1,7 +1,7 @@
-package io.xpipe.api.util;
+package io.xpipe.beacon.util;
 
 import io.xpipe.beacon.BeaconConnection;
-import io.xpipe.beacon.ClientException;
+import io.xpipe.beacon.BeaconException;
 import io.xpipe.beacon.exchange.cli.DialogExchange;
 import io.xpipe.core.dialog.BaseQueryElement;
 import io.xpipe.core.dialog.ChoiceElement;
@@ -17,7 +17,6 @@ public class QuietDialogHandler {
     private final BeaconConnection connection;
     private final Map<String, String> overrides;
     private DialogElement element;
-
     public QuietDialogHandler(DialogReference ref, BeaconConnection connection, Map<String, String> overrides) {
         this.dialogKey = ref.getDialogId();
         this.element = ref.getStart();
@@ -25,7 +24,11 @@ public class QuietDialogHandler {
         this.overrides = overrides;
     }
 
-    public void handle() throws ClientException {
+    public static void handle(DialogReference ref, BeaconConnection connection) {
+        new QuietDialogHandler(ref, connection, Map.of()).handle();
+    }
+
+    public void handle() {
         String response = null;
 
         if (element instanceof ChoiceElement c) {
@@ -37,11 +40,11 @@ public class QuietDialogHandler {
         }
 
         DialogExchange.Response res = connection.performSimpleExchange(DialogExchange.Request.builder()
-                .dialogKey(dialogKey)
-                .value(response)
-                .build());
+                                                                               .dialogKey(dialogKey)
+                                                                               .value(response)
+                                                                               .build());
         if (res.getElement() != null && element.equals(res.getElement())) {
-            throw new ClientException(
+            throw new BeaconException(
                     "Invalid value for key " + res.getElement().toDisplayString());
         }
 

beacon/src/main/java/module-info.java

@@ -1,10 +1,15 @@
 import com.fasterxml.jackson.databind.Module;
 import io.xpipe.beacon.BeaconJacksonModule;
+import io.xpipe.beacon.BeaconProxyImpl;
+import io.xpipe.beacon.SecretProviderImpl;
 import io.xpipe.beacon.exchange.*;
 import io.xpipe.beacon.exchange.api.QueryRawDataExchange;
 import io.xpipe.beacon.exchange.api.QueryTableDataExchange;
 import io.xpipe.beacon.exchange.api.QueryTextDataExchange;
 import io.xpipe.beacon.exchange.cli.*;
+import io.xpipe.core.util.ProxyFunction;
+import io.xpipe.core.util.ProxyProvider;
+import io.xpipe.core.util.SecretProvider;
 
 module io.xpipe.beacon {
     exports io.xpipe.beacon;
@@ -18,6 +23,8 @@ module io.xpipe.beacon {
     opens io.xpipe.beacon.exchange.api;
     opens io.xpipe.beacon.exchange.data;
     opens io.xpipe.beacon.exchange.cli;
+    exports io.xpipe.beacon.util;
+    opens io.xpipe.beacon.util;
 
     requires static com.fasterxml.jackson.core;
     requires static com.fasterxml.jackson.databind;
@@ -25,33 +32,42 @@ module io.xpipe.beacon {
     requires static lombok;
 
     uses MessageExchange;
+    uses ProxyFunction;
 
+    provides ProxyProvider with BeaconProxyImpl;
+    provides SecretProvider with SecretProviderImpl;
     provides Module with BeaconJacksonModule;
     provides io.xpipe.beacon.exchange.MessageExchange with
             ForwardExchange,
             InstanceExchange,
             EditStoreExchange,
             AddSourceExchange,
+            WriteStreamExchange,
+            ReadStreamExchange,
             StoreProviderListExchange,
             ListCollectionsExchange,
             ListEntriesExchange,
             ModeExchange,
+            ProxyWriteConnectionExchange,
+            ProxyFunctionExchange,
+            QueryStoreExchange,
             StatusExchange,
+            FocusExchange,
+            OpenExchange,
             StopExchange,
             RenameStoreExchange,
             RemoveStoreExchange,
             StoreAddExchange,
             ReadDrainExchange,
             WritePreparationExchange,
+            ProxyReadConnectionExchange,
             WriteExecuteExchange,
             SelectExchange,
             ReadExchange,
             QueryTextDataExchange,
-            ReadExecuteExchange,
             ListStoresExchange,
             DialogExchange,
             QueryDataSourceExchange,
-            StoreStreamExchange,
             EditExchange,
             RemoveEntryExchange,
             RemoveCollectionExchange,

build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id 'org.jreleaser' version '1.2.0'
+    id 'org.jreleaser' version '1.3.1'
 }
 
 if(project == rootProject) {

core/build.gradle

@@ -5,9 +5,9 @@ plugins {
     id "org.moditect.gradleplugin" version "1.0.0-rc3"
 }
 
-apply from: "$projectDir/../deps/java.gradle"
-apply from: "$projectDir/../deps/lombok.gradle"
-apply from: "$projectDir/../deps/junit.gradle"
+apply from: "$projectDir/../gradle_scripts/java.gradle"
+apply from: "$projectDir/../gradle_scripts/lombok.gradle"
+apply from: "$projectDir/../gradle_scripts/junit.gradle"
 
 compileJava {
     options.compilerArgs << '-parameters'
@@ -17,6 +17,7 @@ dependencies {
     api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.0"
     implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-parameter-names', version: "2.13.0"
     implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: "2.13.0"
+    implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jdk8', version: "2.13.0"
 }
 
 version = file('../misc/version').text
@@ -28,4 +29,4 @@ repositories {
 }
 
 apply from: 'publish.gradle'
-apply from: "$projectDir/../deps/publish-base.gradle"
+apply from: "$projectDir/../gradle_scripts/publish-base.gradle"

core/src/main/java/io/xpipe/core/charsetter/Charsetter.java

@@ -1,6 +1,6 @@
 package io.xpipe.core.charsetter;
 
-import io.xpipe.core.store.FileStore;
+import io.xpipe.core.impl.FileStore;
 import io.xpipe.core.store.MachineStore;
 import io.xpipe.core.store.StreamDataStore;
 import lombok.Value;
@@ -23,7 +23,8 @@ public abstract class Charsetter {
     public static Charsetter INSTANCE;
     private static CharsetterUniverse universe;
 
-    protected Charsetter() {}
+    protected Charsetter() {
+    }
 
     protected static void checkInit() {
         if (universe == null) {
@@ -77,7 +78,8 @@ public abstract class Charsetter {
     }
 
     public abstract Result read(
-            FailableSupplier<InputStream, Exception> in, FailableConsumer<InputStreamReader, Exception> con)
+            FailableSupplier<InputStream, Exception> in, FailableConsumer<InputStreamReader, Exception> con
+    )
             throws Exception;
 
     public Result detect(StreamDataStore store) throws Exception {

core/src/main/java/io/xpipe/core/charsetter/NewLine.java

@@ -25,7 +25,7 @@ public enum NewLine {
                 .orElseThrow();
     }
 
-    public static NewLine id(String id) {
+    public static NewLine byId(String id) {
         return Arrays.stream(values())
                 .filter(n -> n.getId().equalsIgnoreCase(id))
                 .findFirst()

core/src/main/java/io/xpipe/core/charsetter/StreamCharset.java

@@ -1,49 +1,128 @@
 package io.xpipe.core.charsetter;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
+import io.xpipe.core.util.Identifiers;
 import lombok.Value;
 
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.stream.Stream;
 
 @Value
 public class StreamCharset {
 
-    public static final StreamCharset UTF8 = new StreamCharset(StandardCharsets.UTF_8, null);
-    public static final StreamCharset UTF8_BOM =
-            new StreamCharset(StandardCharsets.UTF_8, new byte[] {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF});
-    public static final StreamCharset UTF16_BE = new StreamCharset(StandardCharsets.UTF_16BE, null);
-    public static final StreamCharset UTF16_BE_BOM =
-            new StreamCharset(StandardCharsets.UTF_16BE, new byte[] {(byte) 0xFE, (byte) 0xFF});
-    public static final StreamCharset UTF16_LE = new StreamCharset(StandardCharsets.UTF_16LE, null);
-    public static final StreamCharset UTF16_LE_BOM =
-            new StreamCharset(StandardCharsets.UTF_16LE, new byte[] {(byte) 0xFF, (byte) 0xFE});
+    public static final StreamCharset UTF8 =
+            new StreamCharset(StandardCharsets.UTF_8, null, Identifiers.get("utf", "8"));
 
-    public static final StreamCharset UTF32_LE = new StreamCharset(Charset.forName("utf-32le"), null);
-    public static final StreamCharset UTF32_LE_BOM =
-            new StreamCharset(Charset.forName("utf-32le"), new byte[] {0x00, 0x00, (byte) 0xFE, (byte) 0xFF});
-    public static final StreamCharset UTF32_BE = new StreamCharset(Charset.forName("utf-32be"), null);
-    public static final StreamCharset UTF32_BE_BOM =
-            new StreamCharset(Charset.forName("utf-32be"), new byte[] {(byte) 0xFF, (byte) 0xFE, 0x00, 0x00, });
+    public static final StreamCharset UTF8_BOM = new StreamCharset(
+            StandardCharsets.UTF_8,
+            new byte[]{
+                    (byte) 0xEF,
+                    (byte) 0xBB,
+                    (byte) 0xBF
+            },
+            Identifiers.get("utf", "8", "bom")
+    );
+
+    // ======
+    // UTF-16
+    // ======
+
+    public static final StreamCharset UTF16_BE =
+            new StreamCharset(StandardCharsets.UTF_16BE, null, Identifiers.get("utf", "16", "be"));
+
+    public static final StreamCharset UTF16_BE_BOM = new StreamCharset(
+            StandardCharsets.UTF_16BE,
+            new byte[]{
+                    (byte) 0xFE,
+                    (byte) 0xFF
+            },
+            Identifiers.get("utf", "16", "be", "bom")
+    );
+
+    public static final StreamCharset UTF16_LE =
+            new StreamCharset(StandardCharsets.UTF_16LE, null, Identifiers.get("utf", "16", "le"));
+
+    public static final StreamCharset UTF16_LE_BOM = new StreamCharset(
+            StandardCharsets.UTF_16LE,
+            new byte[]{
+                    (byte) 0xFF,
+                    (byte) 0xFE
+            },
+            Identifiers.get("utf", "16", "le", "bom")
+    );
+
+    public static final StreamCharset UTF16 = UTF16_LE;
+
+    public static final StreamCharset UTF16_BOM = UTF16_LE_BOM;
 
     public static final List<StreamCharset> COMMON = List.of(
             UTF8,
             UTF8_BOM,
-            UTF16_BE,
-            UTF16_BE_BOM,
-            UTF16_LE,
-            UTF16_LE_BOM,
-            new StreamCharset(StandardCharsets.US_ASCII, null),
-            new StreamCharset(StandardCharsets.ISO_8859_1, null),
-            new StreamCharset(Charset.forName("Windows-1251"), null),
-            new StreamCharset(Charset.forName("Windows-1252"), null));
-    private static final List<StreamCharset> RARE_KNOWN = List.of(UTF32_LE, UTF32_LE_BOM, UTF32_BE, UTF32_BE_BOM);
+            UTF16,
+            UTF16_BOM,
+            new StreamCharset(
+                    StandardCharsets.US_ASCII,
+                    null,
+                    Identifiers.join(Identifiers.get("ascii"), Identifiers.get("us", "ascii"))
+            ),
+            new StreamCharset(
+                    StandardCharsets.ISO_8859_1,
+                    null,
+                    Identifiers.join(
+                            Identifiers.get("iso", "8859"),
+                            Identifiers.get("iso", "8859", "1"),
+                            Identifiers.get("8859"),
+                            Identifiers.get("8859", "1")
+                    )
+            ),
+            new StreamCharset(
+                    Charset.forName("Windows-1251"),
+                    null,
+                    Identifiers.join(Identifiers.get("windows", "1251"), Identifiers.get("1251"))
+            ),
+            new StreamCharset(
+                    Charset.forName("Windows-1252"),
+                    null,
+                    Identifiers.join(Identifiers.get("windows", "1252"), Identifiers.get("1252"))
+            )
+    );
+
+    // ======
+    // UTF-32
+    // ======
+    public static final StreamCharset UTF32_LE =
+            new StreamCharset(Charset.forName("utf-32le"), null, Identifiers.get("utf", "32", "le"));
+    public static final StreamCharset UTF32_LE_BOM = new StreamCharset(
+            Charset.forName("utf-32le"),
+            new byte[]{
+                    0x00,
+                    0x00,
+                    (byte) 0xFE,
+                    (byte) 0xFF
+            },
+            Identifiers.get("utf", "32", "le", "bom")
+    );
+    public static final StreamCharset UTF32_BE =
+            new StreamCharset(Charset.forName("utf-32be"), null, Identifiers.get("utf", "32", "be"));
+    public static final StreamCharset UTF32_BE_BOM = new StreamCharset(
+            Charset.forName("utf-32be"),
+            new byte[]{
+                    (byte) 0xFF,
+                    (byte) 0xFE,
+                    0x00,
+                    0x00,
+                    },
+            Identifiers.get("utf", "32", "be", "bom")
+    );
+    private static final List<StreamCharset> RARE_NAMED =
+            List.of(UTF16_LE, UTF16_LE_BOM, UTF16_BE, UTF16_BE_BOM, UTF32_LE, UTF32_LE_BOM, UTF32_BE, UTF32_BE_BOM);
+
     public static final List<StreamCharset> RARE = Stream.concat(
-                    RARE_KNOWN.stream(),
+                    RARE_NAMED.stream(),
                     Charset.availableCharsets().values().stream()
                             .filter(charset -> !charset.equals(StandardCharsets.UTF_16)
                                     && !charset.equals(Charset.forName("utf-32"))
@@ -51,32 +130,61 @@ public class StreamCharset {
                                     && !charset.displayName().startsWith("X-")
                                     && !charset.displayName().endsWith("-BOM")
                                     && COMMON.stream()
-                                            .noneMatch(c -> c.getCharset().equals(charset))
-                                    && RARE_KNOWN.stream()
-                                            .noneMatch(c -> c.getCharset().equals(charset)))
-                            .map(charset -> new StreamCharset(charset, null)))
+                                    .noneMatch(c -> c.getCharset().equals(charset))
+                                    && RARE_NAMED.stream()
+                                    .noneMatch(c -> c.getCharset().equals(charset)))
+                            .map(charset -> new StreamCharset(
+                                    charset,
+                                    null,
+                                    Identifiers.get(charset.name().split("-"))
+                            ))
+            )
             .toList();
+
+    public static final List<StreamCharset> ALL = Stream.concat(COMMON.stream(), RARE.stream()).toList();
+
+
     Charset charset;
     byte[] byteOrderMark;
+    List<String> names;
 
     public static StreamCharset get(Charset charset, boolean byteOrderMark) {
-        return Stream.concat(COMMON.stream(), RARE.stream())
+        return ALL.stream()
                 .filter(streamCharset ->
-                        streamCharset.getCharset().equals(charset) && streamCharset.hasByteOrderMark() == byteOrderMark)
+                                streamCharset.getCharset().equals(charset) && streamCharset.hasByteOrderMark() == byteOrderMark)
                 .findFirst()
                 .orElseThrow();
     }
 
-    @JsonCreator
     public static StreamCharset get(String s) {
-        var byteOrderMark = s.endsWith("-bom");
-        var charset = Charset.forName(s.substring(0, s.length() - (byteOrderMark ? 4 : 0)));
-        return StreamCharset.get(charset, byteOrderMark);
+        var found = ALL.stream().filter(streamCharset -> streamCharset.getNames().contains(s.toLowerCase(Locale.ROOT))).findFirst();
+        if (found.isEmpty()) {
+            throw new IllegalArgumentException("Unknown charset name: " + s);
+        }
+
+        return found.get();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof StreamCharset that)) {
+            return false;
+        }
+        return charset.equals(that.charset) && Arrays.equals(byteOrderMark, that.byteOrderMark);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(charset);
+        result = 31 * result + Arrays.hashCode(byteOrderMark);
+        return result;
     }
 
-    @JsonValue
     public String toString() {
-        return getCharset().name().toLowerCase(Locale.ROOT) + (hasByteOrderMark() ? "-bom" : "");
+        return getNames().get(0);
     }
 
     public boolean hasByteOrderMark() {

core/src/main/java/io/xpipe/core/data/generic/GenericArrayReader.java

@@ -17,7 +17,8 @@ public class GenericArrayReader implements GenericAbstractReader {
     private GenericAbstractReader currentReader;
     private DataStructureNode created;
 
-    public GenericArrayReader() {}
+    public GenericArrayReader() {
+    }
 
     public static GenericArrayReader newReader(int length) {
         var ar = new GenericArrayReader();

core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamCallback.java

@@ -4,15 +4,21 @@ import java.util.Map;
 
 public interface GenericDataStreamCallback {
 
-    default void onName(String name) {}
+    default void onName(String name) {
+    }
 
-    default void onArrayStart(int length) {}
+    default void onArrayStart(int length) {
+    }
 
-    default void onArrayEnd(Map<Integer, String> metaAttributes) {}
+    default void onArrayEnd(Map<Integer, String> metaAttributes) {
+    }
 
-    default void onTupleStart(int length) {}
+    default void onTupleStart(int length) {
+    }
 
-    default void onTupleEnd(Map<Integer, String> metaAttributes) {}
+    default void onTupleEnd(Map<Integer, String> metaAttributes) {
+    }
 
-    default void onValue(byte[] value, Map<Integer, String> metaAttributes) {}
+    default void onValue(byte[] value, Map<Integer, String> metaAttributes) {
+    }
 }

core/src/main/java/io/xpipe/core/data/generic/GenericTupleReader.java

@@ -19,7 +19,8 @@ public class GenericTupleReader implements GenericAbstractReader {
     private GenericAbstractReader currentReader;
     private DataStructureNode created;
 
-    public GenericTupleReader() {}
+    public GenericTupleReader() {
+    }
 
     public static GenericTupleReader newReader(int length) {
         var tr = new GenericTupleReader();

core/src/main/java/io/xpipe/core/data/node/ArrayNode.java

@@ -8,7 +8,8 @@ import java.util.stream.Collectors;
 
 public abstract class ArrayNode extends DataStructureNode {
 
-    protected ArrayNode() {}
+    protected ArrayNode() {
+    }
 
     public static ArrayNode empty() {
         return of(List.of());
@@ -56,6 +57,7 @@ public abstract class ArrayNode extends DataStructureNode {
         return "array node";
     }
 
+
     @Override
     public final String toString(int indent) {
         var content = getNodes().stream().map(n -> n.toString(indent)).collect(Collectors.joining(", "));

core/src/main/java/io/xpipe/core/data/node/DataStructureNode.java

@@ -129,13 +129,13 @@ public abstract class DataStructureNode implements Iterable<DataStructureNode> {
     public String metaToString() {
         return "("
                 + (metaAttributes != null
-                        ? metaAttributes.entrySet().stream()
-                                .sorted(Comparator.comparingInt(entry -> entry.getKey()))
-                                .map(e -> e.getValue() != null
-                                        ? e.getKey() + ":" + e.getValue()
-                                        : e.getKey().toString())
-                                .collect(Collectors.joining("|"))
-                        : "")
+                ? metaAttributes.entrySet().stream()
+                .sorted(Comparator.comparingInt(entry -> entry.getKey()))
+                .map(e -> e.getValue() != null
+                        ? e.getKey() + ":" + e.getValue()
+                        : e.getKey().toString())
+                .collect(Collectors.joining("|"))
+                : "")
                 + ")";
     }
 
@@ -242,5 +242,6 @@ public abstract class DataStructureNode implements Iterable<DataStructureNode> {
         throw unsupported("iterator creation");
     }
 
-    public record KeyValue(String key, DataStructureNode value) {}
+    public record KeyValue(String key, DataStructureNode value) {
+    }
 }

core/src/main/java/io/xpipe/core/data/node/DataStructureNodePointer.java

@@ -217,7 +217,8 @@ public class DataStructureNodePointer {
         }
 
         public Builder pointerEvaluation(
-                DataStructureNodePointer pointer, Function<DataStructureNode, String> converter) {
+                DataStructureNodePointer pointer, Function<DataStructureNode, String> converter
+        ) {
             path.add(new FunctionElement((current) -> {
                 var res = pointer.get(current);
                 if (res != null) {

core/src/main/java/io/xpipe/core/data/node/LinkedTupleNode.java

@@ -86,7 +86,8 @@ public class LinkedTupleNode extends TupleNode {
     public DataType determineDataType() {
         return TupleType.of(
                 getKeyNames(),
-                getNodes().stream().map(DataStructureNode::determineDataType).toList());
+                getNodes().stream().map(DataStructureNode::determineDataType).toList()
+        );
     }
 
     @Override

core/src/main/java/io/xpipe/core/data/node/SimpleTupleNode.java

@@ -67,7 +67,9 @@ public class SimpleTupleNode extends TupleNode {
     @Override
     public DataStructureNode clear() {
         nodes.clear();
-        if (names != null) names.clear();
+        if (names != null) {
+            names.clear();
+        }
         return this;
     }
 

core/src/main/java/io/xpipe/core/data/node/TupleNode.java

@@ -1,9 +1,8 @@
 package io.xpipe.core.data.node;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public abstract class TupleNode extends DataStructureNode {
 
@@ -41,6 +40,21 @@ public abstract class TupleNode extends DataStructureNode {
         return true;
     }
 
+    @Override
+    public Stream<DataStructureNode> stream() {
+        return getNodes().stream();
+    }
+
+    @Override
+    public Spliterator<DataStructureNode> spliterator() {
+        return stream().spliterator();
+    }
+
+    @Override
+    public Iterator<DataStructureNode> iterator() {
+        return stream().iterator();
+    }
+
     @Override
     public String toString(int indent) {
         var is = " ".repeat(indent);
@@ -102,8 +116,9 @@ public abstract class TupleNode extends DataStructureNode {
             boolean hasKeys = entries.stream().anyMatch(kv -> kv.key() != null);
             return hasKeys
                     ? TupleNode.of(
-                            entries.stream().map(KeyValue::key).toList(),
-                            entries.stream().map(KeyValue::value).toList())
+                    entries.stream().map(KeyValue::key).toList(),
+                    entries.stream().map(KeyValue::value).toList()
+            )
                     : TupleNode.of(entries.stream().map(KeyValue::value).toList());
         }
     }

core/src/main/java/io/xpipe/core/data/node/ValueNode.java

@@ -13,7 +13,8 @@ import java.util.Objects;
 
 public abstract class ValueNode extends DataStructureNode {
 
-    protected ValueNode() {}
+    protected ValueNode() {
+    }
 
     public static ValueNode nullValue() {
         return new SimpleValueNode(new byte[0]).tag(IS_NULL).asValue();
@@ -65,43 +66,57 @@ public abstract class ValueNode extends DataStructureNode {
 
     public static ValueNode ofBytes(byte[] data) {
         var created = of(data);
-        created.tag(IS_BINARY);
+        if (data != null) {
+        created.tag(IS_BINARY);}
         return created;
     }
 
     public static ValueNode ofText(String text) {
         var created = of(text);
-        created.tag(IS_TEXT);
+        if (text != null) {
+            created.tag(IS_TEXT);
+        }
         return created;
     }
 
     public static ValueNode ofInteger(int integer) {
         var created = of(integer);
         created.tag(IS_INTEGER);
+        created.tag(INTEGER_VALUE, integer);
         return created;
     }
 
     public static ValueNode ofInteger(BigInteger integer) {
         var created = of(integer);
-        created.tag(IS_INTEGER);
+        if (integer != null) {
+            created.tag(IS_INTEGER);
+            created.tag(INTEGER_VALUE, integer);
+        }
         return created;
     }
 
     public static ValueNode ofDecimal(double decimal) {
         var created = of(decimal);
         created.tag(IS_DECIMAL);
+        created.tag(DECIMAL_VALUE, decimal);
         return created;
     }
 
     public static ValueNode ofDecimal(BigDecimal decimal) {
         var created = of(decimal);
-        created.tag(IS_DECIMAL);
+        if (decimal != null) {
+            created.tag(IS_DECIMAL);
+            created.tag(DECIMAL_VALUE, decimal);
+        }
         return created;
     }
 
     public static ValueNode ofBoolean(Boolean bool) {
         var created = of(bool);
-        created.tag(IS_BOOLEAN);
+        if (bool != null) {
+            created.tag(IS_BOOLEAN);
+            created.tag(bool ? BOOLEAN_TRUE : BOOLEAN_FALSE);
+        }
         return created;
     }
 

core/src/main/java/io/xpipe/core/data/type/DataType.java

@@ -10,7 +10,10 @@ import java.util.Optional;
  * To check whether a {@link DataStructureNode} instance conforms to the specified type,
  * the method {@link #matches(DataStructureNode)} can be used.
  */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        property = "type"
+)
 public abstract class DataType {
 
     /**

core/src/main/java/io/xpipe/core/data/type/DataTypeVisitor.java

@@ -2,11 +2,15 @@ package io.xpipe.core.data.type;
 
 public interface DataTypeVisitor {
 
-    default void onValue(ValueType type) {}
+    default void onValue(ValueType type) {
+    }
 
-    default void onTuple(TupleType type) {}
+    default void onTuple(TupleType type) {
+    }
 
-    default void onArray(ArrayType type) {}
+    default void onArray(ArrayType type) {
+    }
 
-    default void onWildcard(WildcardType type) {}
+    default void onWildcard(WildcardType type) {
+    }
 }

core/src/main/java/io/xpipe/core/data/type/DataTypeVisitors.java

@@ -41,7 +41,8 @@ public class DataTypeVisitors {
      * Creates a visitor that allows for visiting possible recursive columns of table.
      */
     public static DataTypeVisitor table(
-            Consumer<String> newTuple, Runnable endTuple, BiConsumer<String, DataStructureNodePointer> newValue) {
+            Consumer<String> newTuple, Runnable endTuple, BiConsumer<String, DataStructureNodePointer> newValue
+    ) {
         return new DataTypeVisitor() {
             private final Stack<TupleType> tuples = new Stack<>();
             private final Stack<Integer> keyIndices = new Stack<>();

core/src/main/java/io/xpipe/core/data/type/WildcardType.java

@@ -17,7 +17,8 @@ import java.util.Optional;
 @Value
 public class WildcardType extends DataType {
 
-    private WildcardType() {}
+    private WildcardType() {
+    }
 
     /**
      * Creates a new instance.

core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamCallback.java

@@ -7,19 +7,27 @@ import java.util.Map;
 
 public interface TypedDataStreamCallback {
 
-    default void onValue(byte[] data, Map<Integer, String> metaAttributes) {}
+    default void onValue(byte[] data, Map<Integer, String> metaAttributes) {
+    }
 
-    default void onGenericNode(DataStructureNode node) {}
+    default void onGenericNode(DataStructureNode node) {
+    }
 
-    default void onTupleBegin(TupleType type) {}
+    default void onTupleBegin(TupleType type) {
+    }
 
-    default void onTupleEnd(Map<Integer, String> metaAttributes) {}
+    default void onTupleEnd(Map<Integer, String> metaAttributes) {
+    }
 
-    default void onArrayBegin(int size) {}
+    default void onArrayBegin(int size) {
+    }
 
-    default void onArrayEnd(Map<Integer, String> metaAttributes) {}
+    default void onArrayEnd(Map<Integer, String> metaAttributes) {
+    }
 
-    default void onNodeBegin() {}
+    default void onNodeBegin() {
+    }
 
-    default void onNodeEnd() {}
+    default void onNodeEnd() {
+    }
 }

core/src/main/java/io/xpipe/core/data/typed/TypedDataStructureNodeReader.java

@@ -19,6 +19,7 @@ public class TypedDataStructureNodeReader implements TypedAbstractReader {
     private int arrayDepth;
     private DataType expectedType;
     private int currentExpectedTypeIndex;
+
     private TypedDataStructureNodeReader(DataType type) {
         flattened = new ArrayList<>();
         type.visit(DataTypeVisitors.flatten(flattened::add));

core/src/main/java/io/xpipe/core/dialog/BaseQueryElement.java

@@ -21,7 +21,8 @@ public class BaseQueryElement extends DialogElement {
 
     @JsonCreator
     public BaseQueryElement(
-            String description, boolean newLine, boolean required, boolean secret, boolean quiet, String value) {
+            String description, boolean newLine, boolean required, boolean secret, boolean quiet, String value
+    ) {
         this.description = description;
         this.newLine = newLine;
         this.required = required;

core/src/main/java/io/xpipe/core/dialog/Dialog.java

@@ -30,13 +30,6 @@ public abstract class Dialog {
     protected Object eval;
     private Supplier<?> evaluation;
 
-    /**
-     * Removes all completion listeners. Intended for internal use only.
-     */
-    public void clearCompletion() {
-        completion.clear();
-    }
-
     /**
      * Creates an empty dialogue. This dialogue completes immediately and does not handle any questions or answers.
      */
@@ -65,7 +58,8 @@ public abstract class Dialog {
      * @param selected    the selected element index
      */
     public static Dialog.Choice choice(
-            String description, List<io.xpipe.core.dialog.Choice> elements, boolean required, boolean quiet, int selected) {
+            String description, List<io.xpipe.core.dialog.Choice> elements, boolean required, boolean quiet, int selected
+    ) {
         Dialog.Choice c = new Dialog.Choice(description, elements, required, quiet, selected);
         c.evaluateTo(c::getSelected);
         return c;
@@ -83,7 +77,8 @@ public abstract class Dialog {
      */
     @SafeVarargs
     public static <T> Dialog.Choice choice(
-            String description, Function<T, String> toString, boolean required, boolean quiet, T def, T... vals) {
+            String description, Function<T, String> toString, boolean required, boolean quiet, T def, T... vals
+    ) {
         var elements = Arrays.stream(vals)
                 .map(v -> new io.xpipe.core.dialog.Choice(null, toString.apply(v)))
                 .toList();
@@ -120,7 +115,8 @@ public abstract class Dialog {
             boolean required,
             boolean quiet,
             T value,
-            QueryConverter<T> converter) {
+            QueryConverter<T> converter
+    ) {
         var q = new <T>Dialog.Query(description, newLine, required, quiet, value, converter, false);
         q.evaluateTo(q::getConvertedValue);
         return q;
@@ -166,7 +162,8 @@ public abstract class Dialog {
                 DialogElement currentElement = ds[current].receive(answer);
                 if (currentElement == null) {
                     DialogElement next = null;
-                    while (current < ds.length - 1 && (next = ds[++current].start()) == null) {}
+                    while (current < ds.length - 1 && (next = ds[++current].start()) == null) {
+                    }
                     ;
                     return next;
                 }
@@ -357,7 +354,8 @@ public abstract class Dialog {
             List<io.xpipe.core.dialog.Choice> elements,
             boolean required,
             int selected,
-            Function<Integer, Dialog> c) {
+            Function<Integer, Dialog> c
+    ) {
         var choice = new ChoiceElement(description, elements, required, false, selected);
         return new Dialog() {
 
@@ -391,6 +389,13 @@ public abstract class Dialog {
         };
     }
 
+    /**
+     * Removes all completion listeners. Intended for internal use only.
+     */
+    public void clearCompletion() {
+        completion.clear();
+    }
+
     /* TODO: Implement automatic completion mechanism for start as well
      *   In case start returns null, the completion is not automatically done.
      * */
@@ -497,7 +502,8 @@ public abstract class Dialog {
                 boolean quiet,
                 T value,
                 QueryConverter<T> converter,
-                boolean hidden) {
+                boolean hidden
+        ) {
             this.element = new QueryElement(description, newLine, required, quiet, value, converter, hidden);
         }
 

core/src/main/java/io/xpipe/core/dialog/DialogCancelException.java

@@ -5,7 +5,8 @@ package io.xpipe.core.dialog;
  */
 public class DialogCancelException extends Exception {
 
-    public DialogCancelException() {}
+    public DialogCancelException() {
+    }
 
     public DialogCancelException(String message) {
         super(message);
@@ -20,7 +21,8 @@ public class DialogCancelException extends Exception {
     }
 
     public DialogCancelException(
-            String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+            String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace
+    ) {
         super(message, cause, enableSuppression, writableStackTrace);
     }
 }

core/src/main/java/io/xpipe/core/dialog/DialogElement.java

@@ -8,7 +8,10 @@ import java.util.UUID;
 
 @EqualsAndHashCode
 @ToString
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(
+        use = JsonTypeInfo.Id.NAME,
+        property = "type"
+)
 public abstract class DialogElement {
 
     protected String id;

core/src/main/java/io/xpipe/core/dialog/DialogMapper.java

@@ -14,11 +14,15 @@ public class DialogMapper {
 
     public LinkedHashMap<String, String> handle() throws Exception {
         var element = dialog.start();
+        if (element == null) {
+            return map;
+        }
+
         handle(element);
         return map;
     }
 
-    private  void handle(DialogElement element) throws Exception {
+    private void handle(DialogElement element) throws Exception {
         String response = null;
         if (element instanceof ChoiceElement c) {
             response = handleChoice(c);
@@ -41,8 +45,8 @@ public class DialogMapper {
     }
 
     private String handleQuery(BaseQueryElement q) {
-         map.put(q.getDescription(), q.getValue());
-         return q.getValue();
+        map.put(q.getDescription(), q.getValue());
+        return q.getValue();
     }
 
     private String handleChoice(ChoiceElement c) {

core/src/main/java/io/xpipe/core/dialog/QueryConverter.java

@@ -14,7 +14,7 @@ public abstract class QueryConverter<T> {
     public static final QueryConverter<NewLine> NEW_LINE = new QueryConverter<NewLine>() {
         @Override
         protected NewLine fromString(String s) {
-            return NewLine.id(s);
+            return NewLine.byId(s);
         }
 
         @Override

core/src/main/java/io/xpipe/core/dialog/QueryElement.java

@@ -14,7 +14,8 @@ public class QueryElement extends BaseQueryElement {
             boolean quiet,
             T value,
             QueryConverter<T> converter,
-            boolean hidden) {
+            boolean hidden
+    ) {
         super(description, newLine, required, hidden, quiet, value != null ? converter.toString(value) : null);
         this.converter = converter;
     }

core/src/main/java/io/xpipe/core/impl/BinarySource.java

@@ -44,13 +44,13 @@ public class BinarySource extends RawDataSource<StreamDataStore> {
     protected RawReadConnection newReadConnection() {
         return new RawReadConnection() {
 
+            private InputStream inputStream;
+
             @Override
             public boolean canRead() throws Exception {
                 return getStore().canOpen();
             }
 
-            private InputStream inputStream;
-
             @Override
             public void init() throws Exception {
                 if (inputStream != null) {

core/src/main/java/io/xpipe/core/impl/CollectionEntryDataStore.java

@@ -1,4 +1,8 @@
-package io.xpipe.core.store;
+package io.xpipe.core.impl;
+
+import io.xpipe.core.store.DataStore;
+import io.xpipe.core.store.FilenameStore;
+import io.xpipe.core.store.StreamDataStore;
 
 public abstract class CollectionEntryDataStore implements FilenameStore, StreamDataStore {
 

core/src/main/java/io/xpipe/core/impl/FileNames.java

@@ -0,0 +1,59 @@
+package io.xpipe.core.impl;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class FileNames {
+
+    public static String getFileName(String file) {
+        var split = file.split("[\\\\/]");
+        if (split.length == 0) {
+            return "";
+        }
+        return split[split.length - 1];
+    }
+
+    public static String join(String... parts) {
+        var joined = String.join("/", parts);
+        return normalize(joined);
+    }
+
+    public static boolean isAbsolute(String file) {
+        if (!file.contains("/") && !file.contains("\\")) {
+            return false;
+        }
+
+        if (!file.startsWith("/") && !file.startsWith("~") && !file.matches("^\\w:.*")) {
+            return false;
+        }
+
+        return true;
+    }
+
+    public static String getParent(String file) {
+        return file.substring(0, file.length() - getFileName(file).length() - 1);
+    }
+
+    public static boolean startsWith(String file, String start) {
+        return normalize(file).startsWith(normalize(start));
+    }
+
+    public static String normalize(String file) {
+        var backslash = file.contains("\\");
+        return backslash ? toWindows(file) : toUnix(file);
+    }
+
+    private static List<String> split(String file) {
+        var split = file.split("[\\\\/]");
+        return Arrays.stream(split).filter(s -> !s.isEmpty()).toList();
+    }
+
+    public static String toUnix(String file) {
+        var joined = String.join("/", split(file));
+        return file.startsWith("/") ? "/" + joined : joined;
+    }
+
+    public static String toWindows(String file) {
+        return String.join("\\", split(file));
+    }
+}

core/src/main/java/io/xpipe/core/impl/FileStore.java

@@ -1,7 +1,11 @@
-package io.xpipe.core.store;
+package io.xpipe.core.impl;
 
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.xpipe.core.store.FileSystemStore;
+import io.xpipe.core.store.FilenameStore;
+import io.xpipe.core.store.StreamDataStore;
 import io.xpipe.core.util.JacksonizedValue;
+import io.xpipe.core.util.ValidationException;
 import lombok.Getter;
 import lombok.experimental.SuperBuilder;
 import lombok.extern.jackson.Jacksonized;
@@ -29,7 +33,18 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream
         this.file = file;
     }
 
-    public  String getParent() {
+    public static FileStore local(Path p) {
+        return new FileStore(new LocalStore(), p.toString());
+    }
+
+    /**
+     * Creates a file store for a file that is local to the callers machine.
+     */
+    public static FileStore local(String p) {
+        return new FileStore(new LocalStore(), p);
+    }
+
+    public String getParent() {
         var matcher = Pattern.compile("^(.+?)[^\\\\/]+$").matcher(file);
         if (!matcher.matches()) {
             throw new IllegalArgumentException("Unable to determine parent of " + file);
@@ -42,24 +57,16 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream
         return fileSystem instanceof LocalStore;
     }
 
-    public static FileStore local(Path p) {
-        return new FileStore(new LocalStore(), p.toString());
-    }
-
-    /**
-     * Creates a file store for a file that is local to the callers machine.
-     */
-    public static FileStore local(String p) {
-        return new FileStore(new LocalStore(), p);
-    }
-
     @Override
     public void checkComplete() throws Exception {
         if (fileSystem == null) {
-            throw new IllegalStateException("Machine is missing");
+            throw new ValidationException("File system is missing");
         }
         if (file == null) {
-            throw new IllegalStateException("File is missing");
+            throw new ValidationException("File is missing");
+        }
+        if (!FileNames.isAbsolute(file)) {
+            throw new ValidationException("File path is not absolute");
         }
     }
 

core/src/main/java/io/xpipe/core/impl/InMemoryStore.java

@@ -1,9 +1,11 @@
-package io.xpipe.core.store;
+package io.xpipe.core.impl;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.xpipe.core.store.StreamDataStore;
 import io.xpipe.core.util.JacksonizedValue;
+import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.experimental.SuperBuilder;
 import lombok.extern.jackson.Jacksonized;
 
@@ -16,13 +18,15 @@ import java.io.*;
 @SuperBuilder
 @Jacksonized
 @Getter
+@NoArgsConstructor
+@AllArgsConstructor
 public class InMemoryStore extends JacksonizedValue implements StreamDataStore {
 
     private byte[] value;
 
-    @JsonCreator
-    public InMemoryStore(byte[] value) {
-        this.value = value;
+    @Override
+    public String toString() {
+        return value != null && value.length > 100 ? "<memory>" : (value != null ? new String(value) : "null");
     }
 
     @Override
@@ -32,7 +36,7 @@ public class InMemoryStore extends JacksonizedValue implements StreamDataStore {
 
     @Override
     public InputStream openInput() throws Exception {
-        return new ByteArrayInputStream(value);
+        return value != null ? new ByteArrayInputStream(value) : InputStream.nullInputStream();
     }
 
     @Override