From f12ce8293346f6d7b5e93f57a27937c365c0da2e Mon Sep 17 00:00:00 2001 From: Christopher Schnick Date: Thu, 29 Sep 2022 14:51:02 +0200 Subject: [PATCH] Various reworks --- api/src/main/java/io/xpipe/api/DataTable.java | 11 ++ .../xpipe/api/connector/XPipeConnection.java | 9 +- .../java/io/xpipe/api/impl/DataTableImpl.java | 101 +++++++++--------- .../java/io/xpipe/beacon/BeaconClient.java | 74 ++++++++----- .../io/xpipe/beacon/BeaconConnection.java | 89 +++++++++------ .../java/io/xpipe/beacon/BeaconFormat.java | 19 ++-- .../java/io/xpipe/beacon/XPipeInstance.java | 17 +++ .../beacon/exchange/cli/InstanceExchange.java | 33 ++++++ beacon/src/main/java/module-info.java | 1 + core/build.gradle | 3 + .../core/data/generic/GenericArrayReader.java | 8 +- .../generic/GenericDataStreamCallback.java | 4 +- .../data/generic/GenericDataStreamParser.java | 4 +- .../data/generic/GenericDataStreamWriter.java | 4 +- .../GenericDataStructureNodeReader.java | 8 +- .../core/data/generic/GenericTupleReader.java | 8 +- .../core/data/node/DataStructureNode.java | 22 ++-- .../core/data/node/DataStructureNodeIO.java | 23 ++-- .../data/node/SimpleImmutableValueNode.java | 6 +- .../io/xpipe/core/data/node/TupleNode.java | 1 + .../io/xpipe/core/data/node/ValueNode.java | 31 +++++- .../data/typed/TypedDataStreamCallback.java | 8 +- .../data/typed/TypedDataStreamParser.java | 11 +- .../data/typed/TypedDataStreamWriter.java | 7 +- .../typed/TypedDataStructureNodeReader.java | 13 +-- .../java/io/xpipe/core/dialog/Dialog.java | 4 +- .../io/xpipe/core/dialog/QueryConverter.java | 10 +- .../core/impl/BatchTableWriteConnection.java | 48 +++++++++ .../java/io/xpipe/core/impl/BinarySource.java | 67 ++++++++++++ .../core/impl/LimitTableReadConnection.java | 59 ++++++++++ .../java/io/xpipe/core/impl/TextSource.java | 35 +++--- .../java/io/xpipe/core/impl/XpbsSource.java | 23 ++++ .../xpipe/core/impl/XpbtReadConnection.java | 15 ++- .../java/io/xpipe/core/impl/XpbtSource.java | 10 +- .../xpipe/core/impl/XpbtWriteConnection.java | 2 + .../core/source/CollectionDataSource.java | 10 +- .../java/io/xpipe/core/source/DataSource.java | 55 ++++++++-- .../io/xpipe/core/source/RawDataSource.java | 6 +- .../core/source/StructureDataSource.java | 6 +- .../io/xpipe/core/source/TableDataSource.java | 12 +-- .../core/source/TableReadConnection.java | 19 +++- .../core/source/TableWriteConnection.java | 13 ++- .../io/xpipe/core/source/TextDataSource.java | 8 +- .../java/io/xpipe/core/store/FileStore.java | 11 +- .../io/xpipe/core/store/InMemoryStore.java | 4 + .../java/io/xpipe/core/store/LocalStore.java | 17 +-- .../io/xpipe/core/store/ProcessControl.java | 1 + .../java/io/xpipe/core/store/ShellStore.java | 12 +-- .../java/io/xpipe/core/store/ShellTypes.java | 8 +- .../xpipe/core/store/StandardShellStore.java | 4 +- .../io/xpipe/core/util/CoreJacksonModule.java | 19 ++-- .../io/xpipe/core/util/JacksonHelper.java | 15 +-- .../util/{Secret.java => SecretValue.java} | 8 +- .../java/io/xpipe/core/util/StreamHelper.java | 23 ---- extension/build.gradle | 2 +- .../xpipe/extension/DataSourceProvider.java | 6 -- .../xpipe/extension/DataStoreProviders.java | 16 +-- .../io/xpipe/extension/comp/ChoiceComp.java | 8 +- .../xpipe/extension/comp/ChoicePaneComp.java | 10 +- .../io/xpipe/extension/comp/IntFieldComp.java | 8 +- .../xpipe/extension/comp/SecretFieldComp.java | 8 +- .../extension/prefs/PrefsChoiceValue.java | 1 + .../io/xpipe/extension/util/DialogHelper.java | 4 +- .../extension/util/DynamicOptionsBuilder.java | 10 +- .../util/ExtensionJacksonModule.java | 54 ---------- .../xpipe/extension/util/ExtensionTest.java | 7 ++ .../xpipe/extension/util/TypeConverter.java | 99 ++++++++++++++++- extension/src/main/java/module-info.java | 7 -- 68 files changed, 854 insertions(+), 425 deletions(-) create mode 100644 beacon/src/main/java/io/xpipe/beacon/XPipeInstance.java create mode 100644 beacon/src/main/java/io/xpipe/beacon/exchange/cli/InstanceExchange.java create mode 100644 core/src/main/java/io/xpipe/core/impl/BatchTableWriteConnection.java create mode 100644 core/src/main/java/io/xpipe/core/impl/BinarySource.java create mode 100644 core/src/main/java/io/xpipe/core/impl/LimitTableReadConnection.java create mode 100644 core/src/main/java/io/xpipe/core/impl/XpbsSource.java rename core/src/main/java/io/xpipe/core/util/{Secret.java => SecretValue.java} (74%) delete mode 100644 core/src/main/java/io/xpipe/core/util/StreamHelper.java delete mode 100644 extension/src/main/java/io/xpipe/extension/util/ExtensionJacksonModule.java diff --git a/api/src/main/java/io/xpipe/api/DataTable.java b/api/src/main/java/io/xpipe/api/DataTable.java index 7a3e6df21..43a99b106 100644 --- a/api/src/main/java/io/xpipe/api/DataTable.java +++ b/api/src/main/java/io/xpipe/api/DataTable.java @@ -4,6 +4,7 @@ import io.xpipe.core.data.node.ArrayNode; import io.xpipe.core.data.node.TupleNode; import io.xpipe.core.source.DataSourceInfo; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; public interface DataTable extends Iterable, DataSource { @@ -15,4 +16,14 @@ public interface DataTable extends Iterable, DataSource { ArrayNode readAll(); ArrayNode read(int maxRows); + + default int countAndDiscard() { + AtomicInteger count = new AtomicInteger(); + try (var stream = stream()) { + stream.forEach(dataStructureNodes -> { + count.getAndIncrement(); + }); + } + return count.get(); + } } diff --git a/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java b/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java index caf554533..5009d8211 100644 --- a/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java +++ b/api/src/main/java/io/xpipe/api/connector/XPipeConnection.java @@ -3,7 +3,6 @@ package io.xpipe.api.connector; import io.xpipe.beacon.*; import io.xpipe.beacon.exchange.cli.DialogExchange; import io.xpipe.core.dialog.DialogReference; -import io.xpipe.core.util.JacksonHelper; import java.util.Optional; @@ -64,10 +63,6 @@ public final class XPipeConnection extends BeaconConnection { @Override protected void constructSocket() { - if (!JacksonHelper.isInit()) { - JacksonHelper.initModularized(ModuleLayer.boot()); - } - if (!BeaconServer.isRunning()) { try { start(); @@ -79,13 +74,13 @@ public final class XPipeConnection extends BeaconConnection { if (r.isEmpty()) { throw new BeaconException("Wait for xpipe daemon timed out"); } else { - socket = r.get(); + beaconClient = r.get(); return; } } try { - socket = new BeaconClient(); + beaconClient = new BeaconClient(); } catch (Exception ex) { throw new BeaconException("Unable to connect to running xpipe daemon", ex); } diff --git a/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java b/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java index 321215b7e..30be11139 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java @@ -4,6 +4,7 @@ import io.xpipe.api.DataSourceConfig; import io.xpipe.api.DataTable; import io.xpipe.api.connector.XPipeConnection; import io.xpipe.beacon.BeaconConnection; +import io.xpipe.beacon.BeaconException; import io.xpipe.beacon.exchange.api.QueryTableDataExchange; import io.xpipe.core.data.node.ArrayNode; import io.xpipe.core.data.node.DataStructureNode; @@ -16,10 +17,9 @@ import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceReference; import io.xpipe.core.source.DataSourceType; +import java.io.IOException; import java.io.InputStream; import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -43,8 +43,9 @@ public class DataTableImpl extends DataSourceImpl implements DataTable { } public Stream stream() { + var iterator = new TableIterator(); return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(iterator(), Spliterator.ORDERED), false); + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false).onClose(iterator::finish); } @Override @@ -71,54 +72,54 @@ public class DataTableImpl extends DataSourceImpl implements DataTable { return ArrayNode.of(nodes); } + private class TableIterator implements Iterator { + + private final BeaconConnection connection; + private final TypedDataStreamParser parser; + private final TypedAbstractReader nodeReader; + private TupleNode node; + + { + nodeReader = TypedDataStructureNodeReader.of(info.getDataType()); + parser = new TypedDataStreamParser(info.getDataType()); + + connection = XPipeConnection.open(); + var req = QueryTableDataExchange.Request.builder() + .ref(DataSourceReference.id(getId())).maxRows(Integer.MAX_VALUE).build(); + connection.sendRequest(req); + connection.receiveResponse(); + connection.receiveBody(); + } + + private void finish() { + connection.close(); + } + + @Override + public boolean hasNext() { + connection.checkClosed(); + + try { + node = (TupleNode) parser.parseStructure(connection.getInputStream(), nodeReader); + } catch (IOException e) { + throw new BeaconException(e); + } + if (node == null) { + // finish(); + } + return node != null; + } + + @Override + public TupleNode next() { + connection.checkClosed(); + + return node; + } + }; + @Override public Iterator iterator() { - return new Iterator<>() { - - private final BeaconConnection connection; - private final TypedDataStreamParser parser; - private final TypedAbstractReader nodeReader; - - { - nodeReader = TypedDataStructureNodeReader.of(info.getDataType()); - parser = new TypedDataStreamParser(info.getDataType()); - - connection = XPipeConnection.open(); - var req = QueryTableDataExchange.Request.builder() - .ref(DataSourceReference.id(getId())).build(); - connection.sendRequest(req); - connection.receiveResponse(); - connection.receiveBody(); - } - - private void finish() { - connection.close(); - } - - @Override - public boolean hasNext() { - connection.checkClosed(); - - AtomicBoolean hasNext = new AtomicBoolean(false); - connection.withInputStream(in -> { - hasNext.set(parser.hasNext(in)); - }); - if (!hasNext.get()) { - finish(); - } - return hasNext.get(); - } - - @Override - public TupleNode next() { - connection.checkClosed(); - - AtomicReference current = new AtomicReference<>(); - connection.withInputStream(in -> { - current.set((TupleNode) parser.parseStructure(connection.getInputStream(), nodeReader)); - }); - return current.get(); - } - }; + return new TableIterator(); } } diff --git a/beacon/src/main/java/io/xpipe/beacon/BeaconClient.java b/beacon/src/main/java/io/xpipe/beacon/BeaconClient.java index a72965da7..49611953f 100644 --- a/beacon/src/main/java/io/xpipe/beacon/BeaconClient.java +++ b/beacon/src/main/java/io/xpipe/beacon/BeaconClient.java @@ -1,7 +1,7 @@ package io.xpipe.beacon; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -11,12 +11,11 @@ import io.xpipe.beacon.exchange.data.ClientErrorMessage; import io.xpipe.beacon.exchange.data.ServerErrorMessage; import io.xpipe.core.util.JacksonHelper; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.InetAddress; import java.net.Socket; import java.net.SocketException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -50,21 +49,28 @@ public class BeaconClient implements AutoCloseable { } } - private final Socket socket; + private final Closeable closeable; private final InputStream in; private final OutputStream out; public BeaconClient() throws IOException { - socket = new Socket(InetAddress.getLoopbackAddress(), BeaconConfig.getUsedPort()); + var socket = new Socket(InetAddress.getLoopbackAddress(), BeaconConfig.getUsedPort()); + closeable = socket; in = socket.getInputStream(); out = socket.getOutputStream(); } + public BeaconClient(Closeable closeable, InputStream in, OutputStream out) { + this.closeable = closeable; + this.in = in; + this.out = out; + } + public void close() throws ConnectorException { try { - socket.close(); + closeable.close(); } catch (IOException ex) { - throw new ConnectorException("Couldn't close socket", ex); + throw new ConnectorException("Couldn't close client", ex); } } @@ -74,7 +80,7 @@ public class BeaconClient implements AutoCloseable { if (sep.length != 0 && !Arrays.equals(BODY_SEPARATOR, sep)) { throw new ConnectorException("Invalid body separator"); } - return BeaconFormat.readBlocks(socket); + return BeaconFormat.readBlocks(in); } catch (IOException ex) { throw new ConnectorException(ex); } @@ -83,13 +89,13 @@ public class BeaconClient implements AutoCloseable { public OutputStream sendBody() throws ConnectorException { try { out.write(BODY_SEPARATOR); - return BeaconFormat.writeBlocks(socket); + return BeaconFormat.writeBlocks(out); } catch (IOException ex) { throw new ConnectorException(ex); } } - public void sendRequest(T req) throws ClientException, ConnectorException { + public void sendRequest(T req) throws ClientException, ConnectorException { ObjectNode json = JacksonHelper.newMapper().valueToTree(req); var prov = MessageExchanges.byRequest(req); if (prov.isEmpty()) { @@ -106,25 +112,31 @@ public class BeaconClient implements AutoCloseable { System.out.println("Sending request to server of type " + req.getClass().getName()); } - if (BeaconConfig.printMessages()) { - System.out.println("Sending raw request:"); - System.out.println(msg.toPrettyString()); + var writer = new StringWriter(); + var mapper = JacksonHelper.newMapper(); + try (JsonGenerator g = mapper.createGenerator(writer).setPrettyPrinter(new DefaultPrettyPrinter())) { + g.writeTree(msg); + } catch (IOException ex) { + throw new ConnectorException("Couldn't serialize request", ex); } - try { - var mapper = JacksonHelper.newMapper().disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); - var gen = mapper.createGenerator(socket.getOutputStream()); - gen.writeTree(msg); + var content = writer.toString(); + if (BeaconConfig.printMessages()) { + System.out.println("Sending raw request:"); + System.out.println(content); + } + + try (OutputStream blockOut = BeaconFormat.writeBlocks(out)) { + blockOut.write(content.getBytes(StandardCharsets.UTF_8)); } catch (IOException ex) { throw new ConnectorException("Couldn't write to socket", ex); } } public T receiveResponse() throws ConnectorException, ClientException, ServerException { - JsonNode read; - try { - var in = socket.getInputStream(); - read = JacksonHelper.newMapper().disable(JsonParser.Feature.AUTO_CLOSE_SOURCE).readTree(in); + JsonNode node; + try (InputStream blockIn = BeaconFormat.readBlocks(in)) { + node = JacksonHelper.newMapper().readTree(blockIn); } catch (SocketException ex) { throw new ConnectorException("Connection to xpipe daemon closed unexpectedly", ex); } catch (IOException ex) { @@ -133,24 +145,24 @@ public class BeaconClient implements AutoCloseable { if (BeaconConfig.printMessages()) { System.out.println("Received response:"); - System.out.println(read.toPrettyString()); + System.out.println(node.toPrettyString()); } - if (read.isMissingNode()) { + if (node.isMissingNode()) { throw new ConnectorException("Received unexpected EOF"); } - var se = parseServerError(read); + var se = parseServerError(node); if (se.isPresent()) { se.get().throwError(); } - var ce = parseClientError(read); + var ce = parseClientError(node); if (ce.isPresent()) { throw ce.get().throwException(); } - return parseResponse(read); + return parseResponse(node); } private Optional parseClientError(JsonNode node) throws ConnectorException { @@ -206,4 +218,12 @@ public class BeaconClient implements AutoCloseable { throw new ConnectorException("Couldn't parse response", ex); } } + + public InputStream getRawInputStream() { + return in; + } + + public OutputStream getRawOutputStream() { + return out; + } } diff --git a/beacon/src/main/java/io/xpipe/beacon/BeaconConnection.java b/beacon/src/main/java/io/xpipe/beacon/BeaconConnection.java index 21b564d4d..2222946f5 100644 --- a/beacon/src/main/java/io/xpipe/beacon/BeaconConnection.java +++ b/beacon/src/main/java/io/xpipe/beacon/BeaconConnection.java @@ -6,22 +6,27 @@ import java.io.OutputStream; public abstract class BeaconConnection implements AutoCloseable { - protected BeaconClient socket; + + protected BeaconClient beaconClient; private InputStream bodyInput; private OutputStream bodyOutput; protected abstract void constructSocket(); + public BeaconClient getBeaconClient() { + return beaconClient; + } + @Override public void close() { try { - if (socket != null) { - socket.close(); + if (beaconClient != null) { + beaconClient.close(); } - socket = null; + beaconClient = null; } catch (Exception e) { - socket = null; + beaconClient = null; throw new BeaconException("Could not close beacon connection", e); } } @@ -43,7 +48,7 @@ public abstract class BeaconConnection implements AutoCloseable { } public void checkClosed() { - if (socket == null) { + if (beaconClient == null) { throw new BeaconException("Socket is closed"); } } @@ -70,7 +75,8 @@ public abstract class BeaconConnection implements AutoCloseable { public void performInputExchange( REQ req, - BeaconClient.FailableBiConsumer responseConsumer) { + BeaconClient.FailableBiConsumer responseConsumer + ) { checkClosed(); performInputOutputExchange(req, null, responseConsumer); @@ -79,33 +85,35 @@ public abstract class BeaconConnection implements AutoCloseable { public void performInputOutputExchange( REQ req, BeaconClient.FailableConsumer reqWriter, - BeaconClient.FailableBiConsumer responseConsumer) { + BeaconClient.FailableBiConsumer responseConsumer + ) { checkClosed(); try { - socket.sendRequest(req); + beaconClient.sendRequest(req); if (reqWriter != null) { - try (var out = socket.sendBody()) { + try (var out = sendBody()) { reqWriter.accept(out); } } - RES res = socket.receiveResponse(); - try (var in = socket.receiveBody()) { + RES res = beaconClient.receiveResponse(); + try (var in = receiveBody()) { responseConsumer.accept(res, in); } } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } public void sendRequest( - REQ req) { + REQ req + ) { checkClosed(); try { - socket.sendRequest(req); + beaconClient.sendRequest(req); } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } @@ -113,9 +121,9 @@ public abstract class BeaconConnection implements AutoCloseable { checkClosed(); try { - return socket.receiveResponse(); + return beaconClient.receiveResponse(); } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } @@ -123,10 +131,10 @@ public abstract class BeaconConnection implements AutoCloseable { checkClosed(); try { - bodyOutput = socket.sendBody(); + bodyOutput = beaconClient.sendBody(); return bodyOutput; } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } @@ -134,38 +142,57 @@ public abstract class BeaconConnection implements AutoCloseable { checkClosed(); try { - bodyInput = socket.receiveBody(); + bodyInput = beaconClient.receiveBody(); return bodyInput; } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } public RES performOutputExchange( REQ req, - BeaconClient.FailableConsumer reqWriter) { + BeaconClient.FailableConsumer reqWriter + ) { checkClosed(); try { - socket.sendRequest(req); - try (var out = socket.sendBody()) { + beaconClient.sendRequest(req); + try (var out = sendBody()) { reqWriter.accept(out); } - return socket.receiveResponse(); + return beaconClient.receiveResponse(); } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } public RES performSimpleExchange( - REQ req) { + REQ req + ) { checkClosed(); try { - socket.sendRequest(req); - return socket.receiveResponse(); + beaconClient.sendRequest(req); + return beaconClient.receiveResponse(); } catch (Exception e) { - throw new BeaconException("Could not communicate with beacon", e); + throw unwrapException(e); } } + + private BeaconException unwrapException(Exception exception) { + if (exception instanceof ServerException s) { + return new BeaconException("And internal server error occurred", s.getCause()); + } + + if (exception instanceof ClientException s) { + return new BeaconException("A client error occurred", s.getCause()); + } + + if (exception instanceof ConnectorException s) { + return new BeaconException("A beacon connection error occurred", s.getCause()); + } + + + return new BeaconException("An unexpected error occurred", exception); + } } diff --git a/beacon/src/main/java/io/xpipe/beacon/BeaconFormat.java b/beacon/src/main/java/io/xpipe/beacon/BeaconFormat.java index 1f2c07e67..8ebdbae3a 100644 --- a/beacon/src/main/java/io/xpipe/beacon/BeaconFormat.java +++ b/beacon/src/main/java/io/xpipe/beacon/BeaconFormat.java @@ -5,7 +5,6 @@ import lombok.experimental.UtilityClass; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.Socket; import java.nio.ByteBuffer; @UtilityClass @@ -13,8 +12,7 @@ public class BeaconFormat { private static final int SEGMENT_SIZE = 65536; - public static OutputStream writeBlocks(Socket socket) throws IOException { - var out = socket.getOutputStream(); + public static OutputStream writeBlocks(OutputStream out) throws IOException { return new OutputStream() { private final byte[] currentBytes = new byte[SEGMENT_SIZE]; private int index; @@ -23,6 +21,7 @@ public class BeaconFormat { public void close() throws IOException { finishBlock(); out.flush(); + index = -1; } @Override @@ -49,8 +48,7 @@ public class BeaconFormat { }; } - public static InputStream readBlocks(Socket socket) throws IOException { - var in = socket.getInputStream(); + public static InputStream readBlocks(InputStream in) throws IOException { return new InputStream() { private byte[] currentBytes; @@ -60,7 +58,9 @@ public class BeaconFormat { @Override public int read() throws IOException { if ((currentBytes == null || index == currentBytes.length) && !lastBlock) { - readBlock(); + if (!readBlock()) { + return -1; + } } if (currentBytes != null && index == currentBytes.length && lastBlock) { @@ -72,8 +72,12 @@ public class BeaconFormat { return out; } - private void readBlock() throws IOException { + private boolean readBlock() throws IOException { var length = in.readNBytes(4); + if (length.length < 4) { + return false; + } + var lengthInt = ByteBuffer.wrap(length).getInt(); if (BeaconConfig.printMessages()) { @@ -85,6 +89,7 @@ public class BeaconFormat { if (lengthInt < SEGMENT_SIZE) { lastBlock = true; } + return true; } }; } diff --git a/beacon/src/main/java/io/xpipe/beacon/XPipeInstance.java b/beacon/src/main/java/io/xpipe/beacon/XPipeInstance.java new file mode 100644 index 000000000..272086c04 --- /dev/null +++ b/beacon/src/main/java/io/xpipe/beacon/XPipeInstance.java @@ -0,0 +1,17 @@ +package io.xpipe.beacon; + +import io.xpipe.core.store.ShellStore; +import lombok.Value; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Value +public class XPipeInstance { + + UUID uuid; + String name; + Map adjacent; + List reachable; +} diff --git a/beacon/src/main/java/io/xpipe/beacon/exchange/cli/InstanceExchange.java b/beacon/src/main/java/io/xpipe/beacon/exchange/cli/InstanceExchange.java new file mode 100644 index 000000000..b237fe426 --- /dev/null +++ b/beacon/src/main/java/io/xpipe/beacon/exchange/cli/InstanceExchange.java @@ -0,0 +1,33 @@ +package io.xpipe.beacon.exchange.cli; + +import io.xpipe.beacon.RequestMessage; +import io.xpipe.beacon.ResponseMessage; +import io.xpipe.beacon.XPipeInstance; +import io.xpipe.beacon.exchange.MessageExchange; +import lombok.Builder; +import lombok.NonNull; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; + +public class InstanceExchange implements MessageExchange { + + @Override + public String getId() { + return "instance"; + } + + @Jacksonized + @Builder + @Value + public static class Request implements RequestMessage { + + } + + @Jacksonized + @Builder + @Value + public static class Response implements ResponseMessage { + @NonNull + XPipeInstance instance; + } +} diff --git a/beacon/src/main/java/module-info.java b/beacon/src/main/java/module-info.java index 75cf65084..7b64cbeab 100644 --- a/beacon/src/main/java/module-info.java +++ b/beacon/src/main/java/module-info.java @@ -23,6 +23,7 @@ module io.xpipe.beacon { uses MessageExchange; provides io.xpipe.beacon.exchange.MessageExchange with ForwardExchange, + InstanceExchange, EditStoreExchange, AddSourceExchange, StoreProviderListExchange, diff --git a/core/build.gradle b/core/build.gradle index cd5577087..5f54d4177 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -9,6 +9,9 @@ apply from: "$rootDir/deps/java.gradle" apply from: "$rootDir/deps/lombok.gradle" apply from: "$rootDir/deps/junit.gradle" +compileJava { + options.compilerArgs << '-parameters' +} dependencies{ api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "2.13.0" diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericArrayReader.java b/core/src/main/java/io/xpipe/core/data/generic/GenericArrayReader.java index d6d5119dc..ec39e56c0 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericArrayReader.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericArrayReader.java @@ -74,9 +74,9 @@ public class GenericArrayReader implements GenericAbstractReader { } @Override - public void onArrayEnd() { + public void onArrayEnd(Map metaAttributes) { if (hasReader()) { - currentReader.onArrayEnd(); + currentReader.onArrayEnd(metaAttributes); if (currentReader.isDone()) { put(currentReader.create()); currentReader = null; @@ -114,9 +114,9 @@ public class GenericArrayReader implements GenericAbstractReader { } @Override - public void onTupleEnd() { + public void onTupleEnd(Map metaAttributes) { if (hasReader()) { - currentReader.onTupleEnd(); + currentReader.onTupleEnd(metaAttributes); if (currentReader.isDone()) { put(currentReader.create()); currentReader = null; diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamCallback.java b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamCallback.java index 7e2a0e19e..445e99e13 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamCallback.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamCallback.java @@ -10,13 +10,13 @@ public interface GenericDataStreamCallback { default void onArrayStart(int length) { } - default void onArrayEnd() { + default void onArrayEnd(Map metaAttributes) { } default void onTupleStart(int length) { } - default void onTupleEnd() { + default void onTupleEnd(Map metaAttributes) { } diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamParser.java b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamParser.java index a4f717325..97206286a 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamParser.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamParser.java @@ -67,7 +67,7 @@ public class GenericDataStreamParser { parse(in, cb); } var attributes = DataStructureNodeIO.parseAttributes(in); - cb.onTupleEnd(); + cb.onTupleEnd(attributes); } private static void parseArray(InputStream in, GenericDataStreamCallback cb) throws IOException { @@ -77,7 +77,7 @@ public class GenericDataStreamParser { parse(in, cb); } var attributes = DataStructureNodeIO.parseAttributes(in); - cb.onArrayEnd(); + cb.onArrayEnd(attributes); } private static void parseValue(InputStream in, GenericDataStreamCallback cb) throws IOException { diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamWriter.java b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamWriter.java index 40c8c501d..be01adaa5 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamWriter.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStreamWriter.java @@ -52,8 +52,8 @@ public class GenericDataStreamWriter { private static void writeValue(OutputStream out, ValueNode n) throws IOException { out.write(DataStructureNodeIO.GENERIC_VALUE_ID); - DataStructureNodeIO.writeShort(out, n.getRawData().length); - out.write(n.getRawData()); + var length = DataStructureNodeIO.writeShort(out, n.getRawData().length); + out.write(n.getRawData(), 0, length); DataStructureNodeIO.writeAttributes(out, n); } } diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStructureNodeReader.java b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStructureNodeReader.java index a33a49683..d053d712f 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericDataStructureNodeReader.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericDataStructureNodeReader.java @@ -44,12 +44,12 @@ public class GenericDataStructureNodeReader implements GenericDataStreamCallback } @Override - public void onArrayEnd() { + public void onArrayEnd(Map metaAttributes) { if (!hasReader()) { throw new IllegalStateException("No array to close"); } - reader.onArrayEnd(); + reader.onArrayEnd(metaAttributes); if (reader.isDone()) { node = reader.create(); reader = null; @@ -67,12 +67,12 @@ public class GenericDataStructureNodeReader implements GenericDataStreamCallback } @Override - public void onTupleEnd() { + public void onTupleEnd(Map metaAttributes) { if (!hasReader()) { throw new IllegalStateException("No tuple to close"); } - reader.onTupleEnd(); + reader.onTupleEnd(metaAttributes); if (reader.isDone()) { node = reader.create(); reader = null; diff --git a/core/src/main/java/io/xpipe/core/data/generic/GenericTupleReader.java b/core/src/main/java/io/xpipe/core/data/generic/GenericTupleReader.java index 375847db8..a84d3adc8 100644 --- a/core/src/main/java/io/xpipe/core/data/generic/GenericTupleReader.java +++ b/core/src/main/java/io/xpipe/core/data/generic/GenericTupleReader.java @@ -89,9 +89,9 @@ public class GenericTupleReader implements GenericAbstractReader { } @Override - public void onArrayEnd() { + public void onArrayEnd(Map metaAttributes) { if (hasReader()) { - currentReader.onArrayEnd(); + currentReader.onArrayEnd(metaAttributes); if (currentReader.isDone()) { putNode(currentReader.create()); currentReader = null; @@ -118,9 +118,9 @@ public class GenericTupleReader implements GenericAbstractReader { } @Override - public void onTupleEnd() { + public void onTupleEnd(Map metaAttributes) { if (hasReader()) { - currentReader.onTupleEnd(); + currentReader.onTupleEnd(metaAttributes); if (currentReader.isDone()) { putNode(currentReader.create()); currentReader = null; diff --git a/core/src/main/java/io/xpipe/core/data/node/DataStructureNode.java b/core/src/main/java/io/xpipe/core/data/node/DataStructureNode.java index c049a46c2..a026079f7 100644 --- a/core/src/main/java/io/xpipe/core/data/node/DataStructureNode.java +++ b/core/src/main/java/io/xpipe/core/data/node/DataStructureNode.java @@ -12,18 +12,26 @@ public abstract class DataStructureNode implements Iterable { public static final Integer KEY_TABLE_NAME = 1; public static final Integer KEY_ROW_NAME = 2; public static final Integer BOOLEAN_TRUE = 3; - public static final Integer BOOLEAN_VALUE = 4; + public static final Integer IS_BOOLEAN = 4; public static final Integer BOOLEAN_FALSE = 5; public static final Integer INTEGER_VALUE = 6; - public static final Integer NULL_VALUE = 7; - public static final Integer IS_NUMBER = 8; + public static final Integer IS_NULL = 7; public static final Integer IS_INTEGER = 9; public static final Integer IS_FLOATING_POINT = 10; public static final Integer FLOATING_POINT_VALUE = 11; - public static final Integer TEXT = 12; + public static final Integer IS_TEXT = 12; + public static final Integer IS_INSTANT = 13; + public static final Integer IS_BINARY = 14; private Map metaAttributes; + public void clearMetaAttributes() { + metaAttributes = null; + if (isTuple() || isArray()) { + getNodes().forEach(dataStructureNode -> dataStructureNode.clearMetaAttributes()); + } + } + public Map getMetaAttributes() { return metaAttributes != null ? Collections.unmodifiableMap(metaAttributes) : null; } @@ -138,12 +146,6 @@ public abstract class DataStructureNode implements Iterable { public boolean isValue() { return false; } - - public boolean isNull() { - return false; - } - - public DataStructureNode set(int index, DataStructureNode node) { throw unsupported("set at index"); } diff --git a/core/src/main/java/io/xpipe/core/data/node/DataStructureNodeIO.java b/core/src/main/java/io/xpipe/core/data/node/DataStructureNodeIO.java index 06680c2b3..b9281c7e0 100644 --- a/core/src/main/java/io/xpipe/core/data/node/DataStructureNodeIO.java +++ b/core/src/main/java/io/xpipe/core/data/node/DataStructureNodeIO.java @@ -22,18 +22,25 @@ public class DataStructureNodeIO { public static final int TYPED_ARRAY_ID = 7; public static final int TYPED_VALUE_ID = 8; - public static void writeShort(OutputStream out, int value) throws IOException { + public static int writeShort(OutputStream out, int value) throws IOException { + if (value > Short.MAX_VALUE) { + value = Short.MAX_VALUE; + } + var buffer = ByteBuffer.allocate(2); buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putShort((short) value); out.write(buffer.array()); + return value; } public static void writeString(OutputStream out, String s) throws IOException { if (s != null) { var b = s.getBytes(StandardCharsets.UTF_8); - DataStructureNodeIO.writeShort(out, b.length); - out.write(b); + var length = DataStructureNodeIO.writeShort(out, b.length); + out.write(b, 0, length); + } else { + writeShort(out, -1); } } @@ -50,6 +57,10 @@ public class DataStructureNodeIO { public static String parseString(InputStream in) throws IOException { var nameLength = parseShort(in); + if (nameLength == -1) { + return null; + } + var name = new String(in.readNBytes(nameLength), StandardCharsets.UTF_8); return name; } @@ -74,13 +85,11 @@ public class DataStructureNodeIO { writeShort(out, s.getMetaAttributes().size()); for (Map.Entry entry : s.getMetaAttributes().entrySet()) { Integer integer = entry.getKey(); - var value = entry.getValue().getBytes(StandardCharsets.UTF_8); writeShort(out, integer); - writeShort(out, value.length); - out.write(value); + writeString(out, entry.getValue()); } } else { - out.write(0); + writeShort(out, 0); } } } diff --git a/core/src/main/java/io/xpipe/core/data/node/SimpleImmutableValueNode.java b/core/src/main/java/io/xpipe/core/data/node/SimpleImmutableValueNode.java index 4a336c71e..082e92e0b 100644 --- a/core/src/main/java/io/xpipe/core/data/node/SimpleImmutableValueNode.java +++ b/core/src/main/java/io/xpipe/core/data/node/SimpleImmutableValueNode.java @@ -18,7 +18,7 @@ public class SimpleImmutableValueNode extends ValueNode { @Override public final String asString() { - if (getRawData().length == 0 && !hasMetaAttribute(TEXT)) { + if (getRawData().length == 0 && !hasMetaAttribute(IS_TEXT)) { return "null"; } @@ -27,8 +27,8 @@ public class SimpleImmutableValueNode extends ValueNode { @Override public String toString(int indent) { - var string = getRawData().length == 0 && !hasMetaAttribute(TEXT) ? "" : new String(getRawData(), StandardCharsets.UTF_8); - return (hasMetaAttribute(TEXT) ? "\"" : "") + string + (hasMetaAttribute(TEXT) ? "\"" : "") + " " + metaToString(); + var string = getRawData().length == 0 && !hasMetaAttribute(IS_TEXT) ? "" : new String(getRawData(), StandardCharsets.UTF_8); + return (hasMetaAttribute(IS_TEXT) ? "\"" : "") + string + (hasMetaAttribute(IS_TEXT) ? "\"" : "") + " " + metaToString(); } @Override diff --git a/core/src/main/java/io/xpipe/core/data/node/TupleNode.java b/core/src/main/java/io/xpipe/core/data/node/TupleNode.java index df3919576..c47ce3553 100644 --- a/core/src/main/java/io/xpipe/core/data/node/TupleNode.java +++ b/core/src/main/java/io/xpipe/core/data/node/TupleNode.java @@ -20,6 +20,7 @@ public abstract class TupleNode extends DataStructureNode { return new SimpleTupleNode(null, nodes); } + @SuppressWarnings("unchecked") public static TupleNode of(List names, List nodes) { if (names == null) { throw new IllegalArgumentException("Names must be not null"); diff --git a/core/src/main/java/io/xpipe/core/data/node/ValueNode.java b/core/src/main/java/io/xpipe/core/data/node/ValueNode.java index 01fb0f19b..da06d9c5b 100644 --- a/core/src/main/java/io/xpipe/core/data/node/ValueNode.java +++ b/core/src/main/java/io/xpipe/core/data/node/ValueNode.java @@ -3,6 +3,8 @@ package io.xpipe.core.data.node; import io.xpipe.core.data.type.DataType; import io.xpipe.core.data.type.ValueType; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Objects; @@ -34,13 +36,40 @@ public abstract class ValueNode extends DataStructureNode { } public static ValueNode nullValue() { - return new SimpleImmutableValueNode(new byte[0]).tag(NULL_VALUE).asValue(); + return new SimpleImmutableValueNode(new byte[0]).tag(IS_NULL).asValue(); } public static ValueNode of(byte[] data) { + if (data == null) { + return nullValue(); + } + return new SimpleImmutableValueNode(data); } + public static ValueNode ofBytes(byte[] data) { + var created = of(data); + created.tag(IS_BINARY); + return created; + } + + public static ValueNode ofInteger(BigInteger integer) { + var created = of(integer); + created.tag(IS_INTEGER); + return created; + } + + public static ValueNode ofDecimal(BigDecimal decimal) { + var created = of(decimal); + created.tag(IS_FLOATING_POINT); + return created; + } + public static ValueNode ofBoolean(Boolean bool) { + var created = of(bool); + created.tag(IS_BOOLEAN); + return created; + } + public static ValueNode of(Object o) { if (o == null) { return nullValue(); diff --git a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamCallback.java b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamCallback.java index f1e9b68db..9edea4410 100644 --- a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamCallback.java +++ b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamCallback.java @@ -3,9 +3,11 @@ package io.xpipe.core.data.typed; import io.xpipe.core.data.node.DataStructureNode; import io.xpipe.core.data.type.TupleType; +import java.util.Map; + public interface TypedDataStreamCallback { - default void onValue(byte[] data) { + default void onValue(byte[] data, Map metaAttributes) { } default void onGenericNode(DataStructureNode node) { @@ -14,13 +16,13 @@ public interface TypedDataStreamCallback { default void onTupleBegin(TupleType type) { } - default void onTupleEnd() { + default void onTupleEnd(Map metaAttributes) { } default void onArrayBegin(int size) { } - default void onArrayEnd() { + default void onArrayEnd(Map metaAttributes) { } default void onNodeBegin() { diff --git a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamParser.java b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamParser.java index 99da0e492..1ca48ddef 100644 --- a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamParser.java +++ b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamParser.java @@ -21,7 +21,7 @@ public class TypedDataStreamParser { this.dataType = dataType; } - public boolean hasNext(InputStream in) throws IOException { + private boolean hasNext(InputStream in) throws IOException { var b = in.read(); if (b == -1) { return false; @@ -109,7 +109,8 @@ public class TypedDataStreamParser { for (int i = 0; i < type.getSize(); i++) { parse(in, cb, type.getTypes().get(i)); } - cb.onTupleEnd(); + var attributes = DataStructureNodeIO.parseAttributes(in); + cb.onTupleEnd(attributes); } private GenericDataStructureNodeReader getGenericReader() { @@ -125,12 +126,14 @@ public class TypedDataStreamParser { for (int i = 0; i < size; i++) { parse(in, cb, type.getSharedType()); } - cb.onArrayEnd(); + var attributes = DataStructureNodeIO.parseAttributes(in); + cb.onArrayEnd(attributes); } private void parseValue(InputStream in, TypedDataStreamCallback cb) throws IOException { var size = DataStructureNodeIO.parseShort(in); var data = in.readNBytes(size); - cb.onValue(data); + var attributes = DataStructureNodeIO.parseAttributes(in); + cb.onValue(data, attributes); } } diff --git a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamWriter.java b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamWriter.java index b67117314..f6ab535d4 100644 --- a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamWriter.java +++ b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStreamWriter.java @@ -32,8 +32,9 @@ public class TypedDataStreamWriter { private static void writeValue(OutputStream out, ValueNode n) throws IOException { out.write(DataStructureNodeIO.TYPED_VALUE_ID); - DataStructureNodeIO.writeShort(out, n.getRawData().length); - out.write(n.getRawData()); + var length = DataStructureNodeIO.writeShort(out, n.getRawData().length); + out.write(n.getRawData(), 0, length); + DataStructureNodeIO.writeAttributes(out, n); } private static void writeTuple(OutputStream out, TupleNode tuple, TupleType type) throws IOException { @@ -45,6 +46,7 @@ public class TypedDataStreamWriter { for (int i = 0; i < tuple.size(); i++) { write(out, tuple.at(i), type.getTypes().get(i)); } + DataStructureNodeIO.writeAttributes(out, tuple); } private static void writeArray(OutputStream out, ArrayNode array, ArrayType type) throws IOException { @@ -53,5 +55,6 @@ public class TypedDataStreamWriter { for (int i = 0; i < array.size(); i++) { write(out, array.at(i), type.getSharedType()); } + DataStructureNodeIO.writeAttributes(out, array); } } diff --git a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStructureNodeReader.java b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStructureNodeReader.java index 76ed7cc91..abe0f0280 100644 --- a/core/src/main/java/io/xpipe/core/data/typed/TypedDataStructureNodeReader.java +++ b/core/src/main/java/io/xpipe/core/data/typed/TypedDataStructureNodeReader.java @@ -7,6 +7,7 @@ import io.xpipe.core.data.type.TupleType; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Stack; public class TypedDataStructureNodeReader implements TypedAbstractReader { @@ -74,12 +75,12 @@ public class TypedDataStructureNodeReader implements TypedAbstractReader { } @Override - public void onValue(byte[] data) { + public void onValue(byte[] data, Map metaAttributes) { if (!expectedType.isValue()) { throw new IllegalStateException("Expected " + expectedType.getName() + " but got value"); } - var val = ValueNode.of(data); + var val = ValueNode.of(data).tag(metaAttributes); finishNode(val); moveExpectedType(false); } @@ -115,14 +116,14 @@ public class TypedDataStructureNodeReader implements TypedAbstractReader { } @Override - public void onTupleEnd() { + public void onTupleEnd(Map metaAttributes) { children.pop(); var popped = nodes.pop(); if (!popped.isTuple()) { throw new IllegalStateException("No tuple to end"); } - TupleNode tuple = popped.asTuple(); + TupleNode tuple = popped.tag(metaAttributes).asTuple(); if (tuple.getKeyNames().size() != tuple.getNodes().size()) { throw new IllegalStateException("Tuple node size mismatch"); } @@ -154,7 +155,7 @@ public class TypedDataStructureNodeReader implements TypedAbstractReader { } @Override - public void onArrayEnd() { + public void onArrayEnd(Map metaAttributes) { if (!isInArray()) { throw new IllegalStateException("No array to end"); } @@ -163,7 +164,7 @@ public class TypedDataStructureNodeReader implements TypedAbstractReader { moveExpectedType(true); children.pop(); - var popped = nodes.pop(); + var popped = nodes.pop().tag(metaAttributes); finishNode(popped); } } diff --git a/core/src/main/java/io/xpipe/core/dialog/Dialog.java b/core/src/main/java/io/xpipe/core/dialog/Dialog.java index 61c5862e1..9e60307f5 100644 --- a/core/src/main/java/io/xpipe/core/dialog/Dialog.java +++ b/core/src/main/java/io/xpipe/core/dialog/Dialog.java @@ -1,6 +1,6 @@ package io.xpipe.core.dialog; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import java.util.*; import java.util.function.Consumer; @@ -164,7 +164,7 @@ public abstract class Dialog { /** * A special wrapper for secret values of {@link #query(String, boolean, boolean, boolean, Object, QueryConverter)}. */ - public static Dialog.Query querySecret(String description, boolean newLine, boolean required, Secret value) { + public static Dialog.Query querySecret(String description, boolean newLine, boolean required, SecretValue value) { var q = new Dialog.Query(description, newLine, required, false, value, QueryConverter.SECRET, true); q.evaluateTo(q::getConvertedValue); return q; diff --git a/core/src/main/java/io/xpipe/core/dialog/QueryConverter.java b/core/src/main/java/io/xpipe/core/dialog/QueryConverter.java index 4ca67a01d..24c7e9ae9 100644 --- a/core/src/main/java/io/xpipe/core/dialog/QueryConverter.java +++ b/core/src/main/java/io/xpipe/core/dialog/QueryConverter.java @@ -2,7 +2,7 @@ package io.xpipe.core.dialog; import io.xpipe.core.charsetter.NewLine; import io.xpipe.core.charsetter.StreamCharset; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import java.net.URI; import java.net.URISyntaxException; @@ -48,14 +48,14 @@ public abstract class QueryConverter { } }; - public static final QueryConverter SECRET = new QueryConverter() { + public static final QueryConverter SECRET = new QueryConverter() { @Override - protected Secret fromString(String s) { - return new Secret(s); + protected SecretValue fromString(String s) { + return new SecretValue(s); } @Override - protected String toString(Secret value) { + protected String toString(SecretValue value) { return value.getEncryptedValue(); } }; diff --git a/core/src/main/java/io/xpipe/core/impl/BatchTableWriteConnection.java b/core/src/main/java/io/xpipe/core/impl/BatchTableWriteConnection.java new file mode 100644 index 000000000..2282077ac --- /dev/null +++ b/core/src/main/java/io/xpipe/core/impl/BatchTableWriteConnection.java @@ -0,0 +1,48 @@ +package io.xpipe.core.impl; + +import io.xpipe.core.data.node.ArrayNode; +import io.xpipe.core.data.node.DataStructureNode; +import io.xpipe.core.data.node.DataStructureNodeAcceptor; +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.source.TableWriteConnection; + +import java.util.ArrayList; +import java.util.List; + +public abstract class BatchTableWriteConnection implements TableWriteConnection { + + public static final int BATCH_SIZE = 2000; + + private final List batch = new ArrayList<>(); + + @Override + public final DataStructureNodeAcceptor writeLinesAcceptor() { + return node -> { + if (batch.size() < BATCH_SIZE) { + batch.add(node); + return true; + } + + var array = ArrayNode.of(batch); + var returned = writeBatchLinesAcceptor().accept(array); + batch.clear(); + return returned; + }; + } + + @Override + public final void close() throws Exception { + if (batch.size() > 0) { + var array = ArrayNode.of(batch); + var returned = writeBatchLinesAcceptor().accept(array); + batch.clear(); + } + onClose(); + } + + protected abstract void onClose() throws Exception; + + protected abstract DataStructureNodeAcceptor writeBatchLinesAcceptor(); +} + + diff --git a/core/src/main/java/io/xpipe/core/impl/BinarySource.java b/core/src/main/java/io/xpipe/core/impl/BinarySource.java new file mode 100644 index 000000000..7c9adcde3 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/impl/BinarySource.java @@ -0,0 +1,67 @@ +package io.xpipe.core.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.source.RawDataSource; +import io.xpipe.core.source.RawReadConnection; +import io.xpipe.core.source.RawWriteConnection; +import io.xpipe.core.store.StreamDataStore; +import lombok.experimental.SuperBuilder; + +import java.io.InputStream; +import java.io.OutputStream; + +@JsonTypeName("binary") +@SuperBuilder +public class BinarySource extends RawDataSource { + + @Override + protected RawWriteConnection newWriteConnection() { + return new RawWriteConnection() { + + private OutputStream out; + + @Override + public void init() throws Exception { + out = getStore().openOutput(); + } + + @Override + public void close() throws Exception { + out.close(); + } + + @Override + public void write(byte[] bytes) throws Exception { + out.write(bytes); + } + }; + } + + + @Override + protected RawReadConnection newReadConnection() { + return new RawReadConnection() { + + private InputStream inputStream; + + @Override + public void init() throws Exception { + if (inputStream != null) { + throw new IllegalStateException("Already initialized"); + } + + inputStream = getStore().openInput(); + } + + @Override + public void close() throws Exception { + inputStream.close(); + } + + @Override + public byte[] readBytes(int max) throws Exception { + return inputStream.readNBytes(max); + } + }; + } +} diff --git a/core/src/main/java/io/xpipe/core/impl/LimitTableReadConnection.java b/core/src/main/java/io/xpipe/core/impl/LimitTableReadConnection.java new file mode 100644 index 000000000..595bc52bc --- /dev/null +++ b/core/src/main/java/io/xpipe/core/impl/LimitTableReadConnection.java @@ -0,0 +1,59 @@ +package io.xpipe.core.impl; + +import io.xpipe.core.data.node.DataStructureNodeAcceptor; +import io.xpipe.core.data.node.TupleNode; +import io.xpipe.core.data.type.TupleType; +import io.xpipe.core.source.TableReadConnection; + +import java.util.OptionalInt; +import java.util.concurrent.atomic.AtomicInteger; + +public class LimitTableReadConnection implements TableReadConnection { + + private final TableReadConnection connection; + private final int maxCount; + private int count = 0; + + public LimitTableReadConnection(TableReadConnection connection, int maxCount) { + this.connection = connection; + this.maxCount = maxCount; + } + + @Override + public void init() throws Exception { + connection.init(); + } + + @Override + public void close() throws Exception { + connection.close(); + } + + @Override + public TupleType getDataType() { + return connection.getDataType(); + } + + @Override + public OptionalInt getRowCount() throws Exception { + return connection.getRowCount(); + } + + @Override + public int withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { + AtomicInteger localCounter = new AtomicInteger(); + connection.withRows(node -> { + if (count == maxCount) { + return false; + } + count++; + + var returned = lineAcceptor + .accept(node); + localCounter.getAndIncrement(); + + return returned; + }); + return localCounter.get(); + } +} diff --git a/core/src/main/java/io/xpipe/core/impl/TextSource.java b/core/src/main/java/io/xpipe/core/impl/TextSource.java index dcdbbf883..f966e07fa 100644 --- a/core/src/main/java/io/xpipe/core/impl/TextSource.java +++ b/core/src/main/java/io/xpipe/core/impl/TextSource.java @@ -1,37 +1,34 @@ package io.xpipe.core.impl; -import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; import io.xpipe.core.charsetter.Charsettable; import io.xpipe.core.charsetter.NewLine; import io.xpipe.core.charsetter.StreamCharset; import io.xpipe.core.source.TextDataSource; import io.xpipe.core.store.StreamDataStore; -import lombok.EqualsAndHashCode; -import lombok.Value; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; -@Value -@EqualsAndHashCode(callSuper = true) -public class TextSource extends TextDataSource implements Charsettable { +@Getter +@JsonTypeName("text") +@SuperBuilder +@Jacksonized +public final class TextSource extends TextDataSource implements Charsettable { - StreamCharset charset; - NewLine newLine; - - public TextSource(StreamDataStore store){ - this(store, StreamCharset.UTF8, NewLine.LF); - } - - @JsonCreator - public TextSource(StreamDataStore store, StreamCharset charset, NewLine newLine) { - super(store); - this.charset = charset; - this.newLine = newLine; - } + private final StreamCharset charset; + private final NewLine newLine; @Override protected io.xpipe.core.source.TextWriteConnection newWriteConnection() { return new TextWriteConnection(this); } + @Override + protected io.xpipe.core.source.TextWriteConnection newPrependingWriteConnection() { + return new PreservingTextWriteConnection(this, newWriteConnection(), false); + } + @Override protected io.xpipe.core.source.TextWriteConnection newAppendingWriteConnection() { return new PreservingTextWriteConnection(this, newWriteConnection(), true); diff --git a/core/src/main/java/io/xpipe/core/impl/XpbsSource.java b/core/src/main/java/io/xpipe/core/impl/XpbsSource.java new file mode 100644 index 000000000..91601bad5 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/impl/XpbsSource.java @@ -0,0 +1,23 @@ +package io.xpipe.core.impl; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.xpipe.core.source.StructureDataSource; +import io.xpipe.core.source.StructureReadConnection; +import io.xpipe.core.source.StructureWriteConnection; +import io.xpipe.core.store.StreamDataStore; +import lombok.experimental.SuperBuilder; + +@JsonTypeName("xpbs") +@SuperBuilder +public class XpbsSource extends StructureDataSource { + + @Override + protected StructureWriteConnection newWriteConnection() { + return null; + } + + @Override + protected StructureReadConnection newReadConnection() { + return null; + } +} diff --git a/core/src/main/java/io/xpipe/core/impl/XpbtReadConnection.java b/core/src/main/java/io/xpipe/core/impl/XpbtReadConnection.java index 0d4063227..028858b31 100644 --- a/core/src/main/java/io/xpipe/core/impl/XpbtReadConnection.java +++ b/core/src/main/java/io/xpipe/core/impl/XpbtReadConnection.java @@ -1,5 +1,6 @@ package io.xpipe.core.impl; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import io.xpipe.core.data.node.DataStructureNodeAcceptor; @@ -33,10 +34,11 @@ public class XpbtReadConnection implements TableReadConnection { } var headerLength = header.getBytes(StandardCharsets.UTF_8).length; - this.inputStream.skip(headerLength); + this.inputStream.skip(headerLength + 1); List names = JacksonHelper.newMapper() .disable(JsonParser.Feature.AUTO_CLOSE_SOURCE) - .readerFor(new TypeReference>(){}).readValue(header); + .readerFor(new TypeReference>() { + }).readValue(header); TupleType dataType = TupleType.tableType(names); this.dataType = dataType; this.parser = new TypedDataStreamParser(dataType); @@ -53,7 +55,7 @@ public class XpbtReadConnection implements TableReadConnection { private TypedDataStreamParser parser; private boolean empty; - protected XpbtReadConnection(StreamDataStore store) { + protected XpbtReadConnection(StreamDataStore store) { this.store = store; } @@ -63,14 +65,15 @@ public class XpbtReadConnection implements TableReadConnection { } @Override - public void withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { + public int withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { if (empty) { - return; + return 0; } var reader = TypedDataStructureNodeReader.of(dataType); AtomicBoolean quit = new AtomicBoolean(false); AtomicReference exception = new AtomicReference<>(); + var counter = 0; while (!quit.get()) { var node = parser.parseStructure(inputStream, reader); if (node == null) { @@ -82,6 +85,7 @@ public class XpbtReadConnection implements TableReadConnection { if (!lineAcceptor.accept(node.asTuple())) { quit.set(true); } + counter++; } catch (Exception ex) { quit.set(true); exception.set(ex); @@ -91,5 +95,6 @@ public class XpbtReadConnection implements TableReadConnection { if (exception.get() != null) { throw exception.get(); } + return counter; } } diff --git a/core/src/main/java/io/xpipe/core/impl/XpbtSource.java b/core/src/main/java/io/xpipe/core/impl/XpbtSource.java index cf4a0c112..ae3980b78 100644 --- a/core/src/main/java/io/xpipe/core/impl/XpbtSource.java +++ b/core/src/main/java/io/xpipe/core/impl/XpbtSource.java @@ -1,18 +1,16 @@ package io.xpipe.core.impl; -import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonTypeName; import io.xpipe.core.source.TableDataSource; import io.xpipe.core.source.TableReadConnection; import io.xpipe.core.source.TableWriteConnection; import io.xpipe.core.store.StreamDataStore; +import lombok.experimental.SuperBuilder; +@JsonTypeName("xpbt") +@SuperBuilder public class XpbtSource extends TableDataSource { - @JsonCreator - public XpbtSource(StreamDataStore store) { - super(store); - } - @Override protected TableWriteConnection newWriteConnection() { return new XpbtWriteConnection(store); diff --git a/core/src/main/java/io/xpipe/core/impl/XpbtWriteConnection.java b/core/src/main/java/io/xpipe/core/impl/XpbtWriteConnection.java index 24e7a7b18..b638ddc3a 100644 --- a/core/src/main/java/io/xpipe/core/impl/XpbtWriteConnection.java +++ b/core/src/main/java/io/xpipe/core/impl/XpbtWriteConnection.java @@ -60,6 +60,8 @@ public class XpbtWriteConnection implements TableWriteConnection { .setPrettyPrinter(new DefaultPrettyPrinter())) { JacksonHelper.newMapper().disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) .writeValue(g, tupleNode.getKeyNames()); + writer.append("\n"); } + writer.flush(); } } diff --git a/core/src/main/java/io/xpipe/core/source/CollectionDataSource.java b/core/src/main/java/io/xpipe/core/source/CollectionDataSource.java index a88ced22b..2eee7aa8a 100644 --- a/core/src/main/java/io/xpipe/core/source/CollectionDataSource.java +++ b/core/src/main/java/io/xpipe/core/source/CollectionDataSource.java @@ -1,19 +1,17 @@ package io.xpipe.core.source; import io.xpipe.core.store.DataStore; +import lombok.Singular; +import lombok.experimental.SuperBuilder; -import java.util.HashMap; import java.util.Map; +@SuperBuilder public abstract class CollectionDataSource extends DataSource { + @Singular private final Map preferredProviders; - public CollectionDataSource(DS store) { - super(store); - this.preferredProviders = new HashMap<>(); - } - public CollectionDataSource annotate(String file, String provider) { preferredProviders.put(file, provider); return this; diff --git a/core/src/main/java/io/xpipe/core/source/DataSource.java b/core/src/main/java/io/xpipe/core/source/DataSource.java index 7c07df20e..2c6a9ed8f 100644 --- a/core/src/main/java/io/xpipe/core/source/DataSource.java +++ b/core/src/main/java/io/xpipe/core/source/DataSource.java @@ -1,15 +1,16 @@ package io.xpipe.core.source; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.util.TokenBuffer; +import io.xpipe.core.charsetter.NewLine; +import io.xpipe.core.charsetter.StreamCharset; import io.xpipe.core.impl.TextSource; import io.xpipe.core.impl.XpbtSource; import io.xpipe.core.store.DataFlow; import io.xpipe.core.store.DataStore; import io.xpipe.core.util.JacksonHelper; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.SneakyThrows; +import lombok.experimental.SuperBuilder; import java.util.Optional; @@ -19,18 +20,21 @@ import java.util.Optional; *

* This instance is only valid in combination with its associated data store instance. */ -@Data -@NoArgsConstructor -@AllArgsConstructor +@SuperBuilder +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "type" +) public abstract class DataSource { public static DataSource createInternalDataSource(DataSourceType t, DataStore store) { try { return switch (t) { - case TABLE -> new XpbtSource(store.asNeeded()); + case TABLE -> XpbtSource.builder().store(store.asNeeded()).build(); case STRUCTURE -> null; - case TEXT -> new TextSource(store.asNeeded()); + case TEXT -> TextSource.builder().store(store.asNeeded()).newLine(NewLine.LF).charset( + StreamCharset.UTF8).build(); case RAW -> null; //TODO case COLLECTION -> null; @@ -55,6 +59,15 @@ public abstract class DataSource { store.validate(); } + public WriteMode[] getAvailableWriteModes() { + if (getFlow() != null && !getFlow().hasOutput()) { + return new WriteMode[0]; + } + + return WriteMode.values(); + } + + public DataFlow getFlow() { if (store == null) { return null; @@ -72,6 +85,32 @@ public abstract class DataSource { return (T) mapper.readValue(tb.asParser(), getClass()); } + @SneakyThrows + public final String toString() { + var tree = JacksonHelper.newMapper().valueToTree(this); + return tree.toPrettyString(); + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (getClass() != o.getClass()) { + return false; + } + + var tree = JacksonHelper.newMapper().valueToTree(this); + var otherTree = JacksonHelper.newMapper().valueToTree(o); + return tree.equals(otherTree); + } + + @Override + public final int hashCode() { + var tree = JacksonHelper.newMapper().valueToTree(this); + return tree.hashCode(); + } + public DataSource withStore(DS store) { var c = copy(); c.store = store; diff --git a/core/src/main/java/io/xpipe/core/source/RawDataSource.java b/core/src/main/java/io/xpipe/core/source/RawDataSource.java index 1ed85f820..5930d9c93 100644 --- a/core/src/main/java/io/xpipe/core/source/RawDataSource.java +++ b/core/src/main/java/io/xpipe/core/source/RawDataSource.java @@ -1,15 +1,13 @@ package io.xpipe.core.source; import io.xpipe.core.store.DataStore; +import lombok.experimental.SuperBuilder; +@SuperBuilder public abstract class RawDataSource extends DataSource { private static final int MAX_BYTES_READ = 100000; - public RawDataSource(DS store) { - super(store); - } - @Override public final DataSourceInfo determineInfo() throws Exception { try (var con = openReadConnection()) { diff --git a/core/src/main/java/io/xpipe/core/source/StructureDataSource.java b/core/src/main/java/io/xpipe/core/source/StructureDataSource.java index 778989f38..460986c19 100644 --- a/core/src/main/java/io/xpipe/core/source/StructureDataSource.java +++ b/core/src/main/java/io/xpipe/core/source/StructureDataSource.java @@ -2,13 +2,11 @@ package io.xpipe.core.source; import io.xpipe.core.data.node.DataStructureNode; import io.xpipe.core.store.DataStore; +import lombok.experimental.SuperBuilder; +@SuperBuilder public abstract class StructureDataSource extends DataSource { - public StructureDataSource(DS store) { - super(store); - } - private int countEntries(DataStructureNode n) { if (n.isValue()) { return 1; diff --git a/core/src/main/java/io/xpipe/core/source/TableDataSource.java b/core/src/main/java/io/xpipe/core/source/TableDataSource.java index caa8881ee..8832402ea 100644 --- a/core/src/main/java/io/xpipe/core/source/TableDataSource.java +++ b/core/src/main/java/io/xpipe/core/source/TableDataSource.java @@ -2,19 +2,11 @@ package io.xpipe.core.source; import io.xpipe.core.impl.PreservingTableWriteConnection; import io.xpipe.core.store.DataStore; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; -@Data -@EqualsAndHashCode(callSuper = true) -@NoArgsConstructor +@SuperBuilder public abstract class TableDataSource extends DataSource { - public TableDataSource(DS store) { - super(store); - } - @Override public final DataSourceInfo determineInfo() throws Exception { if (!getFlow().hasInput()) { diff --git a/core/src/main/java/io/xpipe/core/source/TableReadConnection.java b/core/src/main/java/io/xpipe/core/source/TableReadConnection.java index 84577ea5c..b17afcafa 100644 --- a/core/src/main/java/io/xpipe/core/source/TableReadConnection.java +++ b/core/src/main/java/io/xpipe/core/source/TableReadConnection.java @@ -7,6 +7,7 @@ import io.xpipe.core.data.node.ArrayNode; import io.xpipe.core.data.node.TupleNode; import io.xpipe.core.data.type.TupleType; import io.xpipe.core.data.typed.TypedDataStreamWriter; +import io.xpipe.core.impl.LimitTableReadConnection; import java.io.OutputStream; import java.util.ArrayList; @@ -31,8 +32,8 @@ public interface TableReadConnection extends DataSourceReadConnection { } @Override - public void withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { - + public int withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception { + return 0; } @Override @@ -54,10 +55,16 @@ public interface TableReadConnection extends DataSourceReadConnection { return OptionalInt.empty(); } + default TableReadConnection limit(int limit) { + return new LimitTableReadConnection(this, limit); + } + /** * Consumes the table rows until the acceptor returns false. + * + * @return */ - void withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception; + int withRows(DataStructureNodeAcceptor lineAcceptor) throws Exception; /** * Reads multiple rows in bulk. @@ -92,7 +99,11 @@ public interface TableReadConnection extends DataSourceReadConnection { } default void forward(DataSourceConnection con) throws Exception { + forwardAndCount(con); + } + + default int forwardAndCount(DataSourceConnection con) throws Exception { var tCon = (TableWriteConnection) con; - withRows(tCon.writeLinesAcceptor()); + return withRows(tCon.writeLinesAcceptor()); } } diff --git a/core/src/main/java/io/xpipe/core/source/TableWriteConnection.java b/core/src/main/java/io/xpipe/core/source/TableWriteConnection.java index b19bfcfdc..802cb8885 100644 --- a/core/src/main/java/io/xpipe/core/source/TableWriteConnection.java +++ b/core/src/main/java/io/xpipe/core/source/TableWriteConnection.java @@ -10,9 +10,20 @@ import io.xpipe.core.data.node.TupleNode; */ public interface TableWriteConnection extends DataSourceConnection { + public static TableWriteConnection empty() { + return new TableWriteConnection() { + @Override + public DataStructureNodeAcceptor writeLinesAcceptor() { + return node -> { + return true; + }; + } + }; + } + DataStructureNodeAcceptor writeLinesAcceptor(); - default void writeLines(ArrayNode lines) throws Exception{ + default void writeLines(ArrayNode lines) throws Exception { var consumer = writeLinesAcceptor(); for (DataStructureNode dataStructureNode : lines.getNodes()) { consumer.accept(dataStructureNode.asTuple()); diff --git a/core/src/main/java/io/xpipe/core/source/TextDataSource.java b/core/src/main/java/io/xpipe/core/source/TextDataSource.java index f21149848..dfb9d9650 100644 --- a/core/src/main/java/io/xpipe/core/source/TextDataSource.java +++ b/core/src/main/java/io/xpipe/core/source/TextDataSource.java @@ -2,19 +2,15 @@ package io.xpipe.core.source; import io.xpipe.core.impl.PreservingTextWriteConnection; import io.xpipe.core.store.DataStore; -import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; import java.util.concurrent.atomic.AtomicInteger; -@NoArgsConstructor +@SuperBuilder public abstract class TextDataSource extends DataSource { private static final int MAX_LINE_READ = 1000; - public TextDataSource(DS store) { - super(store); - } - @Override public final DataSourceInfo determineInfo() throws Exception { if (!getStore().canOpen()) { diff --git a/core/src/main/java/io/xpipe/core/store/FileStore.java b/core/src/main/java/io/xpipe/core/store/FileStore.java index 1e4cf3656..0be58a3b6 100644 --- a/core/src/main/java/io/xpipe/core/store/FileStore.java +++ b/core/src/main/java/io/xpipe/core/store/FileStore.java @@ -1,8 +1,9 @@ package io.xpipe.core.store; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonTypeName; +import lombok.Builder; import lombok.Value; +import lombok.extern.jackson.Jacksonized; import java.io.InputStream; import java.io.OutputStream; @@ -13,6 +14,8 @@ import java.nio.file.Path; */ @Value @JsonTypeName("file") +@Builder +@Jacksonized public class FileStore implements StreamDataStore, FilenameStore { public static FileStore local(Path p) { @@ -29,12 +32,6 @@ public class FileStore implements StreamDataStore, FilenameStore { MachineFileStore machine; String file; - @JsonCreator - public FileStore(MachineFileStore machine, String file) { - this.machine = machine; - this.file = file; - } - @Override public void validate() throws Exception { if (machine == null) { diff --git a/core/src/main/java/io/xpipe/core/store/InMemoryStore.java b/core/src/main/java/io/xpipe/core/store/InMemoryStore.java index b2a4f3e35..1626c59d0 100644 --- a/core/src/main/java/io/xpipe/core/store/InMemoryStore.java +++ b/core/src/main/java/io/xpipe/core/store/InMemoryStore.java @@ -6,6 +6,7 @@ import lombok.Value; import lombok.experimental.NonFinal; import java.io.*; +import java.nio.charset.StandardCharsets; /** * A store whose contents are stored in memory. @@ -47,4 +48,7 @@ public class InMemoryStore implements StreamDataStore { }; } + public String toString() { + return new String(value, StandardCharsets.UTF_8); + } } diff --git a/core/src/main/java/io/xpipe/core/store/LocalStore.java b/core/src/main/java/io/xpipe/core/store/LocalStore.java index 9107be95d..a2b17812f 100644 --- a/core/src/main/java/io/xpipe/core/store/LocalStore.java +++ b/core/src/main/java/io/xpipe/core/store/LocalStore.java @@ -2,7 +2,7 @@ package io.xpipe.core.store; import com.fasterxml.jackson.annotation.JsonTypeName; import io.xpipe.core.charsetter.NewLine; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import lombok.EqualsAndHashCode; import lombok.Value; @@ -27,14 +27,14 @@ public class LocalStore extends StandardShellStore implements MachineFileStore { class LocalProcessControl extends ProcessControl { - private final List input; + private final List input; private final Integer timeout; private final List command; private Charset charset; private Process process; - LocalProcessControl(List input, List cmd, Integer timeout) { + LocalProcessControl(List input, List cmd, Integer timeout) { this.input = input; this.timeout = timeout; this.command = cmd; @@ -80,7 +80,12 @@ public class LocalStore extends StandardShellStore implements MachineFileStore { return process.getInputStream(); } - @Override + @Override + public OutputStream getStdin() { + return process.getOutputStream(); + } + + @Override public InputStream getStderr() { return process.getErrorStream(); } @@ -125,12 +130,12 @@ public class LocalStore extends StandardShellStore implements MachineFileStore { } @Override - public ProcessControl prepareCommand(List input, List cmd, Integer timeout) { + public ProcessControl prepareCommand(List input, List cmd, Integer timeout) { return new LocalProcessControl(input, cmd, getEffectiveTimeOut(timeout)); } @Override - public ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeOut) throws Exception { + public ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeOut) throws Exception { return new LocalProcessControl(input, cmd, getEffectiveTimeOut(timeOut)); } diff --git a/core/src/main/java/io/xpipe/core/store/ProcessControl.java b/core/src/main/java/io/xpipe/core/store/ProcessControl.java index 6c167c518..201da03c2 100644 --- a/core/src/main/java/io/xpipe/core/store/ProcessControl.java +++ b/core/src/main/java/io/xpipe/core/store/ProcessControl.java @@ -68,6 +68,7 @@ public abstract class ProcessControl { public abstract int waitFor() throws Exception; public abstract InputStream getStdout(); + public abstract OutputStream getStdin(); public abstract InputStream getStderr(); diff --git a/core/src/main/java/io/xpipe/core/store/ShellStore.java b/core/src/main/java/io/xpipe/core/store/ShellStore.java index dd60f0159..ef9e571d2 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellStore.java +++ b/core/src/main/java/io/xpipe/core/store/ShellStore.java @@ -1,6 +1,6 @@ package io.xpipe.core.store; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -15,7 +15,7 @@ public abstract class ShellStore implements DataStore { return null; } - public List getInput() { + public List getInput() { return List.of(); } @@ -31,7 +31,7 @@ public abstract class ShellStore implements DataStore { return string; } - public String executeAndCheckOut(List in, List cmd, Integer timeout) throws ProcessOutputException, Exception { + public String executeAndCheckOut(List in, List cmd, Integer timeout) throws ProcessOutputException, Exception { var pc = prepareCommand(in, cmd, getEffectiveTimeOut(timeout)); pc.start(); @@ -76,7 +76,7 @@ public abstract class ShellStore implements DataStore { } } - public Optional executeAndCheckErr(List in, List cmd) throws Exception { + public Optional executeAndCheckErr(List in, List cmd) throws Exception { var pc = prepareCommand(in, cmd, getTimeout()); pc.start(); var outT = pc.discardOut(); @@ -113,13 +113,13 @@ public abstract class ShellStore implements DataStore { return prepareCommand(List.of(), cmd, timeout); } - public abstract ProcessControl prepareCommand(List input, List cmd, Integer timeout) throws Exception; + public abstract ProcessControl prepareCommand(List input, List cmd, Integer timeout) throws Exception; public ProcessControl preparePrivilegedCommand(List cmd, Integer timeout) throws Exception { return preparePrivilegedCommand(List.of(), cmd, timeout); } - public ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeout) throws Exception { + public ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeout) throws Exception { throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/io/xpipe/core/store/ShellTypes.java b/core/src/main/java/io/xpipe/core/store/ShellTypes.java index c474aa5fc..6e7b156ec 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellTypes.java +++ b/core/src/main/java/io/xpipe/core/store/ShellTypes.java @@ -2,7 +2,7 @@ package io.xpipe.core.store; import com.fasterxml.jackson.annotation.JsonTypeName; import io.xpipe.core.charsetter.NewLine; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import lombok.Value; import java.io.ByteArrayInputStream; @@ -83,7 +83,7 @@ public class ShellTypes { } @Override - public ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { + public ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { var l = List.of("net", "session", ";", "if", "%errorLevel%", "!=", "0"); return st.prepareCommand(List.of(), l, timeout); } @@ -186,12 +186,12 @@ public class ShellTypes { } @Override - public ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { + public ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { var l = new ArrayList<>(cmd); l.add(0, "sudo"); l.add(1, "-S"); var pws = new ByteArrayInputStream(pw.getBytes(determineCharset(st))); - return st.prepareCommand(List.of(Secret.createForSecretValue(pw)), l, timeout); + return st.prepareCommand(List.of(SecretValue.createForSecretValue(pw)), l, timeout); } @Override diff --git a/core/src/main/java/io/xpipe/core/store/StandardShellStore.java b/core/src/main/java/io/xpipe/core/store/StandardShellStore.java index 45f90720d..be1e7617d 100644 --- a/core/src/main/java/io/xpipe/core/store/StandardShellStore.java +++ b/core/src/main/java/io/xpipe/core/store/StandardShellStore.java @@ -2,7 +2,7 @@ package io.xpipe.core.store; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.xpipe.core.charsetter.NewLine; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import java.io.InputStream; import java.io.OutputStream; @@ -16,7 +16,7 @@ public abstract class StandardShellStore extends ShellStore implements MachineFi List switchTo(List cmd); - default ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { + default ProcessControl prepareElevatedCommand(ShellStore st, List in, List cmd, Integer timeout, String pw) throws Exception { return st.prepareCommand(in, cmd, timeout); } diff --git a/core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java b/core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java index 8f663d64f..e6473e9ee 100644 --- a/core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java +++ b/core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java @@ -20,6 +20,8 @@ import io.xpipe.core.dialog.BaseQueryElement; import io.xpipe.core.dialog.BusyElement; import io.xpipe.core.dialog.ChoiceElement; import io.xpipe.core.dialog.HeaderElement; +import io.xpipe.core.impl.TextSource; +import io.xpipe.core.impl.XpbtSource; import io.xpipe.core.source.DataSource; import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceReference; @@ -34,6 +36,9 @@ public class CoreJacksonModule extends SimpleModule { @Override public void setupModule(SetupContext context) { context.registerSubtypes( + new NamedType(TextSource.class), + new NamedType(XpbtSource.class), + new NamedType(FileStore.class), new NamedType(StdinDataStore.class), new NamedType(StdoutDataStore.class), @@ -70,8 +75,8 @@ public class CoreJacksonModule extends SimpleModule { addSerializer(Path.class, new LocalPathSerializer()); addDeserializer(Path.class, new LocalPathDeserializer()); - addSerializer(Secret.class, new SecretSerializer()); - addDeserializer(Secret.class, new SecretDeserializer()); + addSerializer(SecretValue.class, new SecretSerializer()); + addDeserializer(SecretValue.class, new SecretDeserializer()); addSerializer(DataSourceReference.class, new DataSourceReferenceSerializer()); addDeserializer(DataSourceReference.class, new DataSourceReferenceDeserializer()); @@ -159,20 +164,20 @@ public class CoreJacksonModule extends SimpleModule { } } - public static class SecretSerializer extends JsonSerializer { + public static class SecretSerializer extends JsonSerializer { @Override - public void serialize(Secret value, JsonGenerator jgen, SerializerProvider provider) + public void serialize(SecretValue value, JsonGenerator jgen, SerializerProvider provider) throws IOException { jgen.writeString(value.getEncryptedValue()); } } - public static class SecretDeserializer extends JsonDeserializer { + public static class SecretDeserializer extends JsonDeserializer { @Override - public Secret deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return new Secret(p.getValueAsString()); + public SecretValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return new SecretValue(p.getValueAsString()); } } diff --git a/core/src/main/java/io/xpipe/core/util/JacksonHelper.java b/core/src/main/java/io/xpipe/core/util/JacksonHelper.java index b572b434e..0f26f8e41 100644 --- a/core/src/main/java/io/xpipe/core/util/JacksonHelper.java +++ b/core/src/main/java/io/xpipe/core/util/JacksonHelper.java @@ -56,23 +56,12 @@ public class JacksonHelper { * Constructs a new ObjectMapper that is able to map all required X-Pipe classes and also possible extensions. */ public static ObjectMapper newMapper() { - if (!init) { - throw new IllegalStateException("Not initialized"); + if (!JacksonHelper.isInit()) { + JacksonHelper.initModularized(ModuleLayer.boot()); } - return INSTANCE.copy(); } - public static ObjectMapper newMapper(Class excludedModule) { - if (!init) { - throw new IllegalStateException("Not initialized"); - } - - var mapper = BASE.copy(); - mapper.registerModules(MODULES.stream().filter(module -> !module.getClass().equals(excludedModule)).toList()); - return mapper; - } - public static boolean isInit() { return init; } diff --git a/core/src/main/java/io/xpipe/core/util/Secret.java b/core/src/main/java/io/xpipe/core/util/SecretValue.java similarity index 74% rename from core/src/main/java/io/xpipe/core/util/Secret.java rename to core/src/main/java/io/xpipe/core/util/SecretValue.java index 9c69f01fd..193a7530b 100644 --- a/core/src/main/java/io/xpipe/core/util/Secret.java +++ b/core/src/main/java/io/xpipe/core/util/SecretValue.java @@ -8,18 +8,18 @@ import java.util.Base64; @AllArgsConstructor @EqualsAndHashCode -public class Secret { +public class SecretValue { - public static Secret createForSecretValue(String s) { + public static SecretValue createForSecretValue(String s) { if (s == null) { return null; } if (s.length() < 2) { - return new Secret(s); + return new SecretValue(s); } - return new Secret(Base64.getEncoder().encodeToString(s.getBytes(StandardCharsets.UTF_8))); + return new SecretValue(Base64.getEncoder().encodeToString(s.getBytes(StandardCharsets.UTF_8))); } String value; diff --git a/core/src/main/java/io/xpipe/core/util/StreamHelper.java b/core/src/main/java/io/xpipe/core/util/StreamHelper.java deleted file mode 100644 index f142be3c4..000000000 --- a/core/src/main/java/io/xpipe/core/util/StreamHelper.java +++ /dev/null @@ -1,23 +0,0 @@ -package io.xpipe.core.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Objects; - -public class StreamHelper { - - private static final int DEFAULT_BUFFER_SIZE = 8192; - - public static long transferTo(InputStream in, OutputStream out) throws IOException { - Objects.requireNonNull(out, "out"); - long transferred = 0; - byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; - int read; - while (in.available() > 0 && (read = in.read(buffer, 0, DEFAULT_BUFFER_SIZE)) >= 0) { - out.write(buffer, 0, read); - transferred += read; - } - return transferred; - } -} diff --git a/extension/build.gradle b/extension/build.gradle index 29b03526f..8a9bd1454 100644 --- a/extension/build.gradle +++ b/extension/build.gradle @@ -7,7 +7,6 @@ plugins { apply from: "$rootDir/deps/java.gradle" apply from: "$rootDir/deps/javafx.gradle" -apply from: "$rootDir/deps/richtextfx.gradle" apply from: "$rootDir/deps/lombok.gradle" configurations { @@ -26,6 +25,7 @@ dependencies { api group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: "2.13.0" compileOnly group: 'org.kordamp.ikonli', name: 'ikonli-javafx', version: "12.2.0" + compileOnly group: 'org.fxmisc.richtext', name: 'richtextfx', version: '0.10.9' compileOnly 'net.synedra:validatorfx:0.3.1' compileOnly 'org.junit.jupiter:junit-jupiter-api:5.9.0' compileOnly 'com.jfoenix:jfoenix:9.0.10' diff --git a/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java b/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java index fdf22d076..600131ae8 100644 --- a/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java +++ b/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java @@ -6,7 +6,6 @@ import io.xpipe.core.source.DataSourceType; import io.xpipe.core.store.DataStore; import javafx.beans.property.Property; import javafx.scene.layout.Region; -import lombok.SneakyThrows; import java.util.List; import java.util.Map; @@ -23,11 +22,6 @@ public interface DataSourceProvider> { getSourceClass(); } - @SneakyThrows - default T create(Object... arguments) { - return (T) getSourceClass().getDeclaredConstructors()[0].newInstance(arguments); - } - default Category getCategory() { if (getFileProvider() != null) { return Category.FILE; diff --git a/extension/src/main/java/io/xpipe/extension/DataStoreProviders.java b/extension/src/main/java/io/xpipe/extension/DataStoreProviders.java index 06849f2d7..9d1d8f152 100644 --- a/extension/src/main/java/io/xpipe/extension/DataStoreProviders.java +++ b/extension/src/main/java/io/xpipe/extension/DataStoreProviders.java @@ -44,17 +44,17 @@ public class DataStoreProviders { } return ALL.stream().map(d -> { - var store = d.storeForString(s); - if (store != null) { - return d.dialogForStore(store); - } else { - return null; - } - } + var store = d.storeForString(s); + if (store != null) { + return d.dialogForStore(store); + } else { + return null; + } + } ).filter(Objects::nonNull).findAny(); } - + @SuppressWarnings("unchecked") public static T byStore(DataStore store) { return (T) byStoreClass(store.getClass()); } diff --git a/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java b/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java index 8de041390..6b97df3dd 100644 --- a/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java +++ b/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java @@ -12,11 +12,15 @@ import javafx.beans.value.ObservableValue; import javafx.collections.FXCollections; import javafx.scene.control.ComboBox; import javafx.util.StringConverter; -import lombok.Value; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; import java.util.Map; -@Value +@FieldDefaults( + makeFinal = true, + level = AccessLevel.PRIVATE +) public class ChoiceComp extends Comp>> { Property value; diff --git a/extension/src/main/java/io/xpipe/extension/comp/ChoicePaneComp.java b/extension/src/main/java/io/xpipe/extension/comp/ChoicePaneComp.java index 4dbd819d1..5aa5f3f51 100644 --- a/extension/src/main/java/io/xpipe/extension/comp/ChoicePaneComp.java +++ b/extension/src/main/java/io/xpipe/extension/comp/ChoicePaneComp.java @@ -12,12 +12,18 @@ import javafx.scene.control.ComboBox; import javafx.scene.layout.Region; import javafx.scene.layout.VBox; import javafx.util.StringConverter; -import lombok.Value; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.experimental.FieldDefaults; import java.util.List; import java.util.function.Function; -@Value +@FieldDefaults( + makeFinal = true, + level = AccessLevel.PRIVATE +) +@AllArgsConstructor public class ChoicePaneComp extends Comp> { List entries; diff --git a/extension/src/main/java/io/xpipe/extension/comp/IntFieldComp.java b/extension/src/main/java/io/xpipe/extension/comp/IntFieldComp.java index a7e169043..0a4106dcd 100644 --- a/extension/src/main/java/io/xpipe/extension/comp/IntFieldComp.java +++ b/extension/src/main/java/io/xpipe/extension/comp/IntFieldComp.java @@ -8,9 +8,13 @@ import javafx.beans.property.Property; import javafx.beans.value.ChangeListener; import javafx.scene.control.TextField; import javafx.scene.input.KeyEvent; -import lombok.Value; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; -@Value +@FieldDefaults( + makeFinal = true, + level = AccessLevel.PRIVATE +) public class IntFieldComp extends Comp> { Property value; diff --git a/extension/src/main/java/io/xpipe/extension/comp/SecretFieldComp.java b/extension/src/main/java/io/xpipe/extension/comp/SecretFieldComp.java index f301dd6ee..f122af865 100644 --- a/extension/src/main/java/io/xpipe/extension/comp/SecretFieldComp.java +++ b/extension/src/main/java/io/xpipe/extension/comp/SecretFieldComp.java @@ -1,6 +1,6 @@ package io.xpipe.extension.comp; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import io.xpipe.fxcomps.Comp; import io.xpipe.fxcomps.CompStructure; import io.xpipe.fxcomps.SimpleCompStructure; @@ -11,9 +11,9 @@ import javafx.scene.control.TextField; public class SecretFieldComp extends Comp> { - private final Property value; + private final Property value; - public SecretFieldComp(Property value) { + public SecretFieldComp(Property value) { this.value = value; } @@ -22,7 +22,7 @@ public class SecretFieldComp extends Comp> { var text = new PasswordField(); text.setText(value.getValue() != null ? value.getValue().getSecretValue() : null); text.textProperty().addListener((c, o, n) -> { - value.setValue(n != null && n.length() > 0 ? Secret.createForSecretValue(n) : null); + value.setValue(n != null && n.length() > 0 ? SecretValue.createForSecretValue(n) : null); }); value.addListener((c, o, n) -> { PlatformThread.runLaterIfNeeded(() -> { diff --git a/extension/src/main/java/io/xpipe/extension/prefs/PrefsChoiceValue.java b/extension/src/main/java/io/xpipe/extension/prefs/PrefsChoiceValue.java index fbd645c7a..a52428210 100644 --- a/extension/src/main/java/io/xpipe/extension/prefs/PrefsChoiceValue.java +++ b/extension/src/main/java/io/xpipe/extension/prefs/PrefsChoiceValue.java @@ -29,6 +29,7 @@ public interface PrefsChoiceValue extends Translatable { } } + @SuppressWarnings("unchecked") static List getSupported(Class type) { try { return (List) type.getDeclaredField("SUPPORTED").get(null); diff --git a/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java b/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java index 9776ea499..951edec93 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java +++ b/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java @@ -6,7 +6,7 @@ import io.xpipe.core.dialog.Dialog; import io.xpipe.core.dialog.QueryConverter; import io.xpipe.core.source.DataSource; import io.xpipe.core.store.*; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import lombok.Value; import java.util.function.Predicate; @@ -131,7 +131,7 @@ public class DialogHelper { }); } - public static Dialog passwordQuery(Secret password) { + public static Dialog passwordQuery(SecretValue password) { return Dialog.querySecret("Password", false, true, password); } diff --git a/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java b/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java index 59d8fb76e..f9a5d4304 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java +++ b/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java @@ -2,7 +2,7 @@ package io.xpipe.extension.util; import io.xpipe.core.charsetter.NewLine; import io.xpipe.core.charsetter.StreamCharset; -import io.xpipe.core.util.Secret; +import io.xpipe.core.util.SecretValue; import io.xpipe.extension.I18n; import io.xpipe.extension.comp.*; import io.xpipe.fxcomps.Comp; @@ -46,7 +46,7 @@ public class DynamicOptionsBuilder { } public DynamicOptionsBuilder addTitle(ObservableValue title) { - entries.add(new DynamicOptionsComp.Entry(null, Comp.of(() -> new Label(title.getValue())).styleClass("title"))); + entries.add(new DynamicOptionsComp.Entry(null, Comp.of(() -> new Label(title.getValue())).styleClass("title-header"))); return this; } @@ -169,11 +169,11 @@ public class DynamicOptionsBuilder { return this; } - public DynamicOptionsBuilder addSecret(String nameKey, Property prop) { + public DynamicOptionsBuilder addSecret(String nameKey, Property prop) { return addSecret(I18n.observable(nameKey), prop); } - public DynamicOptionsBuilder addSecret(ObservableValue name, Property prop) { + public DynamicOptionsBuilder addSecret(ObservableValue name, Property prop) { var comp = new SecretFieldComp(prop); entries.add(new DynamicOptionsComp.Entry(name, comp)); props.add(prop); @@ -233,7 +233,7 @@ public class DynamicOptionsBuilder { public Comp buildComp() { if (title != null) { - entries.add(0, new DynamicOptionsComp.Entry(null, Comp.of(() -> new Label(title.getValue())).styleClass("title"))); + entries.add(0, new DynamicOptionsComp.Entry(null, Comp.of(() -> new Label(title.getValue())).styleClass("title-header"))); } return new DynamicOptionsComp(entries, wrap); } diff --git a/extension/src/main/java/io/xpipe/extension/util/ExtensionJacksonModule.java b/extension/src/main/java/io/xpipe/extension/util/ExtensionJacksonModule.java deleted file mode 100644 index 0f9f86715..000000000 --- a/extension/src/main/java/io/xpipe/extension/util/ExtensionJacksonModule.java +++ /dev/null @@ -1,54 +0,0 @@ -package io.xpipe.extension.util; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.xpipe.core.source.DataSource; -import io.xpipe.core.util.JacksonHelper; -import io.xpipe.extension.DataSourceProviders; - -import java.io.IOException; - -public class ExtensionJacksonModule extends SimpleModule { - - @Override - public void setupModule(SetupContext context) { - addSerializer(DataSource.class, new DataSourceSerializer()); - addDeserializer(DataSource.class, new DataSourceDeserializer()); - - context.addSerializers(_serializers); - context.addDeserializers(_deserializers); - } - - public static class DataSourceSerializer extends JsonSerializer { - - @Override - public void serialize(DataSource value, JsonGenerator jgen, SerializerProvider provider) - throws IOException { - - ObjectMapper mapper = JacksonHelper.newMapper(ExtensionJacksonModule.class); - var prov = DataSourceProviders.byDataSourceClass(value.getClass()); - ObjectNode objectNode = mapper.valueToTree(value); - objectNode.put("type", prov.getId()); - jgen.writeTree(objectNode); - } - } - - public static class DataSourceDeserializer extends JsonDeserializer { - - @Override - public DataSource deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - var mapper = JacksonHelper.newMapper(ExtensionJacksonModule.class); - var tree = (ObjectNode) mapper.readTree(p); - var type = tree.get("type").textValue(); - var prov = DataSourceProviders.byName(type); - if (prov.isEmpty()) { - return null; - } - - return mapper.treeToValue(tree, prov.get().getSourceClass()); - } - } -} diff --git a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java index cba1a02b9..f49841dfb 100644 --- a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java @@ -22,10 +22,17 @@ public class ExtensionTest { return FileStore.local(Path.of(file)); } + public static DataSource getSource(String type, DataStore store) { + return DataSource.create(null, type, store); + } + public static DataSource getSource(String type, String file) { return DataSource.create(null, type, getResource(file)); } + public static DataSource getSource(io.xpipe.core.source.DataSource source) { + return DataSource.create(null, source); + } @BeforeAll public static void setup() throws Exception { diff --git a/extension/src/main/java/io/xpipe/extension/util/TypeConverter.java b/extension/src/main/java/io/xpipe/extension/util/TypeConverter.java index ebfc7d0d3..449db178f 100644 --- a/extension/src/main/java/io/xpipe/extension/util/TypeConverter.java +++ b/extension/src/main/java/io/xpipe/extension/util/TypeConverter.java @@ -4,12 +4,15 @@ import io.xpipe.core.data.node.DataStructureNode; import io.xpipe.core.data.node.ValueNode; import org.apache.commons.lang3.math.NumberUtils; +import java.math.BigDecimal; +import java.math.BigInteger; + public class TypeConverter { public static void tagNullType(ValueNode node, String nullValue) { var string = node.asString(); if (string.equals(nullValue)) { - node.tag(DataStructureNode.NULL_VALUE); + node.tag(DataStructureNode.IS_NULL); } } @@ -17,18 +20,17 @@ public class TypeConverter { var string = node.asString(); if (string.equals(trueValue)) { node.tag(DataStructureNode.BOOLEAN_TRUE); - node.tag(DataStructureNode.BOOLEAN_VALUE); + node.tag(DataStructureNode.IS_BOOLEAN); } if (string.equals(falseValue)) { node.tag(DataStructureNode.BOOLEAN_FALSE); - node.tag(DataStructureNode.BOOLEAN_VALUE); + node.tag(DataStructureNode.IS_BOOLEAN); } } public static void tagNumberType(ValueNode node) { var string = node.asString(); if (NumberUtils.isCreatable(string)) { - node.tag(DataStructureNode.IS_NUMBER); var number = NumberUtils.createNumber(string); if (number instanceof Float || number instanceof Double) { node.tag(DataStructureNode.IS_FLOATING_POINT); @@ -37,4 +39,93 @@ public class TypeConverter { } } } + + public static BigDecimal parseDecimal(DataStructureNode node) { + if (node == null || node.hasMetaAttribute(DataStructureNode.IS_NULL)) { + return BigDecimal.ZERO; + } + + if (node.hasMetaAttribute(DataStructureNode.FLOATING_POINT_VALUE)) { + return new BigDecimal(node.getMetaAttribute(DataStructureNode.FLOATING_POINT_VALUE)); + } + + var parsedDecimal = parseDecimal(node.asString()); + if (parsedDecimal != null) { + return parsedDecimal; + } + + if (node.hasMetaAttribute(DataStructureNode.INTEGER_VALUE)) { + return new BigDecimal(node.getMetaAttribute(DataStructureNode.INTEGER_VALUE)); + } + + var parsedInteger = parseInteger(node.asString()); + if (parsedInteger != null) { + return new BigDecimal(parsedInteger); + } + + return null; + } + + public static Boolean parseBoolean(DataStructureNode node) { + if (node == null || node.hasMetaAttribute(DataStructureNode.IS_NULL)) { + return false; + } + + if (node.hasMetaAttribute(DataStructureNode.BOOLEAN_FALSE)) { + return Boolean.parseBoolean(node.getMetaAttribute(DataStructureNode.BOOLEAN_FALSE)); + } + + if (node.hasMetaAttribute(DataStructureNode.BOOLEAN_TRUE)) { + return Boolean.parseBoolean(node.getMetaAttribute(DataStructureNode.BOOLEAN_TRUE)); + } + + var string = node.asString(); + if (string.length() == 0 || string.equalsIgnoreCase("false")) { + return false; + } + + return true; + } + + public static BigInteger parseInteger(DataStructureNode node) { + if (node == null || node.hasMetaAttribute(DataStructureNode.IS_NULL)) { + return BigInteger.ZERO; + } + + if (node.hasMetaAttribute(DataStructureNode.INTEGER_VALUE)) { + return new BigInteger(node.getMetaAttribute(DataStructureNode.INTEGER_VALUE)); + } + + var parsedInteger = parseInteger(node.asString()); + if (parsedInteger != null) { + return parsedInteger; + } + + if (node.hasMetaAttribute(DataStructureNode.FLOATING_POINT_VALUE)) { + return new BigDecimal(node.getMetaAttribute(DataStructureNode.FLOATING_POINT_VALUE)).toBigInteger(); + } + + var parsedDecimal = parseDecimal(node.asString()); + if (parsedDecimal != null) { + return parsedDecimal.toBigInteger(); + } + + return null; + } + + private static BigInteger parseInteger(String string) { + if (string == null) { + return BigInteger.ZERO; + } + + return NumberUtils.createBigInteger(string); + } + + private static BigDecimal parseDecimal(String string) { + if (string == null) { + return BigDecimal.ZERO; + } + + return NumberUtils.createBigDecimal(string); + } } diff --git a/extension/src/main/java/module-info.java b/extension/src/main/java/module-info.java index 65bd3eef5..091c4d44e 100644 --- a/extension/src/main/java/module-info.java +++ b/extension/src/main/java/module-info.java @@ -1,7 +1,5 @@ -import com.fasterxml.jackson.databind.Module; import io.xpipe.extension.DataSourceProvider; import io.xpipe.extension.SupportedApplicationProvider; -import io.xpipe.extension.util.ExtensionJacksonModule; import io.xpipe.extension.util.XPipeDaemon; open module io.xpipe.extension { @@ -27,9 +25,6 @@ open module io.xpipe.extension { requires static org.fxmisc.richtext; requires static net.synedra.validatorfx; requires static org.fxmisc.flowless; - requires static org.fxmisc.undofx; - requires static org.fxmisc.wellbehavedfx; - requires static org.reactfx; requires static org.kordamp.ikonli.javafx; requires static com.jfoenix; @@ -41,6 +36,4 @@ open module io.xpipe.extension { uses io.xpipe.extension.DataStoreProvider; uses XPipeDaemon; uses io.xpipe.extension.Cache; - - provides Module with ExtensionJacksonModule; } \ No newline at end of file