From 85389d26f91f43f4c6187574ba3aca407104cd3c Mon Sep 17 00:00:00 2001 From: Christopher Schnick Date: Wed, 16 Nov 2022 23:40:59 +0100 Subject: [PATCH] Rework shell stores, other various small changes --- api/src/main/java/io/xpipe/api/DataRaw.java | 4 - .../main/java/io/xpipe/api/DataStructure.java | 4 - api/src/main/java/io/xpipe/api/DataTable.java | 3 - api/src/main/java/io/xpipe/api/DataText.java | 4 - .../java/io/xpipe/api/impl/DataRawImpl.java | 10 - .../io/xpipe/api/impl/DataSourceImpl.java | 18 +- .../io/xpipe/api/impl/DataStructureImpl.java | 12 +- .../java/io/xpipe/api/impl/DataTableImpl.java | 19 +- .../java/io/xpipe/api/impl/DataTextImpl.java | 12 +- .../exchange/QueryDataSourceExchange.java | 13 +- .../io/xpipe/core/charsetter/Charsetter.java | 8 +- .../java/io/xpipe/core/impl/BinarySource.java | 2 + .../java/io/xpipe/core/source/DataSource.java | 10 +- .../core/store/CommandProcessControl.java | 122 +++++++++ .../io/xpipe/core/store/CommandsStore.java | 6 + .../java/io/xpipe/core/store/FileStore.java | 18 +- ...ineFileStore.java => FileSystemStore.java} | 8 +- .../java/io/xpipe/core/store/LocalStore.java | 120 ++------- .../io/xpipe/core/store/MachineStore.java | 47 ++++ .../io/xpipe/core/store/ProcessControl.java | 147 +++------- .../core/store/ProcessControlProvider.java | 6 + .../xpipe/core/store/ShellProcessControl.java | 58 ++++ .../java/io/xpipe/core/store/ShellStore.java | 41 +-- .../java/io/xpipe/core/store/ShellType.java | 55 ++++ .../java/io/xpipe/core/store/ShellTypes.java | 252 ++++++++++++------ .../xpipe/core/store/StandardShellStore.java | 97 ------- core/src/main/java/module-info.java | 1 + .../xpipe/extension/DataSourceProvider.java | 7 +- .../io/xpipe/extension/DataStoreProvider.java | 4 +- .../extension/XPipeServiceProviders.java | 3 + .../io/xpipe/extension/comp/ChoiceComp.java | 4 +- .../xpipe/extension/event/EventHandler.java | 2 +- .../extension/util/DaemonExtensionTest.java | 18 ++ .../io/xpipe/extension/util/DialogHelper.java | 2 +- .../extension/util/DynamicOptionsBuilder.java | 1 - .../xpipe/extension/util/ExtensionTest.java | 11 +- .../extension/util/LocalExtensionTest.java | 4 + .../io/xpipe/extension/util/OsHelper.java | 6 +- .../io/xpipe/extension/util/ThreadHelper.java | 18 +- .../resources/lang/translations_en.properties | 2 +- 40 files changed, 626 insertions(+), 553 deletions(-) create mode 100644 core/src/main/java/io/xpipe/core/store/CommandProcessControl.java create mode 100644 core/src/main/java/io/xpipe/core/store/CommandsStore.java rename core/src/main/java/io/xpipe/core/store/{MachineFileStore.java => FileSystemStore.java} (58%) create mode 100644 core/src/main/java/io/xpipe/core/store/MachineStore.java create mode 100644 core/src/main/java/io/xpipe/core/store/ProcessControlProvider.java create mode 100644 core/src/main/java/io/xpipe/core/store/ShellProcessControl.java create mode 100644 core/src/main/java/io/xpipe/core/store/ShellType.java delete mode 100644 core/src/main/java/io/xpipe/core/store/StandardShellStore.java create mode 100644 extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java create mode 100644 extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java diff --git a/api/src/main/java/io/xpipe/api/DataRaw.java b/api/src/main/java/io/xpipe/api/DataRaw.java index fd6768596..a8eaf11e0 100644 --- a/api/src/main/java/io/xpipe/api/DataRaw.java +++ b/api/src/main/java/io/xpipe/api/DataRaw.java @@ -1,13 +1,9 @@ package io.xpipe.api; -import io.xpipe.core.source.DataSourceInfo; - import java.io.InputStream; public interface DataRaw extends DataSource { - DataSourceInfo.Raw getInfo(); - InputStream open(); byte[] readAll(); diff --git a/api/src/main/java/io/xpipe/api/DataStructure.java b/api/src/main/java/io/xpipe/api/DataStructure.java index c4b8d8dcc..6f43874d3 100644 --- a/api/src/main/java/io/xpipe/api/DataStructure.java +++ b/api/src/main/java/io/xpipe/api/DataStructure.java @@ -1,11 +1,7 @@ package io.xpipe.api; import io.xpipe.core.data.node.DataStructureNode; -import io.xpipe.core.source.DataSourceInfo; public interface DataStructure extends DataSource { - - DataSourceInfo.Structure getInfo(); - DataStructureNode read(); } diff --git a/api/src/main/java/io/xpipe/api/DataTable.java b/api/src/main/java/io/xpipe/api/DataTable.java index 43a99b106..1fa4a761e 100644 --- a/api/src/main/java/io/xpipe/api/DataTable.java +++ b/api/src/main/java/io/xpipe/api/DataTable.java @@ -2,15 +2,12 @@ package io.xpipe.api; import io.xpipe.core.data.node.ArrayNode; import io.xpipe.core.data.node.TupleNode; -import io.xpipe.core.source.DataSourceInfo; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; public interface DataTable extends Iterable, DataSource { - DataSourceInfo.Table getInfo(); - Stream stream(); ArrayNode readAll(); diff --git a/api/src/main/java/io/xpipe/api/DataText.java b/api/src/main/java/io/xpipe/api/DataText.java index a97e104a2..864a34d97 100644 --- a/api/src/main/java/io/xpipe/api/DataText.java +++ b/api/src/main/java/io/xpipe/api/DataText.java @@ -1,14 +1,10 @@ package io.xpipe.api; -import io.xpipe.core.source.DataSourceInfo; - import java.util.List; import java.util.stream.Stream; public interface DataText extends DataSource { - DataSourceInfo.Text getInfo(); - List readAllLines(); List readLines(int maxLines); diff --git a/api/src/main/java/io/xpipe/api/impl/DataRawImpl.java b/api/src/main/java/io/xpipe/api/impl/DataRawImpl.java index be07fc394..c5883901e 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataRawImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataRawImpl.java @@ -3,27 +3,17 @@ package io.xpipe.api.impl; import io.xpipe.api.DataRaw; import io.xpipe.api.DataSourceConfig; import io.xpipe.core.source.DataSourceId; -import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceType; import java.io.InputStream; public class DataRawImpl extends DataSourceImpl implements DataRaw { - private final DataSourceInfo.Raw info; - public DataRawImpl( DataSourceId sourceId, DataSourceConfig sourceConfig, - DataSourceInfo.Raw info, io.xpipe.core.source.DataSource internalSource) { super(sourceId, sourceConfig, internalSource); - this.info = info; - } - - @Override - public DataSourceInfo.Raw getInfo() { - return info; } @Override diff --git a/api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java b/api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java index 265f318c5..42a640a38 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataSourceImpl.java @@ -29,27 +29,23 @@ public abstract class DataSourceImpl implements DataSource { var req = QueryDataSourceExchange.Request.builder().ref(ds).build(); QueryDataSourceExchange.Response res = con.performSimpleExchange(req); var config = new DataSourceConfig(res.getProvider(), res.getConfig()); - return switch (res.getInfo().getType()) { + return switch (res.getType()) { case TABLE -> { - var data = res.getInfo().asTable(); - yield new DataTableImpl(res.getId(), config, data, res.getInternalSource()); + yield new DataTableImpl(res.getId(), config, res.getInternalSource()); } case STRUCTURE -> { - var info = res.getInfo().asStructure(); - yield new DataStructureImpl(res.getId(), config, info, res.getInternalSource()); + yield new DataStructureImpl(res.getId(), config, res.getInternalSource()); } case TEXT -> { - var info = res.getInfo().asText(); - yield new DataTextImpl(res.getId(), config, info, res.getInternalSource()); + yield new DataTextImpl(res.getId(), config, res.getInternalSource()); } case RAW -> { - var info = res.getInfo().asRaw(); - yield new DataRawImpl(res.getId(), config, info, res.getInternalSource()); + yield new DataRawImpl(res.getId(), config, res.getInternalSource()); } case COLLECTION -> throw new UnsupportedOperationException( - "Unimplemented case: " + res.getInfo().getType()); + "Unimplemented case: " + res.getType()); default -> throw new IllegalArgumentException( - "Unexpected value: " + res.getInfo().getType()); + "Unexpected value: " + res.getType()); }; }); } diff --git a/api/src/main/java/io/xpipe/api/impl/DataStructureImpl.java b/api/src/main/java/io/xpipe/api/impl/DataStructureImpl.java index 2f6202a03..1ffc89029 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataStructureImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataStructureImpl.java @@ -4,20 +4,15 @@ import io.xpipe.api.DataSourceConfig; import io.xpipe.api.DataStructure; import io.xpipe.core.data.node.DataStructureNode; import io.xpipe.core.source.DataSourceId; -import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceType; public class DataStructureImpl extends DataSourceImpl implements DataStructure { - private final DataSourceInfo.Structure info; - - public DataStructureImpl( + DataStructureImpl( DataSourceId sourceId, DataSourceConfig sourceConfig, - DataSourceInfo.Structure info, io.xpipe.core.source.DataSource internalSource) { super(sourceId, sourceConfig, internalSource); - this.info = info; } @Override @@ -30,11 +25,6 @@ public class DataStructureImpl extends DataSourceImpl implements DataStructure { return this; } - @Override - public DataSourceInfo.Structure getInfo() { - return info; - } - @Override public DataStructureNode read() { return null; diff --git a/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java b/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java index 6538f947d..5c8b3a4c6 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataTableImpl.java @@ -13,7 +13,6 @@ import io.xpipe.core.data.typed.TypedAbstractReader; import io.xpipe.core.data.typed.TypedDataStreamParser; import io.xpipe.core.data.typed.TypedDataStructureNodeReader; import io.xpipe.core.source.DataSourceId; -import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceReference; import io.xpipe.core.source.DataSourceType; @@ -25,15 +24,11 @@ import java.util.stream.StreamSupport; public class DataTableImpl extends DataSourceImpl implements DataTable { - private final DataSourceInfo.Table info; - DataTableImpl( DataSourceId id, DataSourceConfig sourceConfig, - DataSourceInfo.Table info, io.xpipe.core.source.DataSource internalSource) { super(id, sourceConfig, internalSource); - this.info = info; } @Override @@ -41,11 +36,6 @@ public class DataTableImpl extends DataSourceImpl implements DataTable { return this; } - @Override - public DataSourceInfo.Table getInfo() { - return info; - } - public Stream stream() { var iterator = new TableIterator(); return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) @@ -93,16 +83,17 @@ public class DataTableImpl extends DataSourceImpl implements DataTable { private TupleNode node; { - nodeReader = TypedDataStructureNodeReader.of(info.getDataType()); - parser = new TypedDataStreamParser(info.getDataType()); - connection = XPipeConnection.open(); var req = QueryTableDataExchange.Request.builder() .ref(DataSourceReference.id(getId())) .maxRows(Integer.MAX_VALUE) .build(); connection.sendRequest(req); - connection.receiveResponse(); + QueryTableDataExchange.Response response = connection.receiveResponse(); + + nodeReader = TypedDataStructureNodeReader.of(response.getDataType()); + parser = new TypedDataStreamParser(response.getDataType()); + connection.receiveBody(); } diff --git a/api/src/main/java/io/xpipe/api/impl/DataTextImpl.java b/api/src/main/java/io/xpipe/api/impl/DataTextImpl.java index 4d5f27c4c..7d553cce4 100644 --- a/api/src/main/java/io/xpipe/api/impl/DataTextImpl.java +++ b/api/src/main/java/io/xpipe/api/impl/DataTextImpl.java @@ -7,7 +7,6 @@ import io.xpipe.beacon.BeaconConnection; import io.xpipe.beacon.BeaconException; import io.xpipe.beacon.exchange.api.QueryTextDataExchange; import io.xpipe.core.source.DataSourceId; -import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceReference; import io.xpipe.core.source.DataSourceType; @@ -25,15 +24,11 @@ import java.util.stream.StreamSupport; public class DataTextImpl extends DataSourceImpl implements DataText { - private final DataSourceInfo.Text info; - - public DataTextImpl( + DataTextImpl( DataSourceId sourceId, DataSourceConfig sourceConfig, - DataSourceInfo.Text info, io.xpipe.core.source.DataSource internalSource) { super(sourceId, sourceConfig, internalSource); - this.info = info; } @Override @@ -46,11 +41,6 @@ public class DataTextImpl extends DataSourceImpl implements DataText { return this; } - @Override - public DataSourceInfo.Text getInfo() { - return info; - } - @Override public List readAllLines() { return readLines(Integer.MAX_VALUE); diff --git a/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java b/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java index 3505b3c26..0ead9871c 100644 --- a/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java +++ b/beacon/src/main/java/io/xpipe/beacon/exchange/QueryDataSourceExchange.java @@ -4,8 +4,8 @@ import io.xpipe.beacon.RequestMessage; import io.xpipe.beacon.ResponseMessage; import io.xpipe.core.source.DataSource; import io.xpipe.core.source.DataSourceId; -import io.xpipe.core.source.DataSourceInfo; import io.xpipe.core.source.DataSourceReference; +import io.xpipe.core.source.DataSourceType; import lombok.Builder; import lombok.NonNull; import lombok.Value; @@ -38,17 +38,14 @@ public class QueryDataSourceExchange implements MessageExchange { @NonNull DataSourceId id; - boolean disabled; - boolean hidden; + @NonNull + String information; @NonNull - DataSourceInfo info; - - @NonNull - String storeDisplay; - String provider; + @NonNull DataSourceType type; + @NonNull LinkedHashMap config; diff --git a/core/src/main/java/io/xpipe/core/charsetter/Charsetter.java b/core/src/main/java/io/xpipe/core/charsetter/Charsetter.java index ea0e86bbd..573f0d8c5 100644 --- a/core/src/main/java/io/xpipe/core/charsetter/Charsetter.java +++ b/core/src/main/java/io/xpipe/core/charsetter/Charsetter.java @@ -1,6 +1,7 @@ package io.xpipe.core.charsetter; import io.xpipe.core.store.FileStore; +import io.xpipe.core.store.MachineStore; import io.xpipe.core.store.StreamDataStore; import lombok.Value; @@ -107,10 +108,11 @@ public abstract class Charsetter { } } - if (store instanceof FileStore fileStore) { - var newline = fileStore.getMachine().getNewLine(); + if (store instanceof FileStore fileStore && fileStore.getFileSystem() instanceof MachineStore m) { if (result.getNewLine() == null) { - result = new Result(result.getCharset(), newline); + try (var pc = m.create().start()) { + result = new Result(result.getCharset(), pc.getShellType().getNewLine()); + } } } diff --git a/core/src/main/java/io/xpipe/core/impl/BinarySource.java b/core/src/main/java/io/xpipe/core/impl/BinarySource.java index 92b28e975..970e7bded 100644 --- a/core/src/main/java/io/xpipe/core/impl/BinarySource.java +++ b/core/src/main/java/io/xpipe/core/impl/BinarySource.java @@ -7,12 +7,14 @@ import io.xpipe.core.source.RawWriteConnection; import io.xpipe.core.source.WriteMode; import io.xpipe.core.store.StreamDataStore; import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; import java.io.InputStream; import java.io.OutputStream; @JsonTypeName("binary") @SuperBuilder +@Jacksonized public class BinarySource extends RawDataSource { @Override diff --git a/core/src/main/java/io/xpipe/core/source/DataSource.java b/core/src/main/java/io/xpipe/core/source/DataSource.java index a12f63ae6..60abd9089 100644 --- a/core/src/main/java/io/xpipe/core/source/DataSource.java +++ b/core/src/main/java/io/xpipe/core/source/DataSource.java @@ -46,8 +46,14 @@ public abstract class DataSource extends JacksonizedValue throw new AssertionError(ex); } } - public void test() throws Exception { - store.validate(); + + public boolean isComplete() { + try { + checkComplete(); + return true; + } catch (Exception ignored) { + return false; + } } public void checkComplete() throws Exception { diff --git a/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java b/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java new file mode 100644 index 000000000..e396991d4 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/CommandProcessControl.java @@ -0,0 +1,122 @@ +package io.xpipe.core.store; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +public interface CommandProcessControl extends ProcessControl { + + default InputStream startExternalStdout() throws Exception { + start(); + discardErr(); + return new FilterInputStream(getStdout()) { + @Override + public void close() throws IOException { + getStdout().close(); + CommandProcessControl.this.close(); + } + }; + } + + default OutputStream startExternalStdin() throws Exception { + try (CommandProcessControl pc = start()) { + pc.discardOut(); + pc.discardErr(); + return new FilterOutputStream(getStdin()) { + @Override + public void close() throws IOException { + pc.getStdin().close(); + pc.close(); + } + }; + } catch (Exception e) { + throw e; + } + } + + CommandProcessControl customCharset(Charset charset); + + int getExitCode(); + + CommandProcessControl elevated(); + + @Override + CommandProcessControl start() throws Exception; + + @Override + CommandProcessControl exitTimeout(int timeout); + + String readOnlyStdout() throws Exception; + + public default void discardOrThrow() throws Exception { + readOrThrow(); + } + + public default boolean startAndCheckExit() { + try (var pc = start()) { + return pc.discardAndCheckExit(); + } catch (Exception e) { + return false; + } + } + + public default boolean discardAndCheckExit() { + try { + discardOrThrow(); + return true; + } catch (Exception ex) { + return false; + } + } + + public default Optional readStderrIfPresent() throws Exception { + discardOut(); + var bytes = getStderr().readAllBytes(); + var string = new String(bytes, getCharset()); + var ec = waitFor(); + return ec ? Optional.of(string) : Optional.empty(); + } + + public default String readOrThrow() throws Exception { + AtomicReference readError = new AtomicReference<>(""); + var errorThread = new Thread(() -> { + try { + + readError.set(new String(getStderr().readAllBytes(), getCharset())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + errorThread.setDaemon(true); + errorThread.start(); + + AtomicReference read = new AtomicReference<>(""); + var t = new Thread(() -> { + try { + read.set(readLine()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + t.setDaemon(true); + t.start(); + + var ec = waitFor(); + if (!ec) { + throw new ProcessOutputException("Command timed out"); + } + + var exitCode = getExitCode(); + if (exitCode == 0 && !(read.get().isEmpty() && !readError.get().isEmpty())) { + return read.get().trim(); + } else { + throw new ProcessOutputException( + "Command returned with " + ec + ": " + readError.get().trim()); + } + } + + Thread discardOut(); + + Thread discardErr(); +} diff --git a/core/src/main/java/io/xpipe/core/store/CommandsStore.java b/core/src/main/java/io/xpipe/core/store/CommandsStore.java new file mode 100644 index 000000000..0911c16b8 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/CommandsStore.java @@ -0,0 +1,6 @@ +package io.xpipe.core.store; + +public interface CommandsStore extends DataStore { + + CommandProcessControl create() throws Exception; +} diff --git a/core/src/main/java/io/xpipe/core/store/FileStore.java b/core/src/main/java/io/xpipe/core/store/FileStore.java index b49ccfc53..6200dffb2 100644 --- a/core/src/main/java/io/xpipe/core/store/FileStore.java +++ b/core/src/main/java/io/xpipe/core/store/FileStore.java @@ -11,7 +11,7 @@ import java.io.OutputStream; import java.nio.file.Path; /** - * Represents a file located on a certain machine. + * Represents a file located on a file system. */ @JsonTypeName("file") @SuperBuilder @@ -19,16 +19,16 @@ import java.nio.file.Path; @Getter public class FileStore extends JacksonizedValue implements FilenameStore, StreamDataStore { - MachineFileStore machine; + FileSystemStore fileSystem; String file; - public FileStore(MachineFileStore machine, String file) { - this.machine = machine; + public FileStore(FileSystemStore fileSystem, String file) { + this.fileSystem = fileSystem; this.file = file; } public final boolean isLocal() { - return machine instanceof LocalStore; + return fileSystem instanceof LocalStore; } public static FileStore local(Path p) { @@ -44,7 +44,7 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream @Override public void checkComplete() throws Exception { - if (machine == null) { + if (fileSystem == null) { throw new IllegalStateException("Machine is missing"); } if (file == null) { @@ -54,17 +54,17 @@ public class FileStore extends JacksonizedValue implements FilenameStore, Stream @Override public InputStream openInput() throws Exception { - return machine.openInput(file); + return fileSystem.openInput(file); } @Override public OutputStream openOutput() throws Exception { - return machine.openOutput(file); + return fileSystem.openOutput(file); } @Override public boolean canOpen() throws Exception { - return machine.exists(file); + return fileSystem.exists(file); } @Override diff --git a/core/src/main/java/io/xpipe/core/store/MachineFileStore.java b/core/src/main/java/io/xpipe/core/store/FileSystemStore.java similarity index 58% rename from core/src/main/java/io/xpipe/core/store/MachineFileStore.java rename to core/src/main/java/io/xpipe/core/store/FileSystemStore.java index b36c40a89..db133990a 100644 --- a/core/src/main/java/io/xpipe/core/store/MachineFileStore.java +++ b/core/src/main/java/io/xpipe/core/store/FileSystemStore.java @@ -1,11 +1,9 @@ package io.xpipe.core.store; -import io.xpipe.core.charsetter.NewLine; - import java.io.InputStream; import java.io.OutputStream; -public interface MachineFileStore extends DataStore { +public interface FileSystemStore extends DataStore { InputStream openInput(String file) throws Exception; @@ -13,7 +11,5 @@ public interface MachineFileStore extends DataStore { public boolean exists(String file) throws Exception; - void mkdirs(String file) throws Exception; - - NewLine getNewLine() throws Exception; + boolean mkdirs(String file) throws Exception; } diff --git a/core/src/main/java/io/xpipe/core/store/LocalStore.java b/core/src/main/java/io/xpipe/core/store/LocalStore.java index 5f3f9270b..a2b73c4f8 100644 --- a/core/src/main/java/io/xpipe/core/store/LocalStore.java +++ b/core/src/main/java/io/xpipe/core/store/LocalStore.java @@ -1,21 +1,16 @@ package io.xpipe.core.store; import com.fasterxml.jackson.annotation.JsonTypeName; -import io.xpipe.core.charsetter.NewLine; import io.xpipe.core.util.JacksonizedValue; -import io.xpipe.core.util.SecretValue; -import lombok.Getter; -import java.io.*; -import java.nio.charset.Charset; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.ServiceLoader; @JsonTypeName("local") -public class LocalStore extends JacksonizedValue implements MachineFileStore, StandardShellStore { +public class LocalStore extends JacksonizedValue implements FileSystemStore, MachineStore { @Override public boolean isLocal() { @@ -28,13 +23,13 @@ public class LocalStore extends JacksonizedValue implements MachineFileStore, St } @Override - public void mkdirs(String file) throws Exception { - Files.createDirectories(Path.of(file).getParent()); - } - - @Override - public NewLine getNewLine() { - return ShellTypes.getPlatformDefault().getNewLine(); + public boolean mkdirs(String file) throws Exception { + try { + Files.createDirectories(Path.of(file).getParent()); + return true; + } catch (Exception ex) { + return false; + } } @Override @@ -49,98 +44,23 @@ public class LocalStore extends JacksonizedValue implements MachineFileStore, St var p = Path.of(file); return Files.newOutputStream(p); } - @Override - public ProcessControl prepareCommand(List input, List cmd, Integer timeout, Charset charset) { - return new LocalProcessControl(input, cmd, getEffectiveTimeOut(timeout), charset); + public ShellProcessControl create() { + return LocalProcessControlProvider.create(); } - @Override - public ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeOut, Charset charset) - throws Exception { - return new LocalProcessControl(input, cmd, getEffectiveTimeOut(timeOut), charset); - } + public static abstract class LocalProcessControlProvider { - @Override - public ShellType determineType() throws Exception { - return ShellTypes.getPlatformDefault(); - } + private static LocalProcessControlProvider INSTANCE; - class LocalProcessControl extends ProcessControl { - - private final List input; - private final Integer timeout; - - @Getter - private final List command; - private final Charset charset; - - private Process process; - - LocalProcessControl(List input, List cmd, Integer timeout, Charset charset) { - this.input = input; - this.timeout = timeout; - this.command = cmd; - this.charset = charset; + public static void init(ModuleLayer layer) { + INSTANCE = ServiceLoader.load(layer, LocalProcessControlProvider.class).findFirst().orElseThrow(); } - private InputStream createInputStream() { - var string = - input.stream().map(secret -> secret.getSecretValue()).collect(Collectors.joining("\n")) + "\r\n"; - return new ByteArrayInputStream(string.getBytes(charset)); + public static ShellProcessControl create() { + return INSTANCE.createProcessControl(); } - @Override - public void start() throws Exception { - var type = LocalStore.this.determineType(); - var l = type.switchTo(command); - var builder = new ProcessBuilder(l); - process = builder.start(); - - var t = new Thread(() -> { - try (var inputStream = createInputStream()) { - process.getOutputStream().flush(); - inputStream.transferTo(process.getOutputStream()); - process.getOutputStream().close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - t.setDaemon(true); - //t.start(); - } - - @Override - public int waitFor() throws Exception { - if (timeout != null) { - return process.waitFor(timeout, TimeUnit.SECONDS) ? 0 : -1; - } else { - return process.waitFor(); - } - } - - @Override - public InputStream getStdout() { - return process.getInputStream(); - } - - @Override - public OutputStream getStdin() { - return process.getOutputStream(); - } - - @Override - public InputStream getStderr() { - return process.getErrorStream(); - } - - @Override - public Charset getCharset() { - return charset; - } - - public Integer getTimeout() { - return timeout; - } + public abstract ShellProcessControl createProcessControl(); } } diff --git a/core/src/main/java/io/xpipe/core/store/MachineStore.java b/core/src/main/java/io/xpipe/core/store/MachineStore.java new file mode 100644 index 000000000..34adcf791 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/MachineStore.java @@ -0,0 +1,47 @@ +package io.xpipe.core.store; + +import java.io.InputStream; +import java.io.OutputStream; + +public interface MachineStore extends FileSystemStore, ShellStore { + + public default boolean isLocal() { + return false; + } + + public default String queryMachineName() throws Exception { + try (CommandProcessControl pc = create().commandListFunction(shellProcessControl -> + shellProcessControl.getShellType().getOperatingSystemNameCommand()) + .start()) { + return pc.readOrThrow().trim(); + } + } + + @Override + public default InputStream openInput(String file) throws Exception { + return create().commandListFunction(proc -> proc.getShellType().createFileReadCommand(file)) + .startExternalStdout(); + } + + @Override + public default OutputStream openOutput(String file) throws Exception { + return create().commandListFunction(proc -> proc.getShellType().createFileWriteCommand(file)) + .startExternalStdin(); + } + + @Override + public default boolean exists(String file) throws Exception { + var r = create().commandListFunction(proc -> proc.getShellType().createFileExistsCommand(file)) + .start() + .discardAndCheckExit(); + return r; + } + + @Override + public default boolean mkdirs(String file) throws Exception { + var r = create().commandListFunction(proc -> proc.getShellType().createMkdirsCommand(file)) + .start() + .discardAndCheckExit(); + return r; + } +} diff --git a/core/src/main/java/io/xpipe/core/store/ProcessControl.java b/core/src/main/java/io/xpipe/core/store/ProcessControl.java index 2caeaa3e0..aaea19571 100644 --- a/core/src/main/java/io/xpipe/core/store/ProcessControl.java +++ b/core/src/main/java/io/xpipe/core/store/ProcessControl.java @@ -3,129 +3,60 @@ package io.xpipe.core.store; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.nio.charset.Charset; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; +import java.util.UUID; -public abstract class ProcessControl { +public interface ProcessControl extends AutoCloseable { - public String executeAndReadStdout() throws Exception { - var pc = this; - pc.start(); - pc.discardErr(); - var bytes = pc.getStdout().readAllBytes(); - var string = new String(bytes, pc.getCharset()); - pc.waitFor(); - return string; - } + boolean isRunning(); - public void executeOrThrow() throws Exception { - var pc = this; - pc.start(); - pc.discardOut(); - pc.discardErr(); - pc.waitFor(); - } + ShellType getShellType(); - public boolean executeAndCheckStatus() { - try { - executeOrThrow(); - return true; - } catch (Exception ex) { - return false; + String readResultLine(String input, boolean captureOutput) throws IOException; + + void writeLine(String line) throws IOException; + + void writeLine(String line, boolean captureOutput) throws IOException; + + void typeLine(String line); + + public default String readOutput() throws IOException { + var id = UUID.randomUUID(); + writeLine("echo " + id, false); + String lines = ""; + while (true) { + var newLine = readLine(); + if (newLine.contains(id.toString())) { + if (getShellType().echoesInput()) { + readLine(); + } + + break; + } + + lines = lines + newLine + "\n"; } + return lines; } - public Optional executeAndReadStderrIfPresent() throws Exception { - var pc = this; - pc.start(); - pc.discardOut(); - var bytes = pc.getStderr().readAllBytes(); - var string = new String(bytes, pc.getCharset()); - var ec = pc.waitFor(); - return ec != 0 ? Optional.of(string) : Optional.empty(); - } + @Override + void close() throws IOException; - public String executeAndReadStdoutOrThrow() - throws Exception { - var pc = this; - pc.start(); + String readLine() throws IOException; - AtomicReference readError = new AtomicReference<>(""); - var errorThread = new Thread(() -> { - try { + void kill() throws IOException; - readError.set(new String(pc.getStderr().readAllBytes(), pc.getCharset())); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - errorThread.setDaemon(true); - errorThread.start(); + ProcessControl exitTimeout(int timeout); - AtomicReference read = new AtomicReference<>(""); - var t = new Thread(() -> { - try { - read.set(new String(pc.getStdout().readAllBytes(), pc.getCharset())); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - t.setDaemon(true); - t.start(); + ProcessControl start() throws Exception; - var ec = pc.waitFor(); - if (ec == -1) { - throw new ProcessOutputException("Command timed out"); - } + boolean waitFor() throws Exception; - if (ec == 0 && !(read.get().isEmpty() && !readError.get().isEmpty())) { - return read.get().trim(); - } else { - throw new ProcessOutputException( - "Command returned with " + ec + ": " + readError.get().trim()); - } - } + InputStream getStdout(); - public Thread discardOut() { - var t = new Thread(() -> { - try { - getStdout().transferTo(OutputStream.nullOutputStream()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - t.setDaemon(true); - t.start(); - return t; - } + OutputStream getStdin(); - public Thread discardErr() { - var t = new Thread(() -> { - try { - getStderr().transferTo(OutputStream.nullOutputStream()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - t.setDaemon(true); - t.start(); - return t; - } + InputStream getStderr(); - public abstract void start() throws Exception; - - public abstract int waitFor() throws Exception; - - public abstract InputStream getStdout(); - - public abstract OutputStream getStdin(); - - public abstract InputStream getStderr(); - - public abstract Charset getCharset(); - - public abstract List getCommand(); + Charset getCharset(); } diff --git a/core/src/main/java/io/xpipe/core/store/ProcessControlProvider.java b/core/src/main/java/io/xpipe/core/store/ProcessControlProvider.java new file mode 100644 index 000000000..442265677 --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/ProcessControlProvider.java @@ -0,0 +1,6 @@ +package io.xpipe.core.store; + +public abstract class ProcessControlProvider { + + public abstract ProcessControl local(); +} diff --git a/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java b/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java new file mode 100644 index 000000000..194ea09cd --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/ShellProcessControl.java @@ -0,0 +1,58 @@ +package io.xpipe.core.store; + +import io.xpipe.core.util.SecretValue; +import lombok.NonNull; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +public interface ShellProcessControl extends ProcessControl { + + ShellProcessControl elevation(SecretValue value); + + SecretValue getElevationPassword(); + + default ShellProcessControl shell(@NonNull ShellType type){ + return shell(type.openCommand()); + } + + default ShellProcessControl shell(@NonNull List command) { + return shell( + command.stream().map(s -> s.contains(" ") ? "\"" + s + "\"" : s).collect(Collectors.joining(" "))); + } + + default ShellProcessControl shell(@NonNull String command){ + return shell(processControl -> command); + } + + ShellProcessControl shell(@NonNull Function command); + + void executeCommand(String command) throws IOException; + + default void executeCommand(List command) throws IOException { + executeCommand( + command.stream().map(s -> s.contains(" ") ? "\"" + s + "\"" : s).collect(Collectors.joining(" "))); + } + + @Override + ShellProcessControl start() throws Exception; + + default CommandProcessControl commandListFunction(Function> command) { + return commandFunction(shellProcessControl -> + command.apply(shellProcessControl).stream().map(s -> s.contains(" ") ? "\"" + s + "\"" : s).collect(Collectors.joining(" "))); + } + + CommandProcessControl commandFunction(Function command); + + CommandProcessControl command(String command); + + default CommandProcessControl command(List command) { + return command( + command.stream().map(s -> s.contains(" ") ? "\"" + s + "\"" : s).collect(Collectors.joining(" "))); + } + + void exit() throws IOException; + +} diff --git a/core/src/main/java/io/xpipe/core/store/ShellStore.java b/core/src/main/java/io/xpipe/core/store/ShellStore.java index 7d9a87947..1f57cf33b 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellStore.java +++ b/core/src/main/java/io/xpipe/core/store/ShellStore.java @@ -1,43 +1,20 @@ package io.xpipe.core.store; -import io.xpipe.core.util.SecretValue; - -import java.nio.charset.Charset; -import java.util.List; +import java.util.concurrent.atomic.AtomicReference; public interface ShellStore extends DataStore { - public default Integer getTimeout() { - return null; + public static MachineStore local() { + return new LocalStore(); } - public default List getInput() { - return List.of(); - } + ShellProcessControl create(); - public default Integer getEffectiveTimeOut(Integer timeout) { - if (this.getTimeout() == null) { - return timeout; + public default ShellType determineType() throws Exception { + AtomicReference type = new AtomicReference<>(); + try (var pc = create().start()) { + type.set(pc.getShellType()); } - if (timeout == null) { - return getTimeout(); - } - return Math.min(getTimeout(), timeout); - } - - public default ProcessControl prepareCommand(List cmd, Integer timeout, Charset charset) throws Exception { - return prepareCommand(List.of(), cmd, timeout, charset); - } - - public abstract ProcessControl prepareCommand(List input, List cmd, Integer timeout, Charset charset) - throws Exception; - - public default ProcessControl preparePrivilegedCommand(List cmd, Integer timeout, Charset charset) throws Exception { - return preparePrivilegedCommand(List.of(), cmd, timeout, charset); - } - - public default ProcessControl preparePrivilegedCommand(List input, List cmd, Integer timeout, Charset charset) - throws Exception { - throw new UnsupportedOperationException(); + return type.get(); } } diff --git a/core/src/main/java/io/xpipe/core/store/ShellType.java b/core/src/main/java/io/xpipe/core/store/ShellType.java new file mode 100644 index 000000000..4e07c9acd --- /dev/null +++ b/core/src/main/java/io/xpipe/core/store/ShellType.java @@ -0,0 +1,55 @@ +package io.xpipe.core.store; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.xpipe.core.charsetter.NewLine; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "type" +) +public interface ShellType { + + void elevate(ShellProcessControl control, String command, String displayCommand) throws IOException; + + default void init(ProcessControl proc) throws IOException { + } + + default String getExitCommand() { + return "exit"; + } + + String getExitCodeVariable(); + + default String getConcatenationOperator() { + return ";"; + } + + String getEchoCommand(String s, boolean newLine); + + List openCommand(); + String switchTo(String cmd); + + List createMkdirsCommand(String dirs); + + List createFileReadCommand(String file); + + List createFileWriteCommand(String file); + + List createFileExistsCommand(String file); + + Charset determineCharset(ProcessControl control) throws Exception; + + NewLine getNewLine(); + + String getName(); + + String getDisplayName(); + + List getOperatingSystemNameCommand(); + + boolean echoesInput(); +} diff --git a/core/src/main/java/io/xpipe/core/store/ShellTypes.java b/core/src/main/java/io/xpipe/core/store/ShellTypes.java index 55531c11c..a902773ea 100644 --- a/core/src/main/java/io/xpipe/core/store/ShellTypes.java +++ b/core/src/main/java/io/xpipe/core/store/ShellTypes.java @@ -2,54 +2,57 @@ package io.xpipe.core.store; import com.fasterxml.jackson.annotation.JsonTypeName; import io.xpipe.core.charsetter.NewLine; -import io.xpipe.core.util.SecretValue; import lombok.Value; -import java.io.ByteArrayInputStream; +import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; public class ShellTypes { - public static final StandardShellStore.ShellType POWERSHELL = new PowerShell(); - public static final StandardShellStore.ShellType CMD = new Cmd(); - public static final StandardShellStore.ShellType SH = new Sh(); + public static final ShellType POWERSHELL = new PowerShell(); + public static final ShellType CMD = new Cmd(); + public static final ShellType SH = new Sh(); - public static StandardShellStore.ShellType determine(ShellStore store) throws Exception { - var o = store.prepareCommand(List.of(), List.of("echo", "$0"), null, StandardCharsets.US_ASCII) - .executeAndReadStdoutOrThrow() - .strip(); - if (!o.equals("$0")) { - return SH; + public static ShellType determine(ProcessControl proc) throws Exception { + proc.writeLine("echo -NoEnumerate \"a\"", false); + String line; + while (true) { + line = proc.readLine(); + if (line.equals("-NoEnumerate a")) { + return SH; + } + + if (line.contains("echo -NoEnumerate \"a\"")) { + break; + } + } + + var o = proc.readLine(); + + if (o.equals("a")) { + return POWERSHELL; + } else if (o.equals("-NoEnumerate \"a\"")) { + return CMD; } else { - o = store.prepareCommand( - List.of(), - List.of("(dir 2>&1 *`|echo CMD);&<# rem #>echo PowerShell"), - null, - StandardCharsets.UTF_16LE) - .executeAndReadStdoutOrThrow() - .trim(); - if (o.equals("PowerShell")) { - return POWERSHELL; + return SH; + } + } + + public static ShellType[] getAvailable(ShellStore store) throws Exception { + try (ProcessControl proc = store.create().start()) { + var type = determine(proc); + if (type == SH) { + return getLinuxShells(); } else { - return CMD; + return getWindowsShells(); } } } - public static StandardShellStore.ShellType[] getAvailable(ShellStore store) throws Exception { - var o = store.prepareCommand(List.of(), List.of("echo", "$0"), null, StandardCharsets.US_ASCII) - .executeAndReadStdoutOrThrow(); - if (o.trim().length() > 0 && !o.trim().equals("$0")) { - return getLinuxShells(); - } else { - return getWindowsShells(); - } - } - - public static StandardShellStore.ShellType getRecommendedDefault() { + public static ShellType getRecommendedDefault() { if (System.getProperty("os.name").startsWith("Windows")) { return POWERSHELL; } else { @@ -57,7 +60,7 @@ public class ShellTypes { } } - public static StandardShellStore.ShellType getPlatformDefault() { + public static ShellType getPlatformDefault() { if (System.getProperty("os.name").startsWith("Windows")) { return CMD; } else { @@ -65,17 +68,51 @@ public class ShellTypes { } } - public static StandardShellStore.ShellType[] getWindowsShells() { - return new StandardShellStore.ShellType[] {CMD, POWERSHELL}; + public static ShellType[] getWindowsShells() { + return new ShellType[] {CMD, POWERSHELL}; } - public static StandardShellStore.ShellType[] getLinuxShells() { - return new StandardShellStore.ShellType[] {SH}; + public static ShellType[] getLinuxShells() { + return new ShellType[] {SH}; } @JsonTypeName("cmd") @Value - public static class Cmd implements StandardShellStore.ShellType { + public static class Cmd implements ShellType { + + @Override + public String getEchoCommand(String s, boolean newLine) { + return newLine ? "echo " + s : "echo | set /p dummyName=" + s; + } + + @Override + public String getConcatenationOperator() { + return "&"; + } + + @Override + public void elevate(ShellProcessControl control, String command, String displayCommand) throws IOException { + control.executeCommand("net session >NUL 2>NUL"); + control.executeCommand("echo %errorLevel%"); + var exitCode = Integer.parseInt(control.readLine()); + if (exitCode != 0) { + throw new IllegalStateException("The command \"" + displayCommand + "\" requires elevation."); + } + + control.executeCommand(command); + } + + @Override + public void init(ProcessControl proc) throws IOException { + proc.readLine(); + proc.readLine(); + proc.readLine(); + } + + @Override + public String getExitCodeVariable() { + return "%errorlevel%"; + } @Override public NewLine getNewLine() { @@ -83,24 +120,18 @@ public class ShellTypes { } @Override - public List switchTo(List cmd) { - var l = new ArrayList<>(cmd); - l.add(0, "cmd.exe"); - l.add(1, "/c"); - l.add(2, "@chcp"); - l.add(3, "65001"); - l.add(4, ">"); - l.add(5, "nul"); - l.add(6, "&&"); - return l; + public List openCommand() { + return List.of("cmd"); } @Override - public ProcessControl prepareElevatedCommand( - ShellStore st, List in, List cmd, Integer timeout, String pw, Charset charset) - throws Exception { - var l = List.of("net", "session", ";", "if", "%errorLevel%", "!=", "0"); - return st.prepareCommand(List.of(), l, timeout, charset); + public String switchTo(String cmd) { + return "cmd.exe /c " + cmd; + } + + @Override + public List createMkdirsCommand(String dirs) { + return List.of("lmkdir", dirs); } @Override @@ -119,8 +150,11 @@ public class ShellTypes { } @Override - public Charset determineCharset(ShellStore store) throws Exception { - return StandardCharsets.UTF_8; + public Charset determineCharset(ProcessControl control) throws Exception { + var output = control.readResultLine("chcp", true); + var matcher = Pattern.compile("\\d+").matcher(output); + matcher.find(); + return Charset.forName("ibm" + matcher.group()); } @Override @@ -130,24 +164,63 @@ public class ShellTypes { @Override public String getDisplayName() { - return "cmd.exe"; + return "cmd"; } @Override public List getOperatingSystemNameCommand() { return List.of("Get-ComputerInfo"); } + + @Override + public boolean echoesInput() { + return true; + } } @JsonTypeName("powershell") @Value - public static class PowerShell implements StandardShellStore.ShellType { + public static class PowerShell implements ShellType { @Override - public List switchTo(List cmd) { - var l = new ArrayList<>(cmd); - l.add(0, "powershell.exe"); - return l; + public boolean echoesInput() { + return true; + } + + @Override + public void elevate(ShellProcessControl control, String command, String displayCommand) throws IOException { + control.executeCommand("([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator)"); + var exitCode = Integer.parseInt(control.readLine()); + if (exitCode != 0) { + throw new IllegalStateException("The command \"" + displayCommand + "\" requires elevation."); + } + + control.executeCommand(command); + } + + @Override + public String getExitCodeVariable() { + return "$LASTEXITCODE"; + } + + @Override + public String getEchoCommand(String s, boolean newLine) { + return newLine ? "echo " + s : String.format("Write-Host \"%s\" -NoNewLine", s); + } + + @Override + public List openCommand() { + return List.of("powershell", "/nologo"); + } + + @Override + public String switchTo(String cmd) { + return "powershell.exe -Command " + cmd; + } + + @Override + public List createMkdirsCommand(String dirs) { + return List.of("New-Item", "-Path", "D:\\temp\\Test Folder", "-ItemType", "Directory"); } @Override @@ -166,8 +239,11 @@ public class ShellTypes { } @Override - public Charset determineCharset(ShellStore store) throws Exception { - return StandardCharsets.UTF_16LE; + public Charset determineCharset(ProcessControl control) throws Exception { + var output = control.readResultLine("chcp", true); + var matcher = Pattern.compile("\\d+").matcher(output); + matcher.find(); + return Charset.forName("ibm" + matcher.group()); } @Override @@ -193,22 +269,41 @@ public class ShellTypes { @JsonTypeName("sh") @Value - public static class Sh implements StandardShellStore.ShellType { + public static class Sh implements ShellType { @Override - public List switchTo(List cmd) { - return cmd; + public void elevate(ShellProcessControl control, String command, String displayCommand) throws IOException { + if (control.getElevationPassword().getSecretValue() == null) { + throw new IllegalStateException("No password for sudo has been set"); + } + + control.executeCommand("sudo -S " + switchTo(command)); + control.writeLine(control.getElevationPassword().getSecretValue()); } @Override - public ProcessControl prepareElevatedCommand( - ShellStore st, List in, List cmd, Integer timeout, String pw, Charset charset) - throws Exception { - var l = new ArrayList<>(cmd); - l.add(0, "sudo"); - l.add(1, "-S"); - var pws = new ByteArrayInputStream(pw.getBytes(determineCharset(st))); - return st.prepareCommand(List.of(SecretValue.createForSecretValue(pw)), l, timeout, charset); + public String getExitCodeVariable() { + return "$?"; + } + + @Override + public String getEchoCommand(String s, boolean newLine) { + return newLine ? "echo " + s : "echo -n " + s; + } + + @Override + public List openCommand() { + return List.of("sh"); + } + + @Override + public String switchTo(String cmd) { + return "sh -c \"" + cmd + "\""; + } + + @Override + public List createMkdirsCommand(String dirs) { + return List.of("mkdir", "-p", dirs); } @Override @@ -227,7 +322,7 @@ public class ShellTypes { } @Override - public Charset determineCharset(ShellStore st) throws Exception { + public Charset determineCharset(ProcessControl st) throws Exception { return StandardCharsets.UTF_8; } @@ -250,5 +345,10 @@ public class ShellTypes { public List getOperatingSystemNameCommand() { return List.of("uname", "-o"); } + + @Override + public boolean echoesInput() { + return false; + } } } diff --git a/core/src/main/java/io/xpipe/core/store/StandardShellStore.java b/core/src/main/java/io/xpipe/core/store/StandardShellStore.java deleted file mode 100644 index 876729d1d..000000000 --- a/core/src/main/java/io/xpipe/core/store/StandardShellStore.java +++ /dev/null @@ -1,97 +0,0 @@ -package io.xpipe.core.store; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.xpipe.core.charsetter.NewLine; -import io.xpipe.core.util.SecretValue; - -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.Charset; -import java.util.List; - -public interface StandardShellStore extends MachineFileStore, ShellStore { - - public default ProcessControl prepareLocalCommand(List input, List cmd, Integer timeout) - throws Exception { - return prepareCommand(input, cmd, timeout, determineType().determineCharset(this)); - } - - public default boolean isLocal() { - return false; - } - - public default NewLine getNewLine() throws Exception { - return determineType().getNewLine(); - } - - ShellType determineType() throws Exception; - - public default String querySystemName() throws Exception { - var result = prepareCommand( - List.of(), - determineType().getOperatingSystemNameCommand(), - getTimeout(), - determineType().determineCharset(this)) - .executeAndReadStdoutOrThrow(); - return result.strip(); - } - - @Override - public default InputStream openInput(String file) throws Exception { - var type = determineType(); - var cmd = type.createFileReadCommand(file); - var p = prepareCommand(List.of(), cmd, null, type.determineCharset(this)); - p.start(); - return p.getStdout(); - } - - @Override - public default OutputStream openOutput(String file) throws Exception { - return null; - // var type = determineType(); - // var cmd = type.createFileWriteCommand(file); - // var p = prepare(cmd).redirectErrorStream(true); - // var proc = p.start(); - // return proc.getOutputStream(); - } - - @Override - public default boolean exists(String file) throws Exception { - var type = determineType(); - var cmd = type.createFileExistsCommand(file); - var p = prepareCommand(List.of(), cmd, null, type.determineCharset(this)); - p.start(); - return p.waitFor() == 0; - } - - @Override - public default void mkdirs(String file) throws Exception {} - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") - public static interface ShellType { - - List switchTo(List cmd); - - default ProcessControl prepareElevatedCommand( - ShellStore st, List in, List cmd, Integer timeout, String pw, Charset charset) - throws Exception { - return st.prepareCommand(in, cmd, timeout, charset); - } - - List createFileReadCommand(String file); - - List createFileWriteCommand(String file); - - List createFileExistsCommand(String file); - - Charset determineCharset(ShellStore store) throws Exception; - - NewLine getNewLine(); - - String getName(); - - String getDisplayName(); - - List getOperatingSystemNameCommand(); - } -} diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java index 11dc79e46..520d328c7 100644 --- a/core/src/main/java/module-info.java +++ b/core/src/main/java/module-info.java @@ -22,6 +22,7 @@ open module io.xpipe.core { uses com.fasterxml.jackson.databind.Module; uses io.xpipe.core.source.WriteMode; + uses io.xpipe.core.store.LocalStore.LocalProcessControlProvider; provides WriteMode with WriteMode.Replace, WriteMode.Append, WriteMode.Prepend; provides com.fasterxml.jackson.databind.Module with diff --git a/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java b/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java index 31b29dabd..d49a58f88 100644 --- a/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java +++ b/extension/src/main/java/io/xpipe/extension/DataSourceProvider.java @@ -61,14 +61,13 @@ public interface DataSourceProvider> { return i != -1 ? n.substring(i + 1) : n; } + default String queryInformationString(DataStore store, int length) throws Exception { + return getDisplayName(); + } default String getDisplayIconFileName() { return getModuleName() + ":" + getId() + "_icon.png"; } - default String getSourceDescription(T source) { - return getDisplayName(); - } - Dialog configDialog(T source, boolean all); default boolean shouldShow(DataSourceType type) { diff --git a/extension/src/main/java/io/xpipe/extension/DataStoreProvider.java b/extension/src/main/java/io/xpipe/extension/DataStoreProvider.java index 027ceb666..b26897b19 100644 --- a/extension/src/main/java/io/xpipe/extension/DataStoreProvider.java +++ b/extension/src/main/java/io/xpipe/extension/DataStoreProvider.java @@ -2,7 +2,7 @@ package io.xpipe.extension; import io.xpipe.core.dialog.Dialog; import io.xpipe.core.store.DataStore; -import io.xpipe.core.store.MachineFileStore; +import io.xpipe.core.store.FileSystemStore; import io.xpipe.core.store.ShellStore; import io.xpipe.core.store.StreamDataStore; import io.xpipe.core.util.JacksonizedValue; @@ -28,7 +28,7 @@ public interface DataStoreProvider { return Category.STREAM; } - if (MachineFileStore.class.isAssignableFrom(c) || ShellStore.class.isAssignableFrom(c)) { + if (FileSystemStore.class.isAssignableFrom(c) || ShellStore.class.isAssignableFrom(c)) { return Category.MACHINE; } diff --git a/extension/src/main/java/io/xpipe/extension/XPipeServiceProviders.java b/extension/src/main/java/io/xpipe/extension/XPipeServiceProviders.java index 7218cd1eb..3f73ed35b 100644 --- a/extension/src/main/java/io/xpipe/extension/XPipeServiceProviders.java +++ b/extension/src/main/java/io/xpipe/extension/XPipeServiceProviders.java @@ -1,6 +1,7 @@ package io.xpipe.extension; import com.fasterxml.jackson.databind.jsontype.NamedType; +import io.xpipe.core.store.LocalStore; import io.xpipe.core.util.JacksonMapper; import io.xpipe.extension.event.TrackEvent; import io.xpipe.extension.prefs.PrefsProviders; @@ -8,6 +9,8 @@ import io.xpipe.extension.prefs.PrefsProviders; public class XPipeServiceProviders { public static void load(ModuleLayer layer) { + LocalStore.LocalProcessControlProvider.init(layer); + TrackEvent.info("Loading extension providers ..."); DataSourceProviders.init(layer); for (DataSourceProvider p : DataSourceProviders.getAll()) { diff --git a/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java b/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java index 629a4168f..f6ae1bb56 100644 --- a/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java +++ b/extension/src/main/java/io/xpipe/extension/comp/ChoiceComp.java @@ -4,6 +4,7 @@ import io.xpipe.extension.I18n; import io.xpipe.fxcomps.Comp; import io.xpipe.fxcomps.CompStructure; import io.xpipe.fxcomps.SimpleCompStructure; +import io.xpipe.fxcomps.util.BindingsHelper; import io.xpipe.fxcomps.util.PlatformThread; import io.xpipe.fxcomps.util.SimpleChangeListener; import javafx.beans.property.Property; @@ -64,7 +65,8 @@ public class ChoiceComp extends Comp>> { if (!list.contains(null) && includeNone) { list.add(null); } - cb.setItems(list); + + BindingsHelper.setContent(cb.getItems(), list); }); cb.valueProperty().addListener((observable, oldValue, newValue) -> { diff --git a/extension/src/main/java/io/xpipe/extension/event/EventHandler.java b/extension/src/main/java/io/xpipe/extension/event/EventHandler.java index 4c8f09cc2..507660d79 100644 --- a/extension/src/main/java/io/xpipe/extension/event/EventHandler.java +++ b/extension/src/main/java/io/xpipe/extension/event/EventHandler.java @@ -17,7 +17,7 @@ public abstract class EventHandler { if (cat == null) { cat = "log"; } - System.out.println("[" + cat + "] " + te.getMessage()); + System.out.println("[" + cat + "] " + te.toString()); } @Override diff --git a/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java new file mode 100644 index 000000000..5152670b2 --- /dev/null +++ b/extension/src/main/java/io/xpipe/extension/util/DaemonExtensionTest.java @@ -0,0 +1,18 @@ +package io.xpipe.extension.util; + +import io.xpipe.api.util.XPipeDaemonController; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +public class DaemonExtensionTest { + + @BeforeAll + public static void setup() throws Exception { + XPipeDaemonController.start(); + } + + @AfterAll + public static void teardown() throws Exception { + XPipeDaemonController.stop(); + } +} diff --git a/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java b/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java index e4499743f..fe026c994 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java +++ b/extension/src/main/java/io/xpipe/extension/util/DialogHelper.java @@ -33,7 +33,7 @@ public class DialogHelper { throw new IllegalArgumentException(String.format("Store not found: %s", name)); } - if (!(stored.get() instanceof MachineFileStore)) { + if (!(stored.get() instanceof FileSystemStore)) { throw new IllegalArgumentException(String.format("Store not a machine store: %s", name)); } diff --git a/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java b/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java index a8d31690f..095bf70b1 100644 --- a/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java +++ b/extension/src/main/java/io/xpipe/extension/util/DynamicOptionsBuilder.java @@ -53,7 +53,6 @@ public class DynamicOptionsBuilder { } public DynamicOptionsBuilder decorate(Check c) { - entries.get(entries.size() - 1).comp().apply(s -> c.decorates(s.get())); return this; } diff --git a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java index fa793b47b..a4959d8ad 100644 --- a/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java +++ b/extension/src/main/java/io/xpipe/extension/util/ExtensionTest.java @@ -1,22 +1,21 @@ package io.xpipe.extension.util; import io.xpipe.api.DataSource; -import io.xpipe.api.util.XPipeDaemonController; import io.xpipe.core.store.DataStore; import io.xpipe.core.store.FileStore; import io.xpipe.core.util.JacksonMapper; import io.xpipe.extension.XPipeServiceProviders; import lombok.SneakyThrows; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import java.nio.file.Path; public class ExtensionTest { + @SneakyThrows public static DataStore getResource(String name) { - var url = ExtensionTest.class.getClassLoader().getResource(name); + var url = DaemonExtensionTest.class.getClassLoader().getResource(name); if (url == null) { throw new IllegalArgumentException(String.format("File %s does not exist", name)); } @@ -40,11 +39,5 @@ public class ExtensionTest { public static void setup() throws Exception { JacksonMapper.initModularized(ModuleLayer.boot()); XPipeServiceProviders.load(ModuleLayer.boot()); - XPipeDaemonController.start(); - } - - @AfterAll - public static void teardown() throws Exception { - XPipeDaemonController.stop(); } } diff --git a/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java new file mode 100644 index 000000000..5e29f115a --- /dev/null +++ b/extension/src/main/java/io/xpipe/extension/util/LocalExtensionTest.java @@ -0,0 +1,4 @@ +package io.xpipe.extension.util; + +public class LocalExtensionTest extends ExtensionTest { +} diff --git a/extension/src/main/java/io/xpipe/extension/util/OsHelper.java b/extension/src/main/java/io/xpipe/extension/util/OsHelper.java index 57f3b58a6..ef2af664c 100644 --- a/extension/src/main/java/io/xpipe/extension/util/OsHelper.java +++ b/extension/src/main/java/io/xpipe/extension/util/OsHelper.java @@ -26,7 +26,7 @@ public class OsHelper { return; } - ThreadHelper.run(() -> { + ThreadHelper.runAsync(() -> { try { Desktop.getDesktop().open(file.toFile()); } catch (Exception e) { @@ -41,7 +41,7 @@ public class OsHelper { return; } - ThreadHelper.run(() -> { + ThreadHelper.runAsync(() -> { try { Desktop.getDesktop().open(file.getParent().toFile()); } catch (Exception e) { @@ -51,7 +51,7 @@ public class OsHelper { return; } - ThreadHelper.run(() -> { + ThreadHelper.runAsync(() -> { try { Desktop.getDesktop().browseFileDirectory(file.toFile()); } catch (Exception e) { diff --git a/extension/src/main/java/io/xpipe/extension/util/ThreadHelper.java b/extension/src/main/java/io/xpipe/extension/util/ThreadHelper.java index 0ecbbd480..bd9d5fe8e 100644 --- a/extension/src/main/java/io/xpipe/extension/util/ThreadHelper.java +++ b/extension/src/main/java/io/xpipe/extension/util/ThreadHelper.java @@ -2,9 +2,6 @@ package io.xpipe.extension.util; import org.apache.commons.lang3.function.FailableRunnable; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; - public class ThreadHelper { public static void await(FailableRunnable r) { @@ -14,26 +11,13 @@ public class ThreadHelper { } } - public static Thread run(Runnable r) { + public static Thread runAsync(Runnable r) { var t = new Thread(r); t.setDaemon(true); t.start(); return t; } - public static T run(Supplier r) { - AtomicReference ret = new AtomicReference<>(); - var t = new Thread(() -> ret.set(r.get())); - t.setDaemon(true); - t.start(); - try { - t.join(); - } catch (InterruptedException e) { - return null; - } - return ret.get(); - } - public static Thread create(String name, boolean daemon, Runnable r) { var t = new Thread(r); t.setDaemon(daemon); diff --git a/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties b/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties index 555993963..41c1efffc 100644 --- a/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties +++ b/extension/src/main/resources/io/xpipe/extension/resources/lang/translations_en.properties @@ -15,7 +15,7 @@ namedHostNotActive=$HOST$ is not active noInformationAvailable=No information available input=Input output=Output -inout=In/Out +inout=Input and Output inputDescription=This store only produces input for data sources to read outputDescription=This store only accepts output from data sources to write inoutDescription=This store uses both input and output to essentially create a data transformation