beenull/xpipe-mirror
1
0
Fork 0
mirror of https://github.com/xpipe-io/xpipe.git synced 2024-11-21 23:20:23 +00:00
Code Issues Projects Releases Packages Wiki Activity

Blob rework

Browse source
This commit is contained in:
crschnick 2024-06-20 21:25:28 +00:00
parent 0587dea4ac
commit 02b979de6b
13 changed files with 208 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
app/src/main/java/io/xpipe/app/beacon/AppBeaconCache.java
View file

@ -4,16 +4,13 @@ import io.xpipe.beacon.BeaconClientException;
import lombok.Value;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@Value
public class AppBeaconCache {
Set<BeaconShellSession> shellSessions = new HashSet<>();
Map<UUID, byte[]> savedBlobs = new ConcurrentHashMap<>();
public BeaconShellSession getShellSession(UUID uuid) throws BeaconClientException {
var found = shellSessions.stream().filter(beaconShellSession -> beaconShellSession.getEntry().getUuid().equals(uuid)).findFirst();
@ -22,12 +19,4 @@ public class AppBeaconCache {
}
return found.get();
}
public byte[] getBlob(UUID uuid) throws BeaconClientException {
var found = savedBlobs.get(uuid);
if (found == null) {
throw new BeaconClientException("No saved data known for id " + uuid);
}
return found;
}
}

30
app/src/main/java/io/xpipe/app/beacon/BeaconRequestHandler.java
View file

@ -62,19 +62,23 @@ public class BeaconRequestHandler<T> implements HttpHandler {
T object;
Object response;
try {
try (InputStream is = exchange.getRequestBody()) {
var read = is.readAllBytes();
var rawDataRequestClass = beaconInterface.getRequestClass().getDeclaredFields().length == 1 &&
beaconInterface.getRequestClass().getDeclaredFields()[0].getType().equals(byte[].class);
if (!new String(read, StandardCharsets.US_ASCII).trim().startsWith("{") && rawDataRequestClass) {
object = createRawDataRequest(beaconInterface,read);
} else {
var tree = JacksonMapper.getDefault().readTree(read);
TrackEvent.trace("Parsed raw request:\n" + tree.toPrettyString());
var emptyRequestClass = tree.isEmpty() && beaconInterface.getRequestClass().getDeclaredFields().length == 0;
object = emptyRequestClass ? createDefaultRequest(beaconInterface) : JacksonMapper.getDefault().treeToValue(tree,
beaconInterface.getRequestClass());
TrackEvent.trace("Parsed request object:\n" + object);
if (beaconInterface.readRawRequestBody()) {
object = createDefaultRequest(beaconInterface);
} else {
try (InputStream is = exchange.getRequestBody()) {
var read = is.readAllBytes();
var rawDataRequestClass = beaconInterface.getRequestClass().getDeclaredFields().length == 1 &&
beaconInterface.getRequestClass().getDeclaredFields()[0].getType().equals(byte[].class);
if (!new String(read, StandardCharsets.US_ASCII).trim().startsWith("{") && rawDataRequestClass) {
object = createRawDataRequest(beaconInterface, read);
} else {
var tree = JacksonMapper.getDefault().readTree(read);
TrackEvent.trace("Parsed raw request:\n" + tree.toPrettyString());
var emptyRequestClass = tree.isEmpty() && beaconInterface.getRequestClass().getDeclaredFields().length == 0;
object = emptyRequestClass ? createDefaultRequest(beaconInterface) : JacksonMapper.getDefault().treeToValue(tree,
beaconInterface.getRequestClass());
TrackEvent.trace("Parsed request object:\n" + object);
}
}
}
response = beaconInterface.handle(exchange, object);

79
app/src/main/java/io/xpipe/app/beacon/BlobManager.java Normal file
View file

@ -0,0 +1,79 @@
package io.xpipe.app.beacon;
import io.xpipe.app.issue.ErrorEvent;
import io.xpipe.app.util.ShellTemp;
import io.xpipe.beacon.BeaconClientException;
import org.apache.commons.io.FileUtils;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
public class BlobManager {
private static final Path TEMP = ShellTemp.getLocalTempDataDirectory("blob");
private static BlobManager INSTANCE;
private final Map<UUID, byte[]> memoryBlobs = new ConcurrentHashMap<>();
private final Map<UUID, Path> fileBlobs = new ConcurrentHashMap<>();
public static BlobManager get() {
return INSTANCE;
}
public static void init() {
INSTANCE = new BlobManager();
try {
FileUtils.forceMkdir(TEMP.toFile());
try {
// Remove old files in dir
FileUtils.cleanDirectory(TEMP.toFile());
} catch (IOException ignored) {
}
} catch (IOException e) {
ErrorEvent.fromThrowable(e).handle();
}
}
public static void reset() {
try {
FileUtils.cleanDirectory(TEMP.toFile());
} catch (IOException ignored) {
}
INSTANCE = null;
}
public Path newBlobFile() {
return TEMP.resolve(UUID.randomUUID().toString());
}
public void store(UUID uuid, byte[] blob) {
memoryBlobs.put(uuid, blob);
}
public void store(UUID uuid, InputStream blob) throws IOException {
var file = TEMP.resolve(uuid.toString());
try (var fileOut = Files.newOutputStream(file)) {
blob.transferTo(fileOut);
}
fileBlobs.put(uuid,file);
}
public InputStream getBlob(UUID uuid) throws Exception {
var memory = memoryBlobs.get(uuid);
if (memory != null) {
return new ByteArrayInputStream(memory);
}
var found = fileBlobs.get(uuid);
if (found == null) {
throw new BeaconClientException("No saved data known for id " + uuid);
}
return Files.newInputStream(found);
}
}

10
app/src/main/java/io/xpipe/app/beacon/impl/FsBlobExchangeImpl.java
View file

@ -1,7 +1,7 @@
package io.xpipe.app.beacon.impl;
import com.sun.net.httpserver.HttpExchange;
import io.xpipe.app.beacon.AppBeaconServer;
import io.xpipe.app.beacon.BlobManager;
import io.xpipe.beacon.api.FsBlobExchange;
import lombok.SneakyThrows;
@ -13,7 +13,13 @@ public class FsBlobExchangeImpl extends FsBlobExchange {
@SneakyThrows
public Object handle(HttpExchange exchange, Request msg) {
var id = UUID.randomUUID();
AppBeaconServer.get().getCache().getSavedBlobs().put(id, msg.getPayload());
var size = exchange.getRequestBody().available();
if (size > 100_000_000) {
BlobManager.get().store(id,exchange.getRequestBody());
} else {
BlobManager.get().store(id,exchange.getRequestBody().readAllBytes());
}
return Response.builder().blob(id).build();
}
}

44
app/src/main/java/io/xpipe/app/beacon/impl/FsReadExchangeImpl.java
View file

@ -2,10 +2,16 @@ package io.xpipe.app.beacon.impl;
import com.sun.net.httpserver.HttpExchange;
import io.xpipe.app.beacon.AppBeaconServer;
import io.xpipe.app.beacon.BlobManager;
import io.xpipe.app.util.FixedSizeInputStream;
import io.xpipe.beacon.api.FsReadExchange;
import io.xpipe.core.store.ConnectionFileSystem;
import lombok.SneakyThrows;
import java.io.BufferedInputStream;
import java.io.OutputStream;
import java.nio.file.Files;
public class FsReadExchangeImpl extends FsReadExchange {
@Override
@ -13,14 +19,36 @@ public class FsReadExchangeImpl extends FsReadExchange {
public Object handle(HttpExchange exchange, Request msg) {
var shell = AppBeaconServer.get().getCache().getShellSession(msg.getConnection());
var fs = new ConnectionFileSystem(shell.getControl());
byte[] bytes;
try (var in = fs.openInput(msg.getPath().toString())) {
bytes = in.readAllBytes();
var size = fs.getFileSize(msg.getPath().toString());
if (size > 100_000_000) {
var file = BlobManager.get().newBlobFile();
try (var in = fs.openInput(msg.getPath().toString())) {
try (var fileOut = Files.newOutputStream(file.resolve(msg.getPath().getFileName()));
var fixedIn = new FixedSizeInputStream(new BufferedInputStream(in), size)) {
fixedIn.transferTo(fileOut);
}
in.transferTo(OutputStream.nullOutputStream());
}
exchange.sendResponseHeaders(200, size);
try (var fileIn = Files.newInputStream(file); var out = exchange.getResponseBody()) {
fileIn.transferTo(out);
}
return Response.builder().build();
} else {
byte[] bytes;
try (var in = fs.openInput(msg.getPath().toString())) {
try (var fixedIn = new FixedSizeInputStream(new BufferedInputStream(in), size)) {
bytes = fixedIn.readAllBytes();
}
in.transferTo(OutputStream.nullOutputStream());
}
exchange.sendResponseHeaders(200, bytes.length);
try (var out = exchange.getResponseBody()) {
out.write(bytes);
}
return Response.builder().build();
}
exchange.sendResponseHeaders(200, bytes.length);
try (var out = exchange.getResponseBody()) {
out.write(bytes);
}
return Response.builder().build();
}
}

6
app/src/main/java/io/xpipe/app/beacon/impl/FsScriptExchangeImpl.java
View file

@ -2,6 +2,7 @@ package io.xpipe.app.beacon.impl;
import com.sun.net.httpserver.HttpExchange;
import io.xpipe.app.beacon.AppBeaconServer;
import io.xpipe.app.beacon.BlobManager;
import io.xpipe.app.util.ScriptHelper;
import io.xpipe.beacon.api.FsScriptExchange;
import lombok.SneakyThrows;
@ -14,7 +15,10 @@ public class FsScriptExchangeImpl extends FsScriptExchange {
@SneakyThrows
public Object handle(HttpExchange exchange, Request msg) {
var shell = AppBeaconServer.get().getCache().getShellSession(msg.getConnection());
var data = new String(AppBeaconServer.get().getCache().getBlob(msg.getBlob()), StandardCharsets.UTF_8);
String data;
try (var in = BlobManager.get().getBlob(msg.getBlob())) {
data = new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
var file = ScriptHelper.getExecScriptFile(shell.getControl());
shell.getControl().getShellDialect().createScriptTextFileWriteCommand(shell.getControl(), data, file.toString()).execute();
return Response.builder().path(file).build();

7
app/src/main/java/io/xpipe/app/beacon/impl/FsWriteExchangeImpl.java
View file

@ -2,6 +2,7 @@ package io.xpipe.app.beacon.impl;
import com.sun.net.httpserver.HttpExchange;
import io.xpipe.app.beacon.AppBeaconServer;
import io.xpipe.app.beacon.BlobManager;
import io.xpipe.beacon.api.FsWriteExchange;
import io.xpipe.core.store.ConnectionFileSystem;
import lombok.SneakyThrows;
@ -12,10 +13,10 @@ public class FsWriteExchangeImpl extends FsWriteExchange {
@SneakyThrows
public Object handle(HttpExchange exchange, Request msg) {
var shell = AppBeaconServer.get().getCache().getShellSession(msg.getConnection());
var data = AppBeaconServer.get().getCache().getBlob(msg.getBlob());
var fs = new ConnectionFileSystem(shell.getControl());
try (var os = fs.openOutput(msg.getPath().toString(), data.length)) {
os.write(data);
try (var in = BlobManager.get().getBlob(msg.getBlob());
var os = fs.openOutput(msg.getPath().toString(), in.available())) {
in.transferTo(os);
}
return Response.builder().build();
}

4
app/src/main/java/io/xpipe/app/core/mode/BaseMode.java
View file

@ -1,6 +1,7 @@
package io.xpipe.app.core.mode;
import io.xpipe.app.beacon.AppBeaconServer;
import io.xpipe.app.beacon.BlobManager;
import io.xpipe.app.browser.session.BrowserSessionModel;
import io.xpipe.app.comp.store.StoreViewState;
import io.xpipe.app.core.*;
@ -62,6 +63,7 @@ public class BaseMode extends OperationMode {
DataStoreProviders.init();
AppFileWatcher.init();
FileBridge.init();
BlobManager.init();
ActionProvider.initProviders();
TrackEvent.info("Finished base components initialization");
initialized = true;
@ -81,6 +83,8 @@ public class BaseMode extends OperationMode {
AppResources.reset();
AppExtensionManager.reset();
AppDataLock.unlock();
BlobManager.reset();
FileBridge.reset();
// Shut down server last to keep a non-daemon thread running
AppBeaconServer.reset();
TrackEvent.info("Background mode shutdown finished");

40
app/src/main/java/io/xpipe/app/util/FileBridge.java
View file

@ -27,37 +27,6 @@ import java.util.function.Consumer;
public class FileBridge {
private static class FixedSizeInputStream extends SimpleFilterInputStream {
private long count;
private final long size;
protected FixedSizeInputStream(InputStream in, long size) {
super(in);
this.size = size;
}
@Override
public int read() throws IOException {
if (count >= size) {
return -1;
}
var read = in.read();
count++;
if (read == -1) {
return 0;
} else {
return read;
}
}
@Override
public int available() {
return (int) (size - count);
}
}
private static final Path TEMP = ShellTemp.getLocalTempDataDirectory("bridge");
private static FileBridge INSTANCE;
private final Set<Entry> openEntries = new HashSet<>();
@ -97,6 +66,14 @@ public class FileBridge {
}
}
public static void reset() {
try {
FileUtils.cleanDirectory(TEMP.toFile());
} catch (IOException ignored) {
}
INSTANCE = null;
}
private synchronized void handleWatchEvent(Path changed, WatchEvent.Kind<Path> kind) {
if (kind == StandardWatchEventKinds.ENTRY_DELETE) {
event("Editor entry file " + changed.toString() + " has been removed");
@ -135,6 +112,7 @@ public class FileBridge {
try (var fixedIn = new FixedSizeInputStream(new BufferedInputStream(in), actualSize)) {
e.writer.accept(fixedIn, actualSize);
}
in.transferTo(OutputStream.nullOutputStream());
var taken = Duration.between(started, Instant.now());
event("Wrote " + HumanReadableFormat.byteCount(actualSize) + " in " + taken.toMillis() + "ms");
}

35
app/src/main/java/io/xpipe/app/util/FixedSizeInputStream.java Normal file
View file

@ -0,0 +1,35 @@
package io.xpipe.app.util;
import java.io.IOException;
import java.io.InputStream;
public class FixedSizeInputStream extends SimpleFilterInputStream {
private long count;
private final long size;
public FixedSizeInputStream(InputStream in, long size) {
super(in);
this.size = size;
}
@Override
public int read() throws IOException {
if (count >= size) {
return -1;
}
var read = in.read();
count++;
if (read == -1) {
return 0;
} else {
return read;
}
}
@Override
public int available() {
return (int) (size - count);
}
}

1
app/src/main/java/module-info.java
View file

@ -96,6 +96,7 @@ open module io.xpipe.app {
requires jdk.jdwp.agent;
requires org.kordamp.ikonli.core;
requires jdk.httpserver;
requires java.sql;
uses TerminalLauncher;
uses io.xpipe.app.ext.ActionProvider;

4
beacon/src/main/java/io/xpipe/beacon/BeaconInterface.java
View file

@ -75,4 +75,8 @@ public abstract class BeaconInterface<T> {
throws BeaconClientException, BeaconServerException {
throw new UnsupportedOperationException();
}
public boolean readRawRequestBody() {
return false;
}
}

9
beacon/src/main/java/io/xpipe/beacon/api/FsBlobExchange.java
View file

@ -15,12 +15,15 @@ public class FsBlobExchange extends BeaconInterface<FsBlobExchange.Request> {
return "/fs/blob";
}
@Override
public boolean readRawRequestBody() {
return true;
}
@Jacksonized
@Builder
@Value
public static class Request {
byte @NonNull [] payload;
}
public static class Request {}
@Jacksonized
@Builder