Fix coping and moving files around using the file tree (#568)

* Implement write/read buffers in electron fill

This makes cutting and copy files from the file tree work.

* Implement fs.createReadStream

This is used by the file tree to copy files.

* Allow passing proxies back from client to server

This makes things like piping streams possible.

* Synchronously bind to proxy events

This eliminates any chance whatsoever of missing events due to binding
too late.

* Make it possible to bind some events on demand

* Add some protocol documentation

Author: code-asher

packages/ide/src/fill/electron.ts

@@ -171,8 +171,10 @@ const newCreateElement = <K extends keyof HTMLElementTagNameMap>(tagName: K): HT
 document.createElement = newCreateElement;
 
 class Clipboard {
-	public has(): boolean {
-		return false;
+	private readonly buffers = new Map<string, Buffer>();
+
+	public has(format: string): boolean {
+		return this.buffers.has(format);
 	}
 
 	public readFindText(): string {
@@ -190,6 +192,14 @@ class Clipboard {
 	public readText(): Promise<string> {
 		return clipboard.readText();
 	}
+
+	public writeBuffer(format: string, buffer: Buffer): void {
+		this.buffers.set(format, buffer);
+	}
+
+	public readBuffer(format: string): Buffer | undefined {
+		return this.buffers.get(format);
+	}
 }
 
 class Shell {

packages/protocol/README.md

@@ -0,0 +1,47 @@
+# Protocol
+
+This module provides a way for the browser to run Node modules like `fs`, `net`,
+etc.
+
+## Internals
+
+### Server-side proxies
+The server-side proxies are regular classes that call native Node functions. The
+only thing special about them is that they must return promises and they must
+return serializable values.
+
+The only exception to the promise rule are event-related methods such as
+`onEvent` and `onDone` (these are synchronous). The server will simply
+immediately bind and push all events it can to the client. It doesn't wait for
+the client to start listening. This prevents issues with the server not
+receiving the client's request to start listening in time.
+
+However, there is a way to specify events that should not bind immediately and
+should wait for the client to request it, because some events (like `data` on a
+stream) cannot be bound immediately (because doing so changes how the stream
+behaves).
+
+### Client-side proxies
+Client-side proxies are `Proxy` instances. They simply make remote calls for any
+method you call on it. The only exception is for events. Each client proxy has a
+local emitter which it uses in place of a remote call (this allows the call to
+be completed synchronously on the client). Then when an event is received from
+the server, it gets emitted on that local emitter.
+
+When an event is listened to, the proxy also notifies the server so it can start
+listening in case it isn't already (see the `data` example above). This only
+works for events that only fire after they are bound.
+
+### Client-side fills
+The client-side fills implement the Node API and make calls to the server-side
+proxies using the client-side proxies.
+
+When a proxy returns a proxy (for example `fs.createWriteStream`), that proxy is
+a promise (since communicating with the server is asynchronous). We have to
+return the fill from `fs.createWriteStream` synchronously, so that means the
+fill has to contain a proxy promise. To eliminate the need for calling `then`
+and to keep the code looking clean every time you use the proxy, the proxy is
+itself wrapped in another proxy which just calls the method after a `then`. This
+works since all the methods return promises (aside from the event methods, but
+those are not used by the fills directly—they are only used internally to
+forward events to the fill if it is an event emitter).

packages/protocol/src/browser/client.ts

@@ -4,7 +4,7 @@ import { promisify } from "util";
 import { Emitter } from "@coder/events";
 import { logger, field } from "@coder/logger";
 import { ReadWriteConnection, InitData, SharedProcessData } from "../common/connection";
-import { Module, ServerProxy } from "../common/proxy";
+import { ClientServerProxy, Module, ServerProxy } from "../common/proxy";
 import { argumentToProto, protoToArgument, moduleToProto, protoToModule, protoToOperatingSystem } from "../common/util";
 import { Argument, Ping, ServerMessage, ClientMessage, Method, Event, Callback } from "../proto";
 import { FsModule, ChildProcessModule, NetModule, NodePtyModule, SpdlogModule, TrashModule } from "./modules";
@@ -224,7 +224,11 @@ export class Client {
 			field("method", method),
 		]);
 
-		proxyMessage.setArgsList(args.map((a) => argumentToProto(a, storeCallback)));
+		proxyMessage.setArgsList(args.map((a) => argumentToProto<ClientServerProxy>(
+			a,
+			storeCallback,
+			(p) => p.proxyId,
+		)));
 
 		const clientMessage = new ClientMessage();
 		clientMessage.setMethod(message);
@@ -429,7 +433,7 @@ export class Client {
 	/**
 	 * Return a proxy that makes remote calls.
 	 */
-	private createProxy<T>(proxyId: number | Module, promise: Promise<any> = Promise.resolve()): T {
+	private createProxy<T extends ClientServerProxy>(proxyId: number | Module, promise: Promise<any> = Promise.resolve()): T {
 		logger.trace(() => [
 			"creating proxy",
 			field("proxyId", proxyId),
@@ -449,7 +453,7 @@ export class Client {
 					cb(event.event, ...event.args);
 				});
 			},
-		}, {
+		} as ClientServerProxy, {
 			get: (target: any, name: string): any => {
 				// When resolving a promise with a proxy, it will check for "then".
 				if (name === "then") {

packages/protocol/src/browser/modules/child_process.ts

@@ -2,13 +2,22 @@ import * as cp from "child_process";
 import * as net from "net";
 import * as stream from "stream";
 import { callbackify } from "util";
-import { ClientProxy } from "../../common/proxy";
-import { ChildProcessModuleProxy, ChildProcessProxy, ChildProcessProxies } from "../../node/modules/child_process";
-import { Readable, Writable } from "./stream";
+import { ClientProxy, ClientServerProxy } from "../../common/proxy";
+import { ChildProcessModuleProxy, ChildProcessProxy } from "../../node/modules/child_process";
+import { ClientWritableProxy, ClientReadableProxy, Readable, Writable } from "./stream";
 
 // tslint:disable completed-docs
 
-export class ChildProcess extends ClientProxy<ChildProcessProxy> implements cp.ChildProcess {
+export interface ClientChildProcessProxy extends ChildProcessProxy, ClientServerProxy<cp.ChildProcess> {}
+
+export interface ClientChildProcessProxies {
+	childProcess: ClientChildProcessProxy;
+	stdin?: ClientWritableProxy | null;
+	stdout?: ClientReadableProxy | null;
+	stderr?: ClientReadableProxy | null;
+}
+
+export class ChildProcess extends ClientProxy<ClientChildProcessProxy> implements cp.ChildProcess {
 	public readonly stdin: stream.Writable;
 	public readonly stdout: stream.Readable;
 	public readonly stderr: stream.Readable;
@@ -18,7 +27,7 @@ export class ChildProcess extends ClientProxy<ChildProcessProxy> implements cp.C
 	private _killed: boolean = false;
 	private _pid = -1;
 
-	public constructor(proxyPromises: Promise<ChildProcessProxies>) {
+	public constructor(proxyPromises: Promise<ClientChildProcessProxies>) {
 		super(proxyPromises.then((p) => p.childProcess));
 		this.stdin = new Writable(proxyPromises.then((p) => p.stdin!));
 		this.stdout = new Readable(proxyPromises.then((p) => p.stdout!));
@@ -99,8 +108,14 @@ export class ChildProcess extends ClientProxy<ChildProcessProxy> implements cp.C
 	}
 }
 
+interface ClientChildProcessModuleProxy extends ChildProcessModuleProxy, ClientServerProxy {
+	exec(command: string, options?: { encoding?: string | null } & cp.ExecOptions | null, callback?: ((error: cp.ExecException | null, stdin: string | Buffer, stdout: string | Buffer) => void)): Promise<ClientChildProcessProxies>;
+	fork(modulePath: string, args?: string[], options?: cp.ForkOptions): Promise<ClientChildProcessProxies>;
+	spawn(command: string, args?: string[], options?: cp.SpawnOptions): Promise<ClientChildProcessProxies>;
+}
+
 export class ChildProcessModule {
-	public constructor(private readonly proxy: ChildProcessModuleProxy) {}
+	public constructor(private readonly proxy: ClientChildProcessModuleProxy) {}
 
 	public exec = (
 		command: string,

packages/protocol/src/browser/modules/fs.ts

@@ -1,12 +1,11 @@
 import * as fs from "fs";
 import { callbackify } from "util";
-import { ClientProxy, Batch } from "../../common/proxy";
+import { Batch, ClientProxy, ClientServerProxy } from "../../common/proxy";
 import { IEncodingOptions, IEncodingOptionsCallback } from "../../common/util";
-import { FsModuleProxy, Stats as IStats, WatcherProxy, WriteStreamProxy } from "../../node/modules/fs";
-import { Writable  } from "./stream";
+import { FsModuleProxy, ReadStreamProxy, Stats as IStats, WatcherProxy, WriteStreamProxy } from "../../node/modules/fs";
+import { Readable, Writable  } from "./stream";
 
-// tslint:disable no-any
-// tslint:disable completed-docs
+// tslint:disable completed-docs no-any
 
 class StatBatch extends Batch<IStats, { path: fs.PathLike }> {
 	public constructor(private readonly proxy: FsModuleProxy) {
@@ -38,7 +37,9 @@ class ReaddirBatch extends Batch<Buffer[] | fs.Dirent[] | string[], { path: fs.P
 	}
 }
 
-class Watcher extends ClientProxy<WatcherProxy> implements fs.FSWatcher {
+interface ClientWatcherProxy extends WatcherProxy, ClientServerProxy<fs.FSWatcher> {}
+
+class Watcher extends ClientProxy<ClientWatcherProxy> implements fs.FSWatcher {
 	public close(): void {
 		this.catch(this.proxy.close());
 	}
@@ -48,7 +49,25 @@ class Watcher extends ClientProxy<WatcherProxy> implements fs.FSWatcher {
 	}
 }
 
-class WriteStream extends Writable<WriteStreamProxy> implements fs.WriteStream {
+interface ClientReadStreamProxy extends ReadStreamProxy, ClientServerProxy<fs.ReadStream> {}
+
+class ReadStream extends Readable<ClientReadStreamProxy> implements fs.ReadStream {
+	public get bytesRead(): number {
+		throw new Error("not implemented");
+	}
+
+	public get path(): string | Buffer {
+		throw new Error("not implemented");
+	}
+
+	public close(): void {
+		this.catch(this.proxy.close());
+	}
+}
+
+interface ClientWriteStreamProxy extends WriteStreamProxy, ClientServerProxy<fs.WriteStream> {}
+
+class WriteStream extends Writable<ClientWriteStreamProxy> implements fs.WriteStream {
 	public get bytesWritten(): number {
 		throw new Error("not implemented");
 	}
@@ -62,12 +81,18 @@ class WriteStream extends Writable<WriteStreamProxy> implements fs.WriteStream {
 	}
 }
 
+interface ClientFsModuleProxy extends FsModuleProxy, ClientServerProxy {
+	createReadStream(path: fs.PathLike, options?: any): Promise<ClientReadStreamProxy>;
+	createWriteStream(path: fs.PathLike, options?: any): Promise<ClientWriteStreamProxy>;
+	watch(filename: fs.PathLike, options?: IEncodingOptions): Promise<ClientWatcherProxy>;
+}
+
 export class FsModule {
 	private readonly statBatch: StatBatch;
 	private readonly lstatBatch: LstatBatch;
 	private readonly readdirBatch: ReaddirBatch;
 
-	public constructor(private readonly proxy: FsModuleProxy) {
+	public constructor(private readonly proxy: ClientFsModuleProxy) {
 		this.statBatch = new StatBatch(this.proxy);
 		this.lstatBatch = new LstatBatch(this.proxy);
 		this.readdirBatch = new ReaddirBatch(this.proxy);
@@ -110,6 +135,10 @@ export class FsModule {
 		);
 	}
 
+	public createReadStream = (path: fs.PathLike, options?: any): fs.ReadStream => {
+		return new ReadStream(this.proxy.createReadStream(path, options));
+	}
+
 	public createWriteStream = (path: fs.PathLike, options?: any): fs.WriteStream => {
 		return new WriteStream(this.proxy.createWriteStream(path, options));
 	}

packages/protocol/src/browser/modules/net.ts

@@ -1,16 +1,18 @@
 import * as net from "net";
 import { callbackify } from "util";
-import { ClientProxy } from "../../common/proxy";
+import { ClientProxy, ClientServerProxy } from "../../common/proxy";
 import { NetModuleProxy, NetServerProxy, NetSocketProxy } from "../../node/modules/net";
 import { Duplex } from "./stream";
 
 // tslint:disable completed-docs
 
-export class Socket extends Duplex<NetSocketProxy> implements net.Socket {
+interface ClientNetSocketProxy extends NetSocketProxy, ClientServerProxy<net.Socket> {}
+
+export class Socket extends Duplex<ClientNetSocketProxy> implements net.Socket {
 	private _connecting: boolean = false;
 	private _destroyed: boolean = false;
 
-	public constructor(proxyPromise: Promise<NetSocketProxy> | NetSocketProxy, connecting?: boolean) {
+	public constructor(proxyPromise: Promise<ClientNetSocketProxy> | ClientNetSocketProxy, connecting?: boolean) {
 		super(proxyPromise);
 		if (connecting) {
 			this._connecting = connecting;
@@ -126,12 +128,16 @@ export class Socket extends Duplex<NetSocketProxy> implements net.Socket {
 	}
 }
 
-export class Server extends ClientProxy<NetServerProxy> implements net.Server {
+interface ClientNetServerProxy extends NetServerProxy, ClientServerProxy<net.Server> {
+	onConnection(cb: (proxy: ClientNetSocketProxy) => void): Promise<void>;
+}
+
+export class Server extends ClientProxy<ClientNetServerProxy> implements net.Server {
 	private socketId = 0;
 	private readonly sockets = new Map<number, net.Socket>();
 	private _listening: boolean = false;
 
-	public constructor(proxyPromise: Promise<NetServerProxy> | NetServerProxy) {
+	public constructor(proxyPromise: Promise<ClientNetServerProxy> | ClientNetServerProxy) {
 		super(proxyPromise);
 
 		this.catch(this.proxy.onConnection((socketProxy) => {
@@ -208,11 +214,17 @@ export class Server extends ClientProxy<NetServerProxy> implements net.Server {
 
 type NodeNet = typeof net;
 
+interface ClientNetModuleProxy extends NetModuleProxy, ClientServerProxy {
+	createSocket(options?: net.SocketConstructorOpts): Promise<ClientNetSocketProxy>;
+	createConnection(target: string | number | net.NetConnectOpts, host?: string): Promise<ClientNetSocketProxy>;
+	createServer(options?: { allowHalfOpen?: boolean, pauseOnConnect?: boolean }): Promise<ClientNetServerProxy>;
+}
+
 export class NetModule implements NodeNet {
 	public readonly Socket: typeof net.Socket;
 	public readonly Server: typeof net.Server;
 
-	public constructor(private readonly proxy: NetModuleProxy) {
+	public constructor(private readonly proxy: ClientNetModuleProxy) {
 		// @ts-ignore this is because Socket is missing things from the Stream
 		// namespace but I'm unsure how best to provide them (finished,
 		// finished.__promisify__, pipeline, and some others) or if it even matters.

packages/protocol/src/browser/modules/node-pty.ts

@@ -1,15 +1,17 @@
 import * as pty from "node-pty";
-import { ClientProxy } from "../../common/proxy";
+import { ClientProxy, ClientServerProxy } from "../../common/proxy";
 import { NodePtyModuleProxy, NodePtyProcessProxy } from "../../node/modules/node-pty";
 
 // tslint:disable completed-docs
 
-export class NodePtyProcess extends ClientProxy<NodePtyProcessProxy> implements pty.IPty {
+interface ClientNodePtyProcessProxy extends NodePtyProcessProxy, ClientServerProxy {}
+
+export class NodePtyProcess extends ClientProxy<ClientNodePtyProcessProxy> implements pty.IPty {
 	private _pid = -1;
 	private _process = "";
 
 	public constructor(
-		private readonly moduleProxy: NodePtyModuleProxy,
+		private readonly moduleProxy: ClientNodePtyModuleProxy,
 		private readonly file: string,
 		private readonly args: string[] | string,
 		private readonly options: pty.IPtyForkOptions,
@@ -18,10 +20,12 @@ export class NodePtyProcess extends ClientProxy<NodePtyProcessProxy> implements
 		this.on("process", (process) => this._process = process);
 	}
 
-	protected initialize(proxyPromise: Promise<NodePtyProcessProxy>): void {
-		super.initialize(proxyPromise);
+	protected initialize(proxyPromise: Promise<ClientNodePtyProcessProxy>): ClientNodePtyProcessProxy {
+		const proxy = super.initialize(proxyPromise);
 		this.catch(this.proxy.getPid().then((p) => this._pid = p));
 		this.catch(this.proxy.getProcess().then((p) => this._process = p));
+
+		return proxy;
 	}
 
 	public get pid(): number {
@@ -53,8 +57,12 @@ export class NodePtyProcess extends ClientProxy<NodePtyProcessProxy> implements
 
 type NodePty = typeof pty;
 
+interface ClientNodePtyModuleProxy extends NodePtyModuleProxy, ClientServerProxy {
+	spawn(file: string, args: string[] | string, options: pty.IPtyForkOptions): Promise<ClientNodePtyProcessProxy>;
+}
+
 export class NodePtyModule implements NodePty {
-	public constructor(private readonly proxy: NodePtyModuleProxy) {}
+	public constructor(private readonly proxy: ClientNodePtyModuleProxy) {}
 
 	public spawn = (file: string, args: string[] | string, options: pty.IPtyForkOptions): pty.IPty => {
 		return new NodePtyProcess(this.proxy, file, args, options);

packages/protocol/src/browser/modules/spdlog.ts

@@ -1,12 +1,14 @@
 import * as spdlog from "spdlog";
-import { ClientProxy } from "../../common/proxy";
+import { ClientProxy, ClientServerProxy } from "../../common/proxy";
 import { RotatingLoggerProxy, SpdlogModuleProxy } from "../../node/modules/spdlog";
 
 // tslint:disable completed-docs
 
-class RotatingLogger extends ClientProxy<RotatingLoggerProxy> implements spdlog.RotatingLogger {
+interface ClientRotatingLoggerProxy extends RotatingLoggerProxy, ClientServerProxy {}
+
+class RotatingLogger extends ClientProxy<ClientRotatingLoggerProxy> implements spdlog.RotatingLogger {
 	public constructor(
-		private readonly moduleProxy: SpdlogModuleProxy,
+		private readonly moduleProxy: ClientSpdlogModuleProxy,
 		private readonly name: string,
 		private readonly filename: string,
 		private readonly filesize: number,
@@ -31,10 +33,14 @@ class RotatingLogger extends ClientProxy<RotatingLoggerProxy> implements spdlog.
 	}
 }
 
+interface ClientSpdlogModuleProxy extends SpdlogModuleProxy, ClientServerProxy {
+	createLogger(name: string, filePath: string, fileSize: number, fileCount: number): Promise<ClientRotatingLoggerProxy>;
+}
+
 export class SpdlogModule {
 	public readonly RotatingLogger: typeof spdlog.RotatingLogger;
 
-	public constructor(private readonly proxy: SpdlogModuleProxy) {
+	public constructor(private readonly proxy: ClientSpdlogModuleProxy) {
 		this.RotatingLogger = class extends RotatingLogger {
 			public constructor(name: string, filename: string, filesize: number, filecount: number) {
 				super(proxy, name, filename, filesize, filecount);