diff --git a/BUILD.gn b/BUILD.gn index 8b731cad2aa393..bfc65c05a60954 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -90,6 +90,7 @@ ts_sources = [ "js/metrics.ts", "js/mkdir.ts", "js/mock_builtin.js", + "js/msg_ring.ts", "js/net.ts", "js/os.ts", "js/permissions.ts", diff --git a/core/http_bench.rs b/core/http_bench.rs index 8e5a0e10cfd833..1da5fd604af920 100644 --- a/core/http_bench.rs +++ b/core/http_bench.rs @@ -58,6 +58,12 @@ impl HttpBench { shared32[INDEX_END] = 0; Self { shared32 } } + + #[allow(dead_code)] + fn records_reset(&mut self) { + self.shared32[INDEX_START] = 0; + self.shared32[INDEX_END] = 0; + } } fn idx(i: usize, off: usize) -> usize { @@ -80,7 +86,7 @@ impl Behavior for HttpBench { unimplemented!() } - fn recv( + fn dispatch( &mut self, record: Record, zero_copy_buf: deno_buf, @@ -130,11 +136,6 @@ impl Behavior for HttpBench { (is_sync, op) } - fn records_reset(&mut self) { - self.shared32[INDEX_START] = 0; - self.shared32[INDEX_END] = 0; - } - fn records_push(&mut self, record: Record) -> bool { debug!("push {:?}", record); let i = self.shared32[INDEX_END] as usize; diff --git a/core/isolate.rs b/core/isolate.rs index cdda7b8153de6d..ca8993bfb18852 100644 --- a/core/isolate.rs +++ b/core/isolate.rs @@ -45,10 +45,11 @@ pub trait Behavior { fn resolve(&mut self, specifier: &str, referrer: deno_mod) -> deno_mod; - fn recv(&mut self, record: R, zero_copy_buf: deno_buf) -> (bool, Box>); - - /// Clears the shared buffer. - fn records_reset(&mut self); + fn dispatch( + &mut self, + record: R, + zero_copy_buf: deno_buf, + ) -> (bool, Box>); /// Returns false if not enough room. fn records_push(&mut self, record: R) -> bool; @@ -113,9 +114,9 @@ impl> Isolate { let req_record = isolate.behavior.records_shift().unwrap(); - isolate.behavior.records_reset(); + // TODO assert shared buffer is empty. - let (is_sync, op) = isolate.behavior.recv(req_record, zero_copy_buf); + let (is_sync, op) = isolate.behavior.dispatch(req_record, zero_copy_buf); if is_sync { let res_record = op.wait().unwrap(); let push_success = isolate.behavior.records_push(res_record); @@ -319,7 +320,7 @@ impl> Future for Isolate { self.polled_recently = true; - self.behavior.records_reset(); + // TODO assert shared buffer is empty. let mut i = 0; while i != self.pending_ops.len() { @@ -375,22 +376,20 @@ mod tests { } struct TestBehavior { - recv_count: usize, + dispatch_count: usize, resolve_count: usize, push_count: usize, shift_count: usize, - reset_count: usize, mod_map: HashMap, } impl TestBehavior { fn new() -> Self { Self { - recv_count: 0, + dispatch_count: 0, resolve_count: 0, push_count: 0, shift_count: 0, - reset_count: 0, mod_map: HashMap::new(), } } @@ -409,12 +408,12 @@ mod tests { None } - fn recv( + fn dispatch( &mut self, _record: (), _zero_copy_buf: deno_buf, ) -> (bool, Box>) { - self.recv_count += 1; + self.dispatch_count += 1; (false, Box::new(futures::future::ok(()))) } @@ -426,10 +425,6 @@ mod tests { } } - fn records_reset(&mut self) { - self.reset_count += 1; - } - fn records_push(&mut self, _record: ()) -> bool { self.push_count += 1; true @@ -455,7 +450,7 @@ mod tests { main(); "#, )); - assert_eq!(isolate.behavior.recv_count, 2); + assert_eq!(isolate.behavior.dispatch_count, 2); } #[test] @@ -472,7 +467,7 @@ mod tests { libdeno.send(); "#, ).unwrap(); - assert_eq!(isolate.behavior.recv_count, 0); + assert_eq!(isolate.behavior.dispatch_count, 0); assert_eq!(isolate.behavior.resolve_count, 0); let imports = isolate.mod_get_imports(mod_a); @@ -485,16 +480,16 @@ mod tests { assert_eq!(imports.len(), 0); js_check(isolate.mod_instantiate(mod_b)); - assert_eq!(isolate.behavior.recv_count, 0); + assert_eq!(isolate.behavior.dispatch_count, 0); assert_eq!(isolate.behavior.resolve_count, 0); isolate.behavior.register("b.js", mod_b); js_check(isolate.mod_instantiate(mod_a)); - assert_eq!(isolate.behavior.recv_count, 0); + assert_eq!(isolate.behavior.dispatch_count, 0); assert_eq!(isolate.behavior.resolve_count, 1); js_check(isolate.mod_evaluate(mod_a)); - assert_eq!(isolate.behavior.recv_count, 1); + assert_eq!(isolate.behavior.dispatch_count, 1); assert_eq!(isolate.behavior.resolve_count, 1); } @@ -517,7 +512,7 @@ mod tests { } "#, )); - assert_eq!(isolate.behavior.recv_count, 0); + assert_eq!(isolate.behavior.dispatch_count, 0); js_check(isolate.execute( "check1.js", r#" @@ -526,9 +521,9 @@ mod tests { assertEq(nrecv, 0); "#, )); - assert_eq!(isolate.behavior.recv_count, 1); + assert_eq!(isolate.behavior.dispatch_count, 1); assert_eq!(Ok(Async::Ready(())), isolate.poll()); - assert_eq!(isolate.behavior.recv_count, 1); + assert_eq!(isolate.behavior.dispatch_count, 1); js_check(isolate.execute( "check2.js", r#" @@ -537,10 +532,10 @@ mod tests { assertEq(nrecv, 1); "#, )); - assert_eq!(isolate.behavior.recv_count, 2); + assert_eq!(isolate.behavior.dispatch_count, 2); assert_eq!(Ok(Async::Ready(())), isolate.poll()); js_check(isolate.execute("check3.js", "assertEq(nrecv, 2)")); - assert_eq!(isolate.behavior.recv_count, 2); + assert_eq!(isolate.behavior.dispatch_count, 2); // We are idle, so the next poll should be the last. assert_eq!(Ok(Async::Ready(())), isolate.poll()); } diff --git a/core/libdeno.rs b/core/libdeno.rs deleted file mode 120000 index 32688906e1553e..00000000000000 --- a/core/libdeno.rs +++ /dev/null @@ -1 +0,0 @@ -../src/libdeno.rs \ No newline at end of file diff --git a/core/libdeno.rs b/core/libdeno.rs new file mode 100644 index 00000000000000..8ef81f5592afff --- /dev/null +++ b/core/libdeno.rs @@ -0,0 +1,187 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use libc::c_char; +use libc::c_int; +use libc::c_void; +use libc::size_t; +use std::ops::{Deref, DerefMut}; +use std::ptr::null; + +// TODO(F001): change this definition to `extern { pub type isolate; }` +// After RFC 1861 is stablized. See https://github.com/rust-lang/rust/issues/43467. +#[repr(C)] +pub struct isolate { + _unused: [u8; 0], +} + +/// If "alloc_ptr" is not null, this type represents a buffer which is created +/// in C side, and then passed to Rust side by `deno_recv_cb`. Finally it should +/// be moved back to C side by `deno_respond`. If it is not passed to +/// `deno_respond` in the end, it will be leaked. +/// +/// If "alloc_ptr" is null, this type represents a borrowed slice. +#[repr(C)] +pub struct deno_buf { + alloc_ptr: *const u8, + alloc_len: usize, + data_ptr: *const u8, + data_len: usize, + pub zero_copy_id: usize, +} + +/// `deno_buf` can not clone, and there is no interior mutability. +/// This type satisfies Send bound. +unsafe impl Send for deno_buf {} + +impl deno_buf { + #[inline] + pub fn empty() -> Self { + Self { + alloc_ptr: null(), + alloc_len: 0, + data_ptr: null(), + data_len: 0, + zero_copy_id: 0, + } + } + + #[inline] + pub unsafe fn from_raw_parts(ptr: *const u8, len: usize) -> Self { + Self { + alloc_ptr: null(), + alloc_len: 0, + data_ptr: ptr, + data_len: len, + zero_copy_id: 0, + } + } +} + +/// Converts Rust &Buf to libdeno `deno_buf`. +impl<'a> From<&'a [u8]> for deno_buf { + #[inline] + fn from(x: &'a [u8]) -> Self { + Self { + alloc_ptr: null(), + alloc_len: 0, + data_ptr: x.as_ref().as_ptr(), + data_len: x.len(), + zero_copy_id: 0, + } + } +} + +impl Deref for deno_buf { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(self.data_ptr, self.data_len) } + } +} + +impl DerefMut for deno_buf { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + if self.alloc_ptr.is_null() { + panic!("Can't modify the buf"); + } + std::slice::from_raw_parts_mut(self.data_ptr as *mut u8, self.data_len) + } + } +} + +impl AsRef<[u8]> for deno_buf { + #[inline] + fn as_ref(&self) -> &[u8] { + &*self + } +} + +impl AsMut<[u8]> for deno_buf { + #[inline] + fn as_mut(&mut self) -> &mut [u8] { + if self.alloc_ptr.is_null() { + panic!("Can't modify the buf"); + } + &mut *self + } +} + +#[allow(non_camel_case_types)] +pub type deno_recv_cb = unsafe extern "C" fn( + user_data: *mut c_void, + control_buf: deno_buf, // deprecated + zero_copy_buf: deno_buf, +); + +#[allow(non_camel_case_types)] +pub type deno_mod = i32; + +#[allow(non_camel_case_types)] +type deno_resolve_cb = unsafe extern "C" fn( + user_data: *mut c_void, + specifier: *const c_char, + referrer: deno_mod, +) -> deno_mod; + +#[repr(C)] +pub struct deno_config { + pub will_snapshot: c_int, + pub load_snapshot: deno_buf, + pub shared: deno_buf, + pub recv_cb: deno_recv_cb, +} + +extern "C" { + pub fn deno_init(); + pub fn deno_v8_version() -> *const c_char; + pub fn deno_set_v8_flags(argc: *mut c_int, argv: *mut *mut c_char); + pub fn deno_new(config: deno_config) -> *const isolate; + pub fn deno_delete(i: *const isolate); + pub fn deno_last_exception(i: *const isolate) -> *const c_char; + pub fn deno_check_promise_errors(i: *const isolate); + pub fn deno_lock(i: *const isolate); + pub fn deno_unlock(i: *const isolate); + pub fn deno_respond( + i: *const isolate, + user_data: *const c_void, + buf: deno_buf, + ); + pub fn deno_zero_copy_release(i: *const isolate, zero_copy_id: usize); + pub fn deno_execute( + i: *const isolate, + user_data: *const c_void, + js_filename: *const c_char, + js_source: *const c_char, + ); + + // Modules + + pub fn deno_mod_new( + i: *const isolate, + main: bool, + name: *const c_char, + source: *const c_char, + ) -> deno_mod; + + pub fn deno_mod_imports_len(i: *const isolate, id: deno_mod) -> size_t; + + pub fn deno_mod_imports_get( + i: *const isolate, + id: deno_mod, + index: size_t, + ) -> *const c_char; + + pub fn deno_mod_instantiate( + i: *const isolate, + user_data: *const c_void, + id: deno_mod, + resolve_cb: deno_resolve_cb, + ); + + pub fn deno_mod_evaluate( + i: *const isolate, + user_data: *const c_void, + id: deno_mod, + ); +} diff --git a/core/shared.rs b/core/shared.rs deleted file mode 100644 index 40d83db732b173..00000000000000 --- a/core/shared.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::libdeno::deno_buf; -use std::mem; - -// TODO this is where we abstract flatbuffers at. -// TODO make these constants private to this file. -const INDEX_NUM_RECORDS: usize = 0; -const INDEX_RECORDS: usize = 1; -pub const RECORD_OFFSET_PROMISE_ID: usize = 0; -pub const RECORD_OFFSET_OP: usize = 1; -pub const RECORD_OFFSET_ARG: usize = 2; -pub const RECORD_OFFSET_RESULT: usize = 3; -const RECORD_SIZE: usize = 4; -const NUM_RECORDS: usize = 100; - -/// Represents the shared buffer between JS and Rust. -/// Used for FFI. -pub struct Shared(Vec); - -impl Shared { - pub fn new() -> Shared { - let mut vec = Vec::::new(); - vec.resize(INDEX_RECORDS + RECORD_SIZE * NUM_RECORDS, 0); - Shared(vec) - } - - pub fn set_record(&mut self, i: usize, off: usize, value: i32) { - assert!(i < NUM_RECORDS); - self.0[INDEX_RECORDS + RECORD_SIZE * i + off] = value; - } - - pub fn get_record(&self, i: usize, off: usize) -> i32 { - assert!(i < NUM_RECORDS); - return self.0[INDEX_RECORDS + RECORD_SIZE * i + off]; - } - - pub fn set_num_records(&mut self, num_records: i32) { - self.0[INDEX_NUM_RECORDS] = num_records; - } - - pub fn get_num_records(&self) -> i32 { - return self.0[INDEX_NUM_RECORDS]; - } - - pub fn as_deno_buf(&mut self) -> deno_buf { - let ptr = self.0.as_mut_ptr() as *mut u8; - let len = mem::size_of::() * self.0.len(); - unsafe { deno_buf::from_raw_parts(ptr, len) } - } -} diff --git a/js/dispatch.ts b/js/dispatch.ts index 53729187776163..7bd01f88fa985a 100644 --- a/js/dispatch.ts +++ b/js/dispatch.ts @@ -4,21 +4,19 @@ import * as flatbuffers from "./flatbuffers"; import * as msg from "gen/msg_generated"; import * as errors from "./errors"; import * as util from "./util"; +import * as msgRing from "./msg_ring"; let nextCmdId = 0; const promiseTable = new Map>(); -let fireTimers: () => void; - -export function setFireTimersCallback(fn: () => void): void { - fireTimers = fn; -} - -export function handleAsyncMsgFromRust(ui8: Uint8Array): void { - // If a the buffer is empty, recv() on the native side timed out and we - // did not receive a message. - if (ui8 && ui8.length) { - const bb = new flatbuffers.ByteBuffer(ui8); +export function handleAsyncMsgFromRust(_unused: Uint8Array): void { + util.log("handleAsyncMsgFromRust start"); + util.assert(_unused == null); + let i = 0; + let ui8: Uint8Array | null; + while ((ui8 = msgRing.rx.receive(Uint8Array)) !== null) { + util.log(`handleAsyncMsgFromRust ${i++}`); + const bb = new flatbuffers.ByteBuffer(ui8!); const base = msg.Base.getRootAsBase(bb); const cmdId = base.cmdId(); const promise = promiseTable.get(cmdId); @@ -31,8 +29,6 @@ export function handleAsyncMsgFromRust(ui8: Uint8Array): void { promise!.resolve(base); } } - // Fire timers that have become runnable. - fireTimers(); } function sendInternal( @@ -41,7 +37,7 @@ function sendInternal( inner: flatbuffers.Offset, data: undefined | ArrayBufferView, sync = true -): [number, null | Uint8Array] { +): number { const cmdId = nextCmdId++; msg.Base.startBase(builder); msg.Base.addInner(builder, inner); @@ -49,9 +45,15 @@ function sendInternal( msg.Base.addSync(builder, sync); msg.Base.addCmdId(builder, cmdId); builder.finish(msg.Base.endBase(builder)); - const res = libdeno.send(builder.asUint8Array(), data); + + const ui8 = builder.asUint8Array(); + msgRing.tx.send(ui8); + + // Somehow put this in the shared buffer. + + libdeno.send(null, data); builder.inUse = false; - return [cmdId, res]; + return cmdId; } // @internal @@ -61,8 +63,7 @@ export function sendAsync( inner: flatbuffers.Offset, data?: ArrayBufferView ): Promise { - const [cmdId, resBuf] = sendInternal(builder, innerType, inner, data, false); - util.assert(resBuf == null); + const cmdId = sendInternal(builder, innerType, inner, data, false); const promise = util.createResolvable(); promiseTable.set(cmdId, promise); return promise; @@ -75,13 +76,13 @@ export function sendSync( inner: flatbuffers.Offset, data?: ArrayBufferView ): null | msg.Base { - const [cmdId, resBuf] = sendInternal(builder, innerType, inner, data, true); + const cmdId = sendInternal(builder, innerType, inner, data, true); util.assert(cmdId >= 0); - if (resBuf == null) { + let resBuf: Uint8Array | null = msgRing.rx.receive(Uint8Array); + if (resBuf == null || resBuf.length === 0) { return null; } else { - const u8 = new Uint8Array(resBuf!); - const bb = new flatbuffers.ByteBuffer(u8); + const bb = new flatbuffers.ByteBuffer(resBuf); const baseRes = msg.Base.getRootAsBase(bb); errors.maybeThrowError(baseRes); return baseRes; diff --git a/js/libdeno.ts b/js/libdeno.ts index 33e9e4f5fd071b..485e67bb9e36d2 100644 --- a/js/libdeno.ts +++ b/js/libdeno.ts @@ -19,11 +19,14 @@ interface EvalErrorInfo { interface Libdeno { recv(cb: MessageCallback): void; - send(control: ArrayBufferView, data?: ArrayBufferView): null | Uint8Array; + send( + control: null | ArrayBufferView, + data?: ArrayBufferView + ): null | Uint8Array; print(x: string, isErr?: boolean): void; - shared: ArrayBuffer; + shared: SharedArrayBuffer; /** Evaluate provided code in the current context. * It differs from eval(...) in that it does not create a new context. diff --git a/js/msg_ring.ts b/js/msg_ring.ts new file mode 100644 index 00000000000000..c104d0916069c5 --- /dev/null +++ b/js/msg_ring.ts @@ -0,0 +1,552 @@ +import { libdeno } from "./libdeno"; + +// In the latest EcmaScript draft Atomics.wake has been renamed by +// Atomics.notify. The former is now deprecated and triggers a deprecation +// warning node.js. However TypeScript doesn't know about any of this. +// It's not clear to me how to declare an extra method on a global object, +// therefore this hack. +// TODO: fix this. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const Atomics: any; + +// The 'MsgRing' prefix is necessary to avoid a conflict with a less complete +// definition for TypedArray in a different module. +type MsgRingTypedArray = + | Int8Array + | Uint8Array + | Uint8ClampedArray + | Int16Array + | Uint16Array + | Int32Array + | Uint32Array + | Float32Array + | Float64Array; + +export interface TypedArrayConstructor { + new (arrayOrArrayBuffer: ArrayLike | ArrayBufferLike): T; + new (buffer: ArrayBufferLike, byteOffset: number, length?: number): T; + readonly BYTES_PER_ELEMENT: number; +} + +export interface Slice { + byteOffset: number; + byteLength: number; +} + +export interface MsgRingCounters { + role: string; + message: number; + acquire: number; + release: number; + spin: number; + wait: number; + notify: number; + wrap: number; +} + +export const enum FillDirection { + TopDown, + BottomUp +} + +export let rx!: MsgRingReceiver; +export let tx!: MsgRingSender; + +export function init(): void { + if (libdeno.shared.byteLength === 0) { + // The message ring should not be accessed before a snapshot has been + // created. This is because the size of the shared buffer gets embedded in + // the snapshot which is zero when snapshotting. + // TODO: move shared buffer creation to libdeno, make the size fixed + // (non-configurable), replace c++ getter by fixed property. + throw new Error("Shared buffer can't be used before snapshotting."); + } + + // The SharedArrayBuffer in split half. First half is for the sender, second + // half is for the receiver. + const half = libdeno.shared.byteLength / 2; + const commonConfig = { + byteLength: half + }; + // eslint-disable-next-line @typescript-eslint/no-use-before-define + rx = new MsgRingReceiver(libdeno.shared, { + byteOffset: 0, + ...commonConfig + }); + // eslint-disable-next-line @typescript-eslint/no-use-before-define + tx = new MsgRingSender(libdeno.shared, { + byteOffset: half, + ...commonConfig + }); +} + +// The WaitFn and NotifyFn functions have the same signatures as Atomics.wait +// and Atomics.notify, with the exception that Atomics.notify returns the number +// of threads woken, whereas NotifyFn doesn't return anything at all. +export type WaitResult = "ok" | "not-equal" | "timed-out"; +export type WaitFn = ( + i32: Int32Array, + offset: number, + value: number, + timeout: number +) => WaitResult; +export type NotifyFn = (i32: Int32Array, offset: number, count: number) => void; + +type OptionsObject = { -readonly [P in keyof T]?: T[P] }; +export type MsgRingConfig = OptionsObject; + +const enum FrameAllocation { + // Placeholder value indicating the absence of an allocation. + None = 0, + // Alignment (in bytes) of frame offset and frame length. + Alignment = 4, + // Length of frame header. Note: only 4 bytes are currently used. + HeaderByteLength = 4 +} + +// prettier-ignore +const enum FrameHeader { + // Do not use the highest bit. On 64-bit systems, 32-bit signed ints are very + // efficient in v8, but numbers greater than 2**31-1 are heap allocated. + + // Placeholder value indicating the absence of a header. + None = 0x00000000, + // Low 24 bits are reserved for the length of the frame (including header). + ByteLengthMask = 0x00ffffff, + // The `epoch` serves as a filter for which frames are ready to be acquired. + EpochMask = 0x03000000, + // Initial epoch values for respectively the sender and the receiver. + EpochInitSender = 0x00000000, + EpochInitReceiver = 0x01000000, + // Every time a frame changes hands between receiver and sender, we add 1 to + // the 2-bit epoch number; when the buffer wraps, add 2. + EpochIncrementPass = 0x01000000, + EpochIncrementWrap = 0x02000000, + // Flag that indicates to receiver that a frame contains a message. If it + // doesn't, that's due to insufficient space at the end of the buffer. + HasMessageFlag = 0x04000000, + // Flag that indicates that there are waiter(s) that expect to be notified. + HasWaitersFlag = 0x08000000, +} + +abstract class MsgRingDefaultConfig { + // The slice of the SharedArrayBuffer that this sender/receiver will use. + readonly byteOffset: number = 0; + readonly byteLength: number | null = null; + + // Whether buffers are filled upwards or downwards. + readonly fillDirection: FillDirection = FillDirection.TopDown; + + // The maximum number of times acquireFrame() will spin before sleeping. + readonly spinCount: number = 4000; + + // When spinning, for how long the thread yields the CPU on each cycle, in + // milliseconds. Yielding happens by calling Atomics.wait() with a time-out. + // Set to zero to never yield the CPU while spinning. + readonly spinYieldCpuTime: number = 0; + + // By default the ecmascript standard `Atomics.wait` and `Atomics.notify` + // functions are used, but an alternative implementation may be provided. + readonly wait: WaitFn = Atomics.wait; + readonly notify: NotifyFn = Atomics.notify; +} + +abstract class MsgRingCommon extends MsgRingDefaultConfig { + // We'll create some (public) views on the underlying buffer. It's much + // faster to recycle these than to create them on the spot every time, and + // also much faster than using a DataView. They are public so whoever needs + // to receive/send a message payload can use them too. + public readonly u8: Uint8Array; + public readonly i32: Int32Array; + public readonly u32: Uint32Array; + + // MsgRingCommon inherits the `byteLength` property from MsgRingConfig, where + // it may be null. Logic in the constructor ensures it's always an integer. + // Also, caching the buffer length because it's much faster than accessing + // buffer.byteLength. This is quite surprising -- I suspect that + // SharedArrayBuffer.byteLength incurs a runtime call, possibly to check + // whether the buffer has been neutered.) + public readonly byteLength!: number; + + // The maximum message size of a message. + public readonly maxMessageByteLength: number; + + // These constant properties are used to adjust adjust offsets based on the + // fill direction of the buffer. + private readonly fillDirectionBaseAdjustment: 0 | 1; + private readonly fillDirectionOffsetAdjustment: 1 | -1; + + // A frame's epoch identifies which role (receiver or sender) released a + // frame, and whether the buffer has since wrapped around. This epoch field + // tracks which epoch acquireFrame() will currently acquire frames from. + // Initialization is role-dependent, so it's done by our subclasses. + protected epoch!: number; + + // The head and tail position of the 'window': the range of bytes locked by a + // sender/receiver. The value indicates the offset in bytes from the start of + // ring buffer, ***NOT*** adjusted for fill direction of the buffer. Hence + // the initial (empty) window always starts and ends at position 0. + protected windowHeadPosition = 0; + protected windowTailPosition = 0; + + // The window's byte length and whether it's at the end of the buffer can be + // computed from the head and tail position. However we store them because + // they are used often. + protected windowByteLength = 0; + protected windowIsAtEndOfBuffer = false; + + // Counters for debugging. + protected messageCounter = 0; + private acquireCounter = 0; + private releaseCounter = 0; + private wrapCounter = 0; + private waitCounter = 0; + private notifyCounter = 0; + private spinCounter = 0; + + // `buffer` must be initialized with zeroes. + constructor(readonly buffer: SharedArrayBuffer, options: MsgRingConfig = {}) { + // Initialize (default) options. + super(); + Object.assign(this, options); + + // Set fill direction adjustment constants. + switch (this.fillDirection) { + case FillDirection.TopDown: + this.fillDirectionOffsetAdjustment = 1; + this.fillDirectionBaseAdjustment = 0; + break; + case FillDirection.BottomUp: + this.fillDirectionOffsetAdjustment = -1; + this.fillDirectionBaseAdjustment = 1; + break; + default: + throw new Error("Invalid fill direction."); + } + + if (this.byteLength == null) { + this.byteLength = buffer.byteLength - this.byteOffset; + } + + this.maxMessageByteLength = + (Math.min(FrameHeader.ByteLengthMask, this.byteLength) - + FrameAllocation.HeaderByteLength) & + ~(FrameAllocation.Alignment - 1); + + // Create various views on the SharedArrayBuffer. + this.u8 = new Uint8Array(buffer); + this.i32 = new Int32Array(buffer); + this.u32 = new Uint32Array(buffer); + + // Initialize the frame structure inside the buffer. + // We expect the buffer to be initialized with zeroes. If that's the + // case, define a frame that spans the entire buffer and place it's header + // at offset 0, so the receiver and sender don't get confused. + // Since the other user (receiver/sender) may have gotten here first, only + // do the initializtion if the first slot is still contains zero. + Atomics.compareExchange( + this.i32, + this.getHeaderI32Offset(0), + 0, + this.byteLength + ); + } + + get counters(): MsgRingCounters { + return { + role: this.constructor.name, + message: this.messageCounter, + acquire: this.acquireCounter, + release: this.releaseCounter, + wrap: this.wrapCounter, + wait: this.waitCounter, + notify: this.notifyCounter, + spin: this.spinCounter + }; + } + + protected assert(condition: boolean): void { + if (!condition) { + throw new Error(`${this.constructor.name}: assertion failed.`); + } + } + + protected acquireFrame(wait = true): number { + // Wrap around if the current head position is at the end of the buffer. + if (this.windowIsAtEndOfBuffer) { + // A frame can't wrap around the end of the buffer; if the current window + // is at the end of the buffer, the caller should release the remaining + // bytes before attempting to grow the window. + this.assert(this.windowByteLength === 0); + + // Increment the epoch number. Note that the epoch number itself wraps + // around on overflow, this is intentional. + this.epoch = + (this.epoch + FrameHeader.EpochIncrementWrap) & FrameHeader.EpochMask; + + // Rewind the current frame to the start of the ring buffer. + this.windowHeadPosition = 0; + this.windowTailPosition = 0; + this.windowIsAtEndOfBuffer = false; + this.wrapCounter++; + } + + const headerI32Offset = this.getHeaderI32Offset(this.windowHeadPosition); + let header = this.i32[headerI32Offset]; + + // let spinCountRemaining: number = this.spinCount; + // let futexWaitTime: number = this.spinYieldCpuTime; + + while ((header & FrameHeader.EpochMask) !== this.epoch) { + // Sleep nor spin when acquiring in non-blocking mode. + if (!wait) { + return FrameHeader.None; + } + + throw new Error("Failed to acquire. Can't block on ring."); + /* + if (spinCountRemaining === 0) { + // We're going to put the thread to sleep. + // Use compare-and-swap to set the kHasWaiters flag. + const expect = header; + const target = header | FrameHeader.HasWaitersFlag; + header = Atomics.compareExchange( + this.i32, + headerI32Offset, + expect, + target + ); + if (expect !== header) { + // The buffer slot that holds the header has been modified after we + // last receive it; compareExchange did not set the flag, but `header` + // is now up-to-date again. + continue; + } + header = target; + futexWaitTime = Infinity; + this.waitCounter++; + } else { + // We still have spins left. + spinCountRemaining--; + this.spinCounter++; //} + + // If we're spinning and CPU yielding is enabled, we'll call futexWait + // just as as if we were going to sleep, but with some very small time-out + // value. If yielding is off, just refresh `header` from the ring buffer. + if ( + futexWaitTime <= 0 || + this.wait(this.i32, headerI32Offset, header, futexWaitTime) !== + "timed-out" + ) { + // If futexWait returned "ok" or "not-equal", the value in the buffer + // is different from our local copy, so refresh it. + header = Atomics.load(this.i32, headerI32Offset); + } + */ + } + + const byteLength = header & FrameHeader.ByteLengthMask; + this.assert(byteLength <= this.byteLength - this.windowHeadPosition); + + this.windowHeadPosition += byteLength; + this.windowByteLength += byteLength; + this.windowIsAtEndOfBuffer = this.windowHeadPosition === this.byteLength; + this.acquireCounter++; + + return header; + } + + protected releaseFrame(byteLength: number, flags = FrameHeader.None): void { + this.assert(byteLength >= FrameAllocation.HeaderByteLength); + this.assert(byteLength <= this.windowByteLength); + + if (byteLength < this.windowByteLength) { + // Place a temporary header for which what will become the next message + // right after the message. + const nextHeaderI32Offset = this.windowTailPosition + this.byteLength; + // TODO: Do away with epoch concept, just use zero always. + this.i32[nextHeaderI32Offset] = this.epoch; + } + + const tailEpoch = this.epoch + FrameHeader.EpochIncrementPass; + const newHeader = byteLength | flags | tailEpoch; + + const headerI32Offset = this.getHeaderI32Offset(this.windowTailPosition); + // const oldHeader = Atomics.exchange(this.i32, headerI32Offset, newHeader); + const oldHeader = this.i32[headerI32Offset]; + this.i32[headerI32Offset] = newHeader; + + if (oldHeader & FrameHeader.HasWaitersFlag) { + this.notify(this.i32, headerI32Offset, 1); + this.notifyCounter++; + } + + this.windowTailPosition += byteLength; + this.windowByteLength -= byteLength; + } + + // Returns the byte offset of a frame header, adjusted for the fill direction + // of the buffer, given it's position. + protected getHeaderI32Offset(position: number): number { + const headerByteOffset: number = + this.byteOffset + + this.fillDirectionBaseAdjustment * + (this.byteLength - FrameAllocation.HeaderByteLength) + + this.fillDirectionOffsetAdjustment * position; + return headerByteOffset / this.i32.BYTES_PER_ELEMENT; + } + + // Creates a Slice object, given the byte length of the encapsulating frame. + protected getMessageSlice(frameByteLength: number): Slice { + // Compute the length of the message itself. Note that when writing a + // message, it's length is always rounded up to match the alignment. + const messageByteLength = + frameByteLength - FrameAllocation.HeaderByteLength; + + // Compute the fill-direction adjusted offset of the message payload. + const messageByteOffset = + this.byteOffset + + this.fillDirectionBaseAdjustment * (this.byteLength - messageByteLength) + + this.fillDirectionOffsetAdjustment * + (this.windowTailPosition + FrameAllocation.HeaderByteLength); + + return { + byteOffset: messageByteOffset, + byteLength: messageByteLength + }; + } +} + +export class MsgRingSender extends MsgRingCommon { + protected epoch: number = FrameHeader.EpochInitSender; + + // Number of bytes allocated by beginSend()/resizeSend(). It includes space + // for the frame header and padding for alignment + private allocationByteLength: number = FrameAllocation.None; + + // Note: byteLength will be rounded up to alignment. + beginSend(messageByteLength: number): Slice { + if (this.allocationByteLength !== FrameAllocation.None) { + throw new Error("Already writing."); + } + this.allocate(messageByteLength); + return this.getMessageSlice(this.allocationByteLength); + } + + // Note: byteLength will be rounded up to alignment. + // Noto: already-written data is discarded when buffer wraps. + // TODO: copy bytes when allocation wraps. + resizeSend(messageByteLength: number): Slice { + if (this.allocationByteLength === FrameAllocation.None) { + throw new Error("Not writing."); + } + this.allocate(messageByteLength); + return this.getMessageSlice(this.allocationByteLength); + } + + endSend(submit = true): void { + if (this.allocationByteLength === FrameAllocation.None) { + throw new Error("Not writing."); + } + if (submit) { + // Release a frame that contains the header plus message. + this.releaseFrame(this.allocationByteLength, FrameHeader.HasMessageFlag); + this.messageCounter++; + } + this.allocationByteLength = FrameAllocation.None; + } + + send(data: ArrayBufferView): void { + // Convert `data` to an Uint8Array view if necessary. + const u8data: Uint8Array = + data instanceof Uint8Array + ? data + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + // Allocate space. + const target = this.beginSend(data.byteLength); + // Copy data. + this.u8.set(u8data, target.byteOffset); + // Close the send. + this.endSend(); + } + + private allocate(messageByteLength: number): void { + if (messageByteLength > this.maxMessageByteLength) { + throw new RangeError("Slice too big."); + } else if (messageByteLength < 0) { + throw new RangeError("Slice must have positive byte length."); + } + + // Compute the total required length, including header and padding, + this.allocationByteLength = + FrameAllocation.HeaderByteLength + this.align(messageByteLength); + + while (this.windowByteLength < this.allocationByteLength) { + // An allocation can't wrap around the end of the ring buffer. + if (this.windowIsAtEndOfBuffer && this.windowByteLength > 0) { + // Discard the allocation we've made so far. The allocation process will + // restart at the beginning of the ring buffer. + this.releaseFrame(this.windowByteLength); + } + // Consume the next frame to get closer to the target allocation length. + this.acquireFrame(); + } + } + + private align(byteCount: number): number { + const alignmentMask = FrameAllocation.Alignment - 1; + return (byteCount + alignmentMask) & ~alignmentMask; + } +} + +export class MsgRingReceiver extends MsgRingCommon { + protected epoch: number = FrameHeader.EpochInitReceiver; + + beginReceive(): Slice | null { + if (this.windowByteLength !== FrameAllocation.None) { + throw new Error("Already receiving."); + } + let skipped = false; + for (;;) { + let header = this.acquireFrame(false); + if (header == FrameHeader.None) { + return null; + } + if (header & FrameHeader.HasMessageFlag) { + break; + } + this.releaseFrame(this.windowByteLength); + this.assert(!skipped); + skipped = true; + } + return this.getMessageSlice(this.windowByteLength); + } + + endReceive(): void { + if (this.windowByteLength === FrameAllocation.None) { + throw new Error("Not receiving."); + } + this.releaseFrame(this.windowByteLength); + this.messageCounter++; + } + + receive( + ctor: TypedArrayConstructor + ): T | null { + const messageSlice = this.beginReceive(); + if (messageSlice === null) { + return null; + } + // Create a view of the the requested type on the ring's backing buffer. + // TODO: This is slow (>2x slowdown); find a more efficient solution. + const view: T = new ctor( + this.buffer, + messageSlice.byteOffset, + messageSlice.byteLength / ctor.BYTES_PER_ELEMENT + ); + // Copy the view, implicitly creating a new allocation backing buffer. + const copy = new ctor(view); + this.endReceive(); + return copy; + } +} diff --git a/js/os.ts b/js/os.ts index f95114f0491582..319fd38b71c625 100644 --- a/js/os.ts +++ b/js/os.ts @@ -6,6 +6,7 @@ import { libdeno } from "./libdeno"; import { TextDecoder } from "./text_encoding"; import { assert } from "./util"; import * as util from "./util"; +import * as msgRing from "./msg_ring"; /** The current process id of the runtime. */ export let pid: number; @@ -169,6 +170,7 @@ function sendStart(): msg.StartRes { // the runtime and the compiler environments. // @internal export function start(source?: string): msg.StartRes { + msgRing.init(); libdeno.recv(handleAsyncMsgFromRust); // First we send an empty `Start` message to let the privileged side know we diff --git a/js/timers.ts b/js/timers.ts index d06056cf2a5de5..4089d5f8b08224 100644 --- a/js/timers.ts +++ b/js/timers.ts @@ -2,7 +2,7 @@ import { assert } from "./util"; import * as msg from "gen/msg_generated"; import * as flatbuffers from "./flatbuffers"; -import { sendSync, setFireTimersCallback } from "./dispatch"; +import { sendAsync, sendSync } from "./dispatch"; interface Timer { id: number; @@ -37,28 +37,39 @@ function getTime(): number { return now; } -function setGlobalTimeout(due: number | null, now: number): void { +function clearGlobalTimeout(): void { + const builder = flatbuffers.createBuilder(); + msg.GlobalTimerStop.startGlobalTimerStop(builder); + const inner = msg.GlobalTimerStop.endGlobalTimerStop(builder); + globalTimeoutDue = null; + let res = sendSync(builder, msg.Any.GlobalTimerStop, inner); + assert(res == null); +} + +async function setGlobalTimeout(due: number, now: number): Promise { // Since JS and Rust don't use the same clock, pass the time to rust as a // relative time value. On the Rust side we'll turn that into an absolute // value again. - // Note that a negative time-out value stops the global timer. - let timeout; - if (due === null) { - timeout = -1; - } else { - timeout = due - now; - assert(timeout >= 0); - } + let timeout = due - now; + assert(timeout >= 0); // Send message to the backend. const builder = flatbuffers.createBuilder(); - msg.SetTimeout.startSetTimeout(builder); - msg.SetTimeout.addTimeout(builder, timeout); - const inner = msg.SetTimeout.endSetTimeout(builder); - const res = sendSync(builder, msg.Any.SetTimeout, inner); - assert(res == null); - // Remember when when the global timer will fire. + msg.GlobalTimer.startGlobalTimer(builder); + msg.GlobalTimer.addTimeout(builder, timeout); + const inner = msg.GlobalTimer.endGlobalTimer(builder); globalTimeoutDue = due; + await sendAsync(builder, msg.Any.GlobalTimer, inner); + // eslint-disable-next-line @typescript-eslint/no-use-before-define + fireTimers(); +} + +function setOrClearGlobalTimeout(due: number | null, now: number): void { + if (due == null) { + clearGlobalTimeout(); + } else { + setGlobalTimeout(due, now); + } } function schedule(timer: Timer, now: number): void { @@ -75,7 +86,7 @@ function schedule(timer: Timer, now: number): void { // If the new timer is scheduled to fire before any timer that existed before, // update the global timeout to reflect this. if (globalTimeoutDue === null || globalTimeoutDue > timer.due) { - setGlobalTimeout(timer.due, now); + setOrClearGlobalTimeout(timer.due, now); } } @@ -97,7 +108,7 @@ function unschedule(timer: Timer): void { nextTimerDue = Number(key); break; } - setGlobalTimeout(nextTimerDue, getTime()); + setOrClearGlobalTimeout(nextTimerDue, getTime()); } } else { // Multiple timers that are due at the same point in time. @@ -162,9 +173,10 @@ function fireTimers(): void { Promise.resolve(timer).then(fire); } } + // Update the global alarm to go off when the first-up timer that hasn't fired // yet is due. - setGlobalTimeout(nextTimerDue, now); + setOrClearGlobalTimeout(nextTimerDue, now); } export type Args = unknown[]; @@ -226,7 +238,7 @@ export function setInterval( return setTimer(cb, delay, args, true); } -/** Clears a previously set timer by id. */ +/** Clears a previously set timer by id. AKA clearTimeout and clearInterval. */ export function clearTimer(id: number): void { const timer = idMap.get(id); if (timer === undefined) { @@ -237,7 +249,3 @@ export function clearTimer(id: number): void { unschedule(timer); idMap.delete(timer.id); } - -// Tell the dispatcher which function it should call to fire timers that are -// due. This is done using a callback because circular imports are disallowed. -setFireTimersCallback(fireTimers); diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 00000000000000..3b2ecdf277947f --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,145 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +#![allow(unused_variables)] +#![allow(dead_code)] + +use crate::errors::DenoResult; +use crate::isolate_init::IsolateInit; +use crate::isolate_state::IsolateState; +use crate::msg_ring; +use crate::ops; +use crate::permissions::DenoPermissions; +use deno_core::deno_buf; +use deno_core::deno_mod; +use deno_core::Behavior; +use deno_core::Op; +use std::cell::Cell; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Instant; + +// Buf represents a byte array returned from a "Op". The message might be empty +// (which will be translated into a null object on the javascript side) or it is +// a heap allocated opaque sequence of bytes. Usually a flatbuffer message. +pub type Buf = Box<[u8]>; + +pub type CliOp = Op; + +/// Implements deno_core::Behavior for the main Deno command-line. +pub struct Cli { + shared: Option, // Pin? + tx: msg_ring::Sender, + rx: msg_ring::Receiver, + init: IsolateInit, + timeout_due: Cell>, + pub state: Arc, + pub permissions: Arc, // TODO(ry) move to IsolateState +} + +impl Cli { + pub fn new( + init: IsolateInit, + state: Arc, + permissions: DenoPermissions, + ) -> Self { + let buffer = msg_ring::Buffer::new(8 * 1024 * 1024); + let shared = buffer.into_deno_buf(); + let (tx_buffer, rx_buffer) = buffer.split(); + let (tx, _) = msg_ring::MsgRing::new(tx_buffer).split(); + let (_, rx) = msg_ring::MsgRing::new(rx_buffer).split(); + Self { + init, + shared: Some(shared), + tx, + rx, + timeout_due: Cell::new(None), + state, + permissions: Arc::new(permissions), + } + } + + #[inline] + pub fn get_timeout_due(&self) -> Option { + self.timeout_due.clone().into_inner() + } + + #[inline] + pub fn set_timeout_due(&self, inst: Option) { + self.timeout_due.set(inst); + } + + #[inline] + pub fn check_read(&self, filename: &str) -> DenoResult<()> { + self.permissions.check_read(filename) + } + + #[inline] + pub fn check_write(&self, filename: &str) -> DenoResult<()> { + self.permissions.check_write(filename) + } + + #[inline] + pub fn check_env(&self) -> DenoResult<()> { + self.permissions.check_env() + } + + #[inline] + pub fn check_net(&self, filename: &str) -> DenoResult<()> { + self.permissions.check_net(filename) + } + + #[inline] + pub fn check_run(&self) -> DenoResult<()> { + self.permissions.check_run() + } +} + +impl Behavior for Cli { + fn startup_snapshot(&mut self) -> Option { + self.init.snapshot.take() + } + + fn startup_shared(&mut self) -> Option { + self.shared.take() + } + + fn resolve(&mut self, specifier: &str, referrer: deno_mod) -> deno_mod { + self + .state + .metrics + .resolve_count + .fetch_add(1, Ordering::Relaxed); + let mut modules = self.state.modules.lock().unwrap(); + modules.resolve_cb(&self.state.dir, specifier, referrer) + } + + fn dispatch( + &mut self, + control: Buf, + zero_copy: deno_buf, + ) -> (bool, Box) { + ops::dispatch(self, control, zero_copy) + } + + fn records_push(&mut self, record: Buf) -> bool { + let maybe_msg = self.tx.compose(record.len()); + if let Some(mut msg) = maybe_msg { + msg.copy_from_slice(&record); + msg.send(); + debug!("compose ok"); + true + } else { + debug!("compose fail"); + false + } + } + + fn records_shift(&mut self) -> Option { + self.rx.receive().map(|msg| { + let mut v = Vec::new(); + v.resize(msg.len(), 0); + let mut bs = v.into_boxed_slice(); + bs.copy_from_slice(&msg[..]); + bs + }) + } +} diff --git a/src/compiler.rs b/src/compiler.rs index 5fe335c551de5e..35656358ec1e3b 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1,19 +1,17 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -use crate::isolate::Buf; -use crate::isolate::IsolateState; +use crate::cli::Buf; use crate::isolate_init; +use crate::isolate_state::IsolateState; use crate::msg; use crate::permissions::DenoPermissions; use crate::resources; use crate::resources::Resource; use crate::resources::ResourceId; use crate::workers; - use futures::Future; use serde_json; use std::str; use std::sync::atomic::AtomicBool; -use std::sync::Arc; use std::sync::Mutex; lazy_static! { @@ -49,7 +47,7 @@ impl ModuleMetaData { } } -fn lazy_start(parent_state: &Arc) -> Resource { +fn lazy_start(parent_state: &IsolateState) -> Resource { let mut cell = C_RID.lock().unwrap(); let isolate_init = isolate_init::compiler_isolate_init(); let permissions = DenoPermissions { @@ -59,10 +57,11 @@ fn lazy_start(parent_state: &Arc) -> Resource { allow_net: AtomicBool::new(true), allow_run: AtomicBool::new(false), }; + let rid = cell.get_or_insert_with(|| { let resource = workers::spawn( isolate_init, - parent_state.clone(), + parent_state, "compilerMain()".to_string(), permissions, ); @@ -81,7 +80,7 @@ fn req(specifier: &str, referrer: &str) -> Buf { } pub fn compile_sync( - parent_state: &Arc, + parent_state: &IsolateState, specifier: &str, referrer: &str, module_meta_data: &ModuleMetaData, @@ -94,7 +93,9 @@ pub fn compile_sync( send_future.wait().unwrap(); let recv_future = resources::worker_recv_message(compiler.rid); - let res_msg = recv_future.wait().unwrap().unwrap(); + let result = recv_future.wait().unwrap(); + assert!(result.is_some()); + let res_msg = result.unwrap(); let res_json = std::str::from_utf8(&res_msg).unwrap(); match serde_json::from_str::(res_json) { diff --git a/src/global_timer.rs b/src/global_timer.rs new file mode 100644 index 00000000000000..eef70ddc20496d --- /dev/null +++ b/src/global_timer.rs @@ -0,0 +1,49 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +//! This module helps deno implement timers. +//! +//! As an optimization, we want to avoid an expensive calls into rust for every +//! setTimeout in JavaScript. Thus in //js/timers.ts a data structure is +//! implemented that calls into Rust for only the smallest timeout. Thus we +//! only need to be able to start and cancel a single timer (or Delay, as Tokio +//! calls it) for an entire Isolate. This is what is implemented here. + +use crate::tokio_util::panic_on_error; +use futures::Future; +use std::time::Instant; +use tokio::sync::oneshot; +use tokio::timer::Delay; + +pub struct GlobalTimer { + tx: Option>, +} + +impl GlobalTimer { + pub fn new() -> Self { + Self { tx: None } + } + + pub fn cancel(&mut self) { + if let Some(tx) = self.tx.take() { + tx.send(()).ok(); + } + } + + pub fn new_timeout( + &mut self, + deadline: Instant, + ) -> impl Future { + if self.tx.is_some() { + self.cancel(); + } + assert!(self.tx.is_none()); + + let (tx, rx) = oneshot::channel(); + self.tx = Some(tx); + + let delay = panic_on_error(Delay::new(deadline)); + let rx = panic_on_error(rx); + + delay.select(rx).then(|_| Ok(())) + } +} diff --git a/src/isolate.rs b/src/isolate.rs index d4f0f25399ed45..b7b25f3709026c 100644 --- a/src/isolate.rs +++ b/src/isolate.rs @@ -1,273 +1,35 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -// Do not use FlatBuffers in this module. -// TODO Currently this module uses Tokio, but it would be nice if they were -// decoupled. - -#![allow(dead_code)] - +use crate::cli::Buf; +use crate::cli::Cli; use crate::compiler::compile_sync; use crate::compiler::ModuleMetaData; -use crate::deno_dir; use crate::errors::DenoError; -use crate::errors::DenoResult; use crate::errors::RustOrJsError; -use crate::flags; -use crate::isolate_init::IsolateInit; -use crate::js_errors::apply_source_map; -use crate::libdeno; -use crate::modules::Modules; +use crate::isolate_state::IsolateState; +use crate::js_errors; use crate::msg; -use crate::permissions::DenoPermissions; -use crate::tokio_util; +use deno_core; +use deno_core::deno_mod; use deno_core::JSError; -use futures::sync::mpsc as async_mpsc; +use futures::Async; use futures::Future; -use libc::c_char; -use libc::c_void; -use std; -use std::cell::Cell; -use std::cell::RefCell; -use std::env; -use std::ffi::CStr; -use std::ffi::CString; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::mpsc; use std::sync::Arc; -use std::sync::Mutex; -use std::sync::{Once, ONCE_INIT}; -use std::time::Duration; -use std::time::Instant; -use tokio; - -// Buf represents a byte array returned from a "Op". -// The message might be empty (which will be translated into a null object on -// the javascript side) or it is a heap allocated opaque sequence of bytes. -// Usually a flatbuffer message. -pub type Buf = Box<[u8]>; - -// JS promises in Deno map onto a specific Future -// which yields either a DenoError or a byte array. -pub type Op = dyn Future + Send; - -// Returns (is_sync, op) -pub type Dispatch = fn( - isolate: &Isolate, - buf: libdeno::deno_buf, - zero_copy_buf: libdeno::deno_buf, -) -> (bool, Box); - -pub struct Isolate { - libdeno_isolate: *const libdeno::isolate, - dispatch: Dispatch, - rx: mpsc::Receiver<(usize, Buf)>, - tx: mpsc::Sender<(usize, Buf)>, - ntasks: Cell, - timeout_due: Cell>, - pub modules: RefCell, - pub state: Arc, - pub permissions: Arc, -} - -pub type WorkerSender = async_mpsc::Sender; -pub type WorkerReceiver = async_mpsc::Receiver; -pub type WorkerChannels = (WorkerSender, WorkerReceiver); - -// Isolate cannot be passed between threads but IsolateState can. -// IsolateState satisfies Send and Sync. -// So any state that needs to be accessed outside the main V8 thread should be -// inside IsolateState. -#[cfg_attr(feature = "cargo-clippy", allow(stutter))] -pub struct IsolateState { - pub dir: deno_dir::DenoDir, - pub argv: Vec, - pub flags: flags::DenoFlags, - pub metrics: Metrics, - pub worker_channels: Option>, -} - -impl IsolateState { - pub fn new( - flags: flags::DenoFlags, - argv_rest: Vec, - worker_channels: Option, - ) -> Self { - let custom_root = env::var("DENO_DIR").map(|s| s.into()).ok(); - - Self { - dir: deno_dir::DenoDir::new(flags.reload, flags.recompile, custom_root) - .unwrap(), - argv: argv_rest, - flags, - metrics: Metrics::default(), - worker_channels: worker_channels.map(Mutex::new), - } - } - - pub fn main_module(&self) -> Option { - if self.argv.len() <= 1 { - None - } else { - let specifier = self.argv[1].clone(); - let referrer = "."; - match self.dir.resolve_module_url(&specifier, referrer) { - Ok(url) => Some(url.to_string()), - Err(e) => { - debug!("Potentially swallowed error {}", e); - None - } - } - } - } - - #[cfg(test)] - pub fn mock() -> Arc { - let argv = vec![String::from("./deno"), String::from("hello.js")]; - // For debugging: argv.push_back(String::from("-D")); - let (flags, rest_argv, _) = flags::set_flags(argv).unwrap(); - Arc::new(IsolateState::new(flags, rest_argv, None)) - } - - fn metrics_op_dispatched( - &self, - bytes_sent_control: usize, - bytes_sent_data: usize, - ) { - self.metrics.ops_dispatched.fetch_add(1, Ordering::SeqCst); - self - .metrics - .bytes_sent_control - .fetch_add(bytes_sent_control, Ordering::SeqCst); - self - .metrics - .bytes_sent_data - .fetch_add(bytes_sent_data, Ordering::SeqCst); - } - fn metrics_op_completed(&self, bytes_received: usize) { - self.metrics.ops_completed.fetch_add(1, Ordering::SeqCst); - self - .metrics - .bytes_received - .fetch_add(bytes_received, Ordering::SeqCst); - } -} +type CoreIsolate = deno_core::Isolate; -// AtomicU64 is currently unstable -#[derive(Default)] -pub struct Metrics { - pub ops_dispatched: AtomicUsize, - pub ops_completed: AtomicUsize, - pub bytes_sent_control: AtomicUsize, - pub bytes_sent_data: AtomicUsize, - pub bytes_received: AtomicUsize, - pub resolve_count: AtomicUsize, +/// Wraps deno_core::Isolate to provide source maps, ops for the CLI, and +/// high-level module loading +pub struct Isolate { + inner: CoreIsolate, + state: Arc, } -static DENO_INIT: Once = ONCE_INIT; - impl Isolate { - pub fn new( - init: IsolateInit, - state: Arc, - dispatch: Dispatch, - permissions: DenoPermissions, - ) -> Self { - DENO_INIT.call_once(|| { - unsafe { libdeno::deno_init() }; - }); - let config = libdeno::deno_config { - will_snapshot: 0, - load_snapshot: match init.snapshot { - Some(s) => s, - None => libdeno::deno_buf::empty(), - }, - shared: libdeno::deno_buf::empty(), // TODO Use for message passing. - recv_cb: pre_dispatch, - }; - let libdeno_isolate = unsafe { libdeno::deno_new(config) }; - // This channel handles sending async messages back to the runtime. - let (tx, rx) = mpsc::channel::<(usize, Buf)>(); - - let new_isolate = Self { - libdeno_isolate, - dispatch, - rx, - tx, - ntasks: Cell::new(0), - timeout_due: Cell::new(None), - modules: RefCell::new(Modules::new()), + pub fn new(cli: Cli) -> Isolate { + let state = cli.state.clone(); + Self { + inner: CoreIsolate::new(cli), state, - permissions: Arc::new(permissions), - }; - - // Run init script if present. - match init.init_script { - Some(init_script) => new_isolate - .execute2(init_script.filename.as_str(), init_script.source.as_str()) - .unwrap(), - None => {} - }; - - new_isolate - } - - #[inline] - pub fn as_raw_ptr(&self) -> *const c_void { - self as *const _ as *const c_void - } - - #[inline] - pub unsafe fn from_raw_ptr<'a>(ptr: *const c_void) -> &'a Self { - let ptr = ptr as *const _; - &*ptr - } - - #[inline] - pub fn get_timeout_due(&self) -> Option { - self.timeout_due.clone().into_inner() - } - - #[inline] - pub fn set_timeout_due(&self, inst: Option) { - self.timeout_due.set(inst); - } - - #[inline] - pub fn check_read(&self, filename: &str) -> DenoResult<()> { - self.permissions.check_read(filename) - } - - #[inline] - pub fn check_write(&self, filename: &str) -> DenoResult<()> { - self.permissions.check_write(filename) - } - - #[inline] - pub fn check_env(&self) -> DenoResult<()> { - self.permissions.check_env() - } - - #[inline] - pub fn check_net(&self, filename: &str) -> DenoResult<()> { - self.permissions.check_net(filename) - } - - #[inline] - pub fn check_run(&self) -> DenoResult<()> { - self.permissions.check_run() - } - - pub fn last_exception(&self) -> Option { - let ptr = unsafe { libdeno::deno_last_exception(self.libdeno_isolate) }; - if ptr.is_null() { - None - } else { - let cstr = unsafe { CStr::from_ptr(ptr) }; - let v8_exception = cstr.to_str().unwrap(); - debug!("v8_exception\n{}\n", v8_exception); - let js_error = JSError::from_v8_exception(v8_exception).unwrap(); - let js_error_mapped = apply_source_map(&js_error, &self.state.dir); - Some(js_error_mapped) } } @@ -283,82 +45,39 @@ impl Isolate { js_filename: &str, js_source: &str, ) -> Result<(), JSError> { - let filename = CString::new(js_filename).unwrap(); - let source = CString::new(js_source).unwrap(); - unsafe { - libdeno::deno_execute( - self.libdeno_isolate, - self.as_raw_ptr(), - filename.as_ptr(), - source.as_ptr(), - ) - }; - if let Some(err) = self.last_exception() { - return Err(err); - } - Ok(()) - } - - pub fn mod_new( - &mut self, - main: bool, - name: String, - source: String, - ) -> Result { - let name_ = CString::new(name.clone()).unwrap(); - let name_ptr = name_.as_ptr() as *const c_char; - - let source_ = CString::new(source.clone()).unwrap(); - let source_ptr = source_.as_ptr() as *const c_char; - - let id = unsafe { - libdeno::deno_mod_new(self.libdeno_isolate, main, name_ptr, source_ptr) - }; - if let Some(js_error) = self.last_exception() { - assert_eq!(id, 0); - return Err(js_error); - } - - self.modules.borrow_mut().register(id, &name); - - Ok(id) + self.inner.execute(js_filename, js_source) } // TODO(ry) make this return a future. - pub fn mod_load_deps( - &mut self, - id: libdeno::deno_mod, - ) -> Result<(), RustOrJsError> { + fn mod_load_deps(&self, id: deno_mod) -> Result<(), RustOrJsError> { // basically iterate over the imports, start loading them. - let referrer_name = - { self.modules.borrow_mut().get_name(id).unwrap().clone() }; - let len = - unsafe { libdeno::deno_mod_imports_len(self.libdeno_isolate, id) }; - - for i in 0..len { - let specifier_ptr = - unsafe { libdeno::deno_mod_imports_get(self.libdeno_isolate, id, i) }; - let specifier_c: &CStr = unsafe { CStr::from_ptr(specifier_ptr) }; - let specifier: &str = specifier_c.to_str().unwrap(); + let referrer_name = { + let g = self.state.modules.lock().unwrap(); + g.get_name(id).unwrap().clone() + }; + for specifier in self.inner.mod_get_imports(id) { let (name, _local_filename) = self .state .dir - .resolve_module(specifier, &referrer_name) + .resolve_module(&specifier, &referrer_name) .map_err(DenoError::from) .map_err(RustOrJsError::from)?; - debug!("mod_load_deps {} {}", i, name); + debug!("mod_load_deps {}", name); - if !self.modules.borrow_mut().is_registered(&name) { + if !self.state.modules.lock().unwrap().is_registered(&name) { let out = fetch_module_meta_data_and_maybe_compile( &self.state, - specifier, + &specifier, &referrer_name, )?; - let child_id = - self.mod_new(false, out.module_name.clone(), out.js_source())?; + let child_id = self.mod_new_and_regsiter( + false, + &out.module_name.clone(), + &out.js_source(), + )?; self.mod_load_deps(child_id)?; } @@ -367,141 +86,77 @@ impl Isolate { Ok(()) } - pub fn mod_instantiate(&self, id: libdeno::deno_mod) -> Result<(), JSError> { - unsafe { - libdeno::deno_mod_instantiate( - self.libdeno_isolate, - self.as_raw_ptr(), - id, - resolve_cb, - ) - }; - if let Some(js_error) = self.last_exception() { - return Err(js_error); - } - - Ok(()) - } - - pub fn mod_evaluate(&self, id: libdeno::deno_mod) -> Result<(), JSError> { - unsafe { - libdeno::deno_mod_evaluate(self.libdeno_isolate, self.as_raw_ptr(), id) - }; - if let Some(js_error) = self.last_exception() { - return Err(js_error); - } - Ok(()) - } - /// Executes the provided JavaScript module. pub fn execute_mod( &mut self, js_filename: &str, is_prefetch: bool, ) -> Result<(), RustOrJsError> { - let out = - fetch_module_meta_data_and_maybe_compile(&self.state, js_filename, ".") - .map_err(RustOrJsError::from)?; + // TODO move isolate_state::execute_mod impl here. + self + .execute_mod_inner(js_filename, is_prefetch) + .map_err(|err| match err { + RustOrJsError::Js(err) => RustOrJsError::Js(self.apply_source_map(err)), + x => x, + }) + } + + /// High-level way to execute modules. + /// This will issue HTTP requests and file system calls. + /// Blocks. TODO(ry) Don't block. + fn execute_mod_inner( + &self, + url: &str, + is_prefetch: bool, + ) -> Result<(), RustOrJsError> { + let out = fetch_module_meta_data_and_maybe_compile(&self.state, url, ".") + .map_err(RustOrJsError::from)?; let id = self - .mod_new(true, out.module_name.clone(), out.js_source()) + .mod_new_and_regsiter(true, &out.module_name.clone(), &out.js_source()) .map_err(RustOrJsError::from)?; self.mod_load_deps(id)?; - self.mod_instantiate(id).map_err(RustOrJsError::from)?; + self + .inner + .mod_instantiate(id) + .map_err(RustOrJsError::from)?; if !is_prefetch { - self.mod_evaluate(id).map_err(RustOrJsError::from)?; - } - Ok(()) - } - - pub fn respond(&self, zero_copy_id: usize, buf: Buf) { - self.state.metrics_op_completed(buf.len()); - - // This will be cleaned up in the future. - if zero_copy_id > 0 { - unsafe { - libdeno::deno_zero_copy_release(self.libdeno_isolate, zero_copy_id) - } - } - - // deno_respond will memcpy the buf into V8's heap, - // so borrowing a reference here is sufficient. - unsafe { - libdeno::deno_respond( - self.libdeno_isolate, - self.as_raw_ptr(), - buf.as_ref().into(), - ) - } - } - - fn complete_op(&self, zero_copy_id: usize, buf: Buf) { - // Receiving a message on rx exactly corresponds to an async task - // completing. - self.ntasks_decrement(); - // Call into JS with the buf. - self.respond(zero_copy_id, buf); - } - - fn timeout(&self) { - let dummy_buf = libdeno::deno_buf::empty(); - unsafe { - libdeno::deno_respond(self.libdeno_isolate, self.as_raw_ptr(), dummy_buf) - } - } - - fn check_promise_errors(&self) { - unsafe { - libdeno::deno_check_promise_errors(self.libdeno_isolate); - } - } - - // TODO Use Park abstraction? Note at time of writing Tokio default runtime - // does not have new_with_park(). - pub fn event_loop(&self) -> Result<(), JSError> { - // Main thread event loop. - while !self.is_idle() { - match recv_deadline(&self.rx, self.get_timeout_due()) { - Ok((zero_copy_id, buf)) => self.complete_op(zero_copy_id, buf), - Err(mpsc::RecvTimeoutError::Timeout) => self.timeout(), - Err(e) => panic!("recv_deadline() failed: {:?}", e), - } - self.check_promise_errors(); - if let Some(err) = self.last_exception() { - return Err(err); - } - } - // Check on done - self.check_promise_errors(); - if let Some(err) = self.last_exception() { - return Err(err); + self.inner.mod_evaluate(id).map_err(RustOrJsError::from)?; } Ok(()) } - #[inline] - fn ntasks_increment(&self) { - assert!(self.ntasks.get() >= 0); - self.ntasks.set(self.ntasks.get() + 1); + /// Wraps Isolate::mod_new but registers with modules. + fn mod_new_and_regsiter( + &self, + main: bool, + name: &str, + source: &str, + ) -> Result { + let id = self.inner.mod_new(main, name, source)?; + self.state.modules.lock().unwrap().register(id, &name); + Ok(id) } - #[inline] - fn ntasks_decrement(&self) { - self.ntasks.set(self.ntasks.get() - 1); - assert!(self.ntasks.get() >= 0); + pub fn print_file_info(&self, module: &str) { + let m = self.state.modules.lock().unwrap(); + m.print_file_info(&self.state.dir, module.to_string()); } - #[inline] - fn is_idle(&self) -> bool { - self.ntasks.get() == 0 && self.get_timeout_due().is_none() + /// Applies source map to the error. + fn apply_source_map(&self, err: JSError) -> JSError { + js_errors::apply_source_map(&err, &self.state.dir) } } -impl Drop for Isolate { - fn drop(&mut self) { - unsafe { libdeno::deno_delete(self.libdeno_isolate) } +impl Future for Isolate { + type Item = (); + type Error = JSError; + + fn poll(&mut self) -> Result, Self::Error> { + self.inner.poll().map_err(|err| self.apply_source_map(err)) } } @@ -523,330 +178,69 @@ fn fetch_module_meta_data_and_maybe_compile( Ok(out) } -extern "C" fn resolve_cb( - user_data: *mut c_void, - specifier_ptr: *const c_char, - referrer: libdeno::deno_mod, -) -> libdeno::deno_mod { - let isolate = unsafe { Isolate::from_raw_ptr(user_data) }; - let specifier_c: &CStr = unsafe { CStr::from_ptr(specifier_ptr) }; - let specifier: &str = specifier_c.to_str().unwrap(); - isolate - .state - .metrics - .resolve_count - .fetch_add(1, Ordering::Relaxed); - isolate.modules.borrow_mut().resolve_cb( - &isolate.state.dir, - specifier, - referrer, - ) -} - -// Dereferences the C pointer into the Rust Isolate object. -extern "C" fn pre_dispatch( - user_data: *mut c_void, - control_buf: libdeno::deno_buf, - zero_copy_buf: libdeno::deno_buf, -) { - // for metrics - let bytes_sent_control = control_buf.len(); - let bytes_sent_zero_copy = zero_copy_buf.len(); - - let zero_copy_id = zero_copy_buf.zero_copy_id; - - // We should ensure that there is no other `&mut Isolate` exists. - // And also, it should be in the same thread with other `&Isolate`s. - let isolate = unsafe { Isolate::from_raw_ptr(user_data) }; - let dispatch = isolate.dispatch; - let (is_sync, op) = dispatch(isolate, control_buf, zero_copy_buf); - - isolate - .state - .metrics_op_dispatched(bytes_sent_control, bytes_sent_zero_copy); - - if is_sync { - // Execute op synchronously. - let buf = tokio_util::block_on(op).unwrap(); - let buf_size = buf.len(); - - if buf_size == 0 { - // FIXME - isolate.state.metrics_op_completed(buf.len()); - } else { - // Set the synchronous response, the value returned from isolate.send(). - isolate.respond(zero_copy_id, buf); - } - } else { - // Execute op asynchronously. - let tx = isolate.tx.clone(); - - // TODO Ideally Tokio would could tell us how many tasks are executing, but - // it cannot currently. Therefore we track top-level promises/tasks - // manually. - isolate.ntasks_increment(); - - let task = op - .and_then(move |buf| { - let sender = tx; // tx is moved to new thread - sender.send((zero_copy_id, buf)).expect("tx.send error"); - Ok(()) - }).map_err(|_| ()); - tokio::spawn(task); - } -} - -fn recv_deadline( - rx: &mpsc::Receiver, - maybe_due: Option, -) -> Result { - match maybe_due { - None => rx.recv().map_err(|e| e.into()), - Some(due) => { - // Subtracting two Instants causes a panic if the resulting duration - // would become negative. Avoid this. - let now = Instant::now(); - let timeout = if due > now { - due - now - } else { - Duration::new(0, 0) - }; - // TODO: use recv_deadline() instead of recv_timeout() when this - // feature becomes stable/available. - rx.recv_timeout(timeout) - } - } -} - #[cfg(test)] mod tests { use super::*; - use futures; - - #[test] - fn test_dispatch_sync() { - let state = IsolateState::mock(); - let init = IsolateInit { - snapshot: None, - init_script: None, - }; - let isolate = - Isolate::new(init, state, dispatch_sync, DenoPermissions::default()); - tokio_util::init(|| { - isolate - .execute( - r#" - const m = new Uint8Array([4, 5, 6]); - let n = libdeno.send(m); - if (!(n.byteLength === 3 && - n[0] === 1 && - n[1] === 2 && - n[2] === 3)) { - throw Error("assert error"); - } - "#, - ).expect("execute error"); - isolate.event_loop().ok(); - }); - } - - fn dispatch_sync( - _isolate: &Isolate, - control: libdeno::deno_buf, - data: libdeno::deno_buf, - ) -> (bool, Box) { - assert_eq!(control[0], 4); - assert_eq!(control[1], 5); - assert_eq!(control[2], 6); - assert_eq!(data.len(), 0); - // Send back some sync response. - let vec: Vec = vec![1, 2, 3]; - let control = vec.into_boxed_slice(); - let op = Box::new(futures::future::ok(control)); - (true, op) - } - - #[test] - fn test_metrics_sync() { - let state = IsolateState::mock(); - let init = IsolateInit { - snapshot: None, - init_script: None, - }; - let isolate = Isolate::new( - init, - state, - metrics_dispatch_sync, - DenoPermissions::default(), - ); - tokio_util::init(|| { - // Verify that metrics have been properly initialized. - { - let metrics = &isolate.state.metrics; - assert_eq!(metrics.ops_dispatched.load(Ordering::SeqCst), 0); - assert_eq!(metrics.ops_completed.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_sent_control.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_sent_data.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_received.load(Ordering::SeqCst), 0); - } - - isolate - .execute( - r#" - const control = new Uint8Array([4, 5, 6]); - const data = new Uint8Array([42, 43, 44, 45, 46]); - libdeno.send(control, data); - "#, - ).expect("execute error");; - isolate.event_loop().unwrap(); - let metrics = &isolate.state.metrics; - assert_eq!(metrics.ops_dispatched.load(Ordering::SeqCst), 1); - assert_eq!(metrics.ops_completed.load(Ordering::SeqCst), 1); - assert_eq!(metrics.bytes_sent_control.load(Ordering::SeqCst), 3); - assert_eq!(metrics.bytes_sent_data.load(Ordering::SeqCst), 5); - assert_eq!(metrics.bytes_received.load(Ordering::SeqCst), 4); - }); - } - - #[test] - fn test_metrics_async() { - let state = IsolateState::mock(); - let init = IsolateInit { - snapshot: None, - init_script: None, - }; - let isolate = Isolate::new( - init, - state, - metrics_dispatch_async, - DenoPermissions::default(), - ); - tokio_util::init(|| { - // Verify that metrics have been properly initialized. - { - let metrics = &isolate.state.metrics; - assert_eq!(metrics.ops_dispatched.load(Ordering::SeqCst), 0); - assert_eq!(metrics.ops_completed.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_sent_control.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_sent_data.load(Ordering::SeqCst), 0); - assert_eq!(metrics.bytes_received.load(Ordering::SeqCst), 0); - } - - isolate - .execute( - r#" - const control = new Uint8Array([4, 5, 6]); - const data = new Uint8Array([42, 43, 44, 45, 46]); - let r = libdeno.send(control, data); - libdeno.recv(() => {}); - if (r != null) throw Error("expected null"); - "#, - ).expect("execute error"); - - // Make sure relevant metrics are updated before task is executed. - { - let metrics = &isolate.state.metrics; - assert_eq!(metrics.ops_dispatched.load(Ordering::SeqCst), 1); - assert_eq!(metrics.bytes_sent_control.load(Ordering::SeqCst), 3); - assert_eq!(metrics.bytes_sent_data.load(Ordering::SeqCst), 5); - // Note we cannot check ops_completed nor bytes_received because that - // would be a race condition. It might be nice to have use a oneshot - // with metrics_dispatch_async() to properly validate them. - } - - isolate.event_loop().unwrap(); - - // Make sure relevant metrics are updated after task is executed. - { - let metrics = &isolate.state.metrics; - assert_eq!(metrics.ops_dispatched.load(Ordering::SeqCst), 1); - assert_eq!(metrics.ops_completed.load(Ordering::SeqCst), 1); - assert_eq!(metrics.bytes_sent_control.load(Ordering::SeqCst), 3); - assert_eq!(metrics.bytes_sent_data.load(Ordering::SeqCst), 5); - assert_eq!(metrics.bytes_received.load(Ordering::SeqCst), 4); - } - }); - } - - fn metrics_dispatch_sync( - _isolate: &Isolate, - _control: libdeno::deno_buf, - _data: libdeno::deno_buf, - ) -> (bool, Box) { - // Send back some sync response - let vec: Box<[u8]> = vec![1, 2, 3, 4].into_boxed_slice(); - let op = Box::new(futures::future::ok(vec)); - (true, op) - } - - fn metrics_dispatch_async( - _isolate: &Isolate, - _control: libdeno::deno_buf, - _data: libdeno::deno_buf, - ) -> (bool, Box) { - // Send back some sync response - let vec: Box<[u8]> = vec![1, 2, 3, 4].into_boxed_slice(); - let op = Box::new(futures::future::ok(vec)); - (false, op) - } - - #[test] - fn thread_safety() { - fn is_thread_safe() {} - is_thread_safe::(); - } + use crate::flags; + use crate::isolate_init::IsolateInit; + use crate::permissions::DenoPermissions; + use crate::tokio_util; + use futures::future::lazy; + use std::sync::atomic::Ordering; #[test] fn execute_mod() { let filename = std::env::current_dir() .unwrap() .join("tests/esm_imports_a.js"); - let filename = filename.to_str().unwrap(); + let filename = filename.to_str().unwrap().to_string(); - let argv = vec![String::from("./deno"), String::from(filename)]; + let argv = vec![String::from("./deno"), filename.clone()]; let (flags, rest_argv, _) = flags::set_flags(argv).unwrap(); let state = Arc::new(IsolateState::new(flags, rest_argv, None)); + let state_ = state.clone(); let init = IsolateInit { snapshot: None, init_script: None, }; - let mut isolate = - Isolate::new(init, state, dispatch_sync, DenoPermissions::default()); - tokio_util::init(|| { - isolate - .execute_mod(filename, false) - .expect("execute_mod error"); - isolate.event_loop().ok(); - }); - - let metrics = &isolate.state.metrics; + tokio_util::run(lazy(move || { + let cli = Cli::new(init, state.clone(), DenoPermissions::default()); + let mut isolate = Isolate::new(cli); + if let Err(err) = isolate.execute_mod(&filename, false) { + eprintln!("execute_mod err {:?}", err); + } + tokio_util::panic_on_error(isolate) + })); + + let metrics = &state_.metrics; assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 1); } #[test] fn execute_mod_circular() { let filename = std::env::current_dir().unwrap().join("tests/circular1.js"); - let filename = filename.to_str().unwrap(); + let filename = filename.to_str().unwrap().to_string(); - let argv = vec![String::from("./deno"), String::from(filename)]; + let argv = vec![String::from("./deno"), filename.clone()]; let (flags, rest_argv, _) = flags::set_flags(argv).unwrap(); let state = Arc::new(IsolateState::new(flags, rest_argv, None)); + let state_ = state.clone(); let init = IsolateInit { snapshot: None, init_script: None, }; - let mut isolate = - Isolate::new(init, state, dispatch_sync, DenoPermissions::default()); - tokio_util::init(|| { - isolate - .execute_mod(filename, false) - .expect("execute_mod error"); - isolate.event_loop().ok(); - }); - - let metrics = &isolate.state.metrics; + tokio_util::run(lazy(move || { + let cli = Cli::new(init, state.clone(), DenoPermissions::default()); + let mut isolate = Isolate::new(cli); + if let Err(err) = isolate.execute_mod(&filename, false) { + eprintln!("execute_mod err {:?}", err); + } + tokio_util::panic_on_error(isolate) + })); + + let metrics = &state_.metrics; assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2); } } diff --git a/src/isolate_init.rs b/src/isolate_init.rs index 49fa0d96af24a8..fbdfdd4a5bafdf 100644 --- a/src/isolate_init.rs +++ b/src/isolate_init.rs @@ -1,5 +1,5 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -use crate::libdeno::deno_buf; +use deno_core::deno_buf; pub struct IsolateInitScript { pub source: String, diff --git a/src/isolate_state.rs b/src/isolate_state.rs new file mode 100644 index 00000000000000..b8706071e1dc1e --- /dev/null +++ b/src/isolate_state.rs @@ -0,0 +1,113 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. + +#![allow(dead_code)] + +use crate::cli::Buf; +use crate::deno_dir; +use crate::flags; +use crate::global_timer::GlobalTimer; +use crate::modules::Modules; +use futures::sync::mpsc as async_mpsc; +use std; +use std::env; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; + +pub type WorkerSender = async_mpsc::Sender; +pub type WorkerReceiver = async_mpsc::Receiver; +pub type WorkerChannels = (WorkerSender, WorkerReceiver); + +// AtomicU64 is currently unstable +#[derive(Default)] +pub struct Metrics { + pub ops_dispatched: AtomicUsize, + pub ops_completed: AtomicUsize, + pub bytes_sent_control: AtomicUsize, + pub bytes_sent_data: AtomicUsize, + pub bytes_received: AtomicUsize, + pub resolve_count: AtomicUsize, +} + +// Isolate cannot be passed between threads but IsolateState can. +// IsolateState satisfies Send and Sync. +// So any state that needs to be accessed outside the main V8 thread should be +// inside IsolateState. +#[cfg_attr(feature = "cargo-clippy", allow(stutter))] +pub struct IsolateState { + pub dir: deno_dir::DenoDir, + pub argv: Vec, + pub flags: flags::DenoFlags, + pub metrics: Metrics, + pub modules: Mutex, + pub worker_channels: Option>, + pub global_timer: Mutex, +} + +impl IsolateState { + pub fn new( + flags: flags::DenoFlags, + argv_rest: Vec, + worker_channels: Option, + ) -> Self { + let custom_root = env::var("DENO_DIR").map(|s| s.into()).ok(); + + Self { + dir: deno_dir::DenoDir::new(flags.reload, flags.recompile, custom_root) + .unwrap(), + argv: argv_rest, + flags, + metrics: Metrics::default(), + modules: Mutex::new(Modules::new()), + worker_channels: worker_channels.map(Mutex::new), + global_timer: Mutex::new(GlobalTimer::new()), + } + } + + pub fn main_module(&self) -> Option { + if self.argv.len() <= 1 { + None + } else { + let specifier = self.argv[1].clone(); + let referrer = "."; + match self.dir.resolve_module_url(&specifier, referrer) { + Ok(url) => Some(url.to_string()), + Err(e) => { + debug!("Potentially swallowed error {}", e); + None + } + } + } + } + + #[cfg(test)] + pub fn mock() -> IsolateState { + let argv = vec![String::from("./deno"), String::from("hello.js")]; + // For debugging: argv.push_back(String::from("-D")); + let (flags, rest_argv, _) = flags::set_flags(argv).unwrap(); + IsolateState::new(flags, rest_argv, None) + } + + pub fn metrics_op_dispatched( + &self, + bytes_sent_control: usize, + bytes_sent_data: usize, + ) { + self.metrics.ops_dispatched.fetch_add(1, Ordering::SeqCst); + self + .metrics + .bytes_sent_control + .fetch_add(bytes_sent_control, Ordering::SeqCst); + self + .metrics + .bytes_sent_data + .fetch_add(bytes_sent_data, Ordering::SeqCst); + } + + pub fn metrics_op_completed(&self, bytes_received: usize) { + self.metrics.ops_completed.fetch_add(1, Ordering::SeqCst); + self + .metrics + .bytes_received + .fetch_add(bytes_received, Ordering::SeqCst); + } +} diff --git a/src/js_errors.rs b/src/js_errors.rs index f42d9cb5129e08..90c9f200758d48 100644 --- a/src/js_errors.rs +++ b/src/js_errors.rs @@ -206,36 +206,41 @@ pub fn apply_source_map( } } -fn parse_map_string( - script_name: &str, - getter: &dyn SourceMapGetter, -) -> Option { +// The bundle does not get built for 'cargo check', so we don't embed the +// bundle source map. +#[cfg(feature = "check-only")] +fn builtin_source_map(script_name: &str) -> Option> { + None +} + +#[cfg(not(feature = "check-only"))] +fn builtin_source_map(script_name: &str) -> Option> { match script_name { - // The bundle does not get built for 'cargo check', so we don't embed the - // bundle source map. - #[cfg(not(feature = "check-only"))] - "gen/bundle/main.js" => { - let s = - include_str!(concat!(env!("GN_OUT_DIR"), "/gen/bundle/main.js.map")); - SourceMap::from_json(s) - } - #[cfg(not(feature = "check-only"))] - "gen/bundle/compiler.js" => { - let s = include_str!(concat!( + "gen/bundle/main.js" => Some( + include_bytes!(concat!(env!("GN_OUT_DIR"), "/gen/bundle/main.js.map")) + .to_vec(), + ), + "gen/bundle/compiler.js" => Some( + include_bytes!(concat!( env!("GN_OUT_DIR"), "/gen/bundle/compiler.js.map" - )); - SourceMap::from_json(s) - } - _ => match getter.get_source_map(script_name) { - None => None, - Some(raw_source_map) => { - SourceMap::from_json(str::from_utf8(&raw_source_map).unwrap()) - } - }, + )).to_vec(), + ), + _ => None, } } +fn parse_map_string( + script_name: &str, + getter: &dyn SourceMapGetter, +) -> Option { + builtin_source_map(script_name) + .or_else(|| getter.get_source_map(script_name)) + .and_then(|raw_source_map| { + SourceMap::from_json(str::from_utf8(&raw_source_map).unwrap()) + }) +} + fn get_mappings<'a>( script_name: &str, mappings_map: &'a mut CachedMaps, diff --git a/src/libdeno.rs b/src/libdeno.rs deleted file mode 100644 index 6696a382bbb6db..00000000000000 --- a/src/libdeno.rs +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. - -// TODO Remove. While core is being developed, it may not use the complete -// libdeno API. Thus we allow dead code until things settle. -#![allow(dead_code)] - -use libc::c_char; -use libc::c_int; -use libc::c_void; -use libc::size_t; -use std::ops::{Deref, DerefMut}; -use std::ptr::null; - -// TODO(F001): change this definition to `extern { pub type isolate; }` -// After RFC 1861 is stablized. See https://github.com/rust-lang/rust/issues/43467. -#[repr(C)] -pub struct isolate { - _unused: [u8; 0], -} - -/// If "alloc_ptr" is not null, this type represents a buffer which is created -/// in C side, and then passed to Rust side by `deno_recv_cb`. Finally it should -/// be moved back to C side by `deno_respond`. If it is not passed to -/// `deno_respond` in the end, it will be leaked. -/// -/// If "alloc_ptr" is null, this type represents a borrowed slice. -#[repr(C)] -pub struct deno_buf { - alloc_ptr: *const u8, - alloc_len: usize, - data_ptr: *const u8, - data_len: usize, - pub zero_copy_id: usize, -} - -/// `deno_buf` can not clone, and there is no interior mutability. -/// This type satisfies Send bound. -unsafe impl Send for deno_buf {} - -impl deno_buf { - #[inline] - pub fn empty() -> Self { - Self { - alloc_ptr: null(), - alloc_len: 0, - data_ptr: null(), - data_len: 0, - zero_copy_id: 0, - } - } - - #[inline] - pub unsafe fn from_raw_parts(ptr: *const u8, len: usize) -> Self { - Self { - alloc_ptr: null(), - alloc_len: 0, - data_ptr: ptr, - data_len: len, - zero_copy_id: 0, - } - } -} - -/// Converts Rust &Buf to libdeno `deno_buf`. -impl<'a> From<&'a [u8]> for deno_buf { - #[inline] - fn from(x: &'a [u8]) -> Self { - Self { - alloc_ptr: null(), - alloc_len: 0, - data_ptr: x.as_ref().as_ptr(), - data_len: x.len(), - zero_copy_id: 0, - } - } -} - -impl Deref for deno_buf { - type Target = [u8]; - #[inline] - fn deref(&self) -> &[u8] { - unsafe { std::slice::from_raw_parts(self.data_ptr, self.data_len) } - } -} - -impl DerefMut for deno_buf { - #[inline] - fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - if self.alloc_ptr.is_null() { - panic!("Can't modify the buf"); - } - std::slice::from_raw_parts_mut(self.data_ptr as *mut u8, self.data_len) - } - } -} - -impl AsRef<[u8]> for deno_buf { - #[inline] - fn as_ref(&self) -> &[u8] { - &*self - } -} - -impl AsMut<[u8]> for deno_buf { - #[inline] - fn as_mut(&mut self) -> &mut [u8] { - if self.alloc_ptr.is_null() { - panic!("Can't modify the buf"); - } - &mut *self - } -} - -#[allow(non_camel_case_types)] -type deno_recv_cb = unsafe extern "C" fn( - user_data: *mut c_void, - control_buf: deno_buf, // deprecated - zero_copy_buf: deno_buf, -); - -#[allow(non_camel_case_types)] -pub type deno_mod = i32; - -#[allow(non_camel_case_types)] -type deno_resolve_cb = unsafe extern "C" fn( - user_data: *mut c_void, - specifier: *const c_char, - referrer: deno_mod, -) -> deno_mod; - -#[repr(C)] -pub struct deno_config { - pub will_snapshot: c_int, - pub load_snapshot: deno_buf, - pub shared: deno_buf, - pub recv_cb: deno_recv_cb, -} - -extern "C" { - pub fn deno_init(); - pub fn deno_v8_version() -> *const c_char; - pub fn deno_set_v8_flags(argc: *mut c_int, argv: *mut *mut c_char); - pub fn deno_new(config: deno_config) -> *const isolate; - pub fn deno_delete(i: *const isolate); - pub fn deno_last_exception(i: *const isolate) -> *const c_char; - pub fn deno_check_promise_errors(i: *const isolate); - pub fn deno_lock(i: *const isolate); - pub fn deno_unlock(i: *const isolate); - pub fn deno_respond( - i: *const isolate, - user_data: *const c_void, - buf: deno_buf, - ); - pub fn deno_zero_copy_release(i: *const isolate, zero_copy_id: usize); - pub fn deno_execute( - i: *const isolate, - user_data: *const c_void, - js_filename: *const c_char, - js_source: *const c_char, - ); - - // Modules - - pub fn deno_mod_new( - i: *const isolate, - main: bool, - name: *const c_char, - source: *const c_char, - ) -> deno_mod; - - pub fn deno_mod_imports_len(i: *const isolate, id: deno_mod) -> size_t; - - pub fn deno_mod_imports_get( - i: *const isolate, - id: deno_mod, - index: size_t, - ) -> *const c_char; - - pub fn deno_mod_instantiate( - i: *const isolate, - user_data: *const c_void, - id: deno_mod, - resolve_cb: deno_resolve_cb, - ); - - pub fn deno_mod_evaluate( - i: *const isolate, - user_data: *const c_void, - id: deno_mod, - ); -} diff --git a/src/main.rs b/src/main.rs index 48d96b04ac3f9b..955c3850feb990 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,6 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +#![allow(unused_variables)] + #[macro_use] extern crate lazy_static; #[macro_use] @@ -9,19 +11,22 @@ extern crate futures; extern crate serde_json; mod ansi; +pub mod cli; pub mod compiler; pub mod deno_dir; pub mod errors; pub mod flags; mod fs; +mod global_timer; mod http_body; mod http_util; pub mod isolate; pub mod isolate_init; +pub mod isolate_state; pub mod js_errors; -pub mod libdeno; pub mod modules; pub mod msg; +mod msg_ring; pub mod msg_util; pub mod ops; pub mod permissions; @@ -36,6 +41,12 @@ pub mod workers; #[cfg(unix)] mod eager_unix; +use crate::cli::Cli; +use crate::errors::RustOrJsError; +use crate::isolate::Isolate; +use crate::isolate_state::IsolateState; +use futures::lazy; +use futures::Future; use log::{LevelFilter, Metadata, Record}; use std::env; use std::sync::Arc; @@ -57,11 +68,20 @@ impl log::Log for Logger { fn flush(&self) {} } -fn print_err_and_exit(err: errors::RustOrJsError) { +fn print_err_and_exit(err: RustOrJsError) { eprintln!("{}", err.to_string()); std::process::exit(1); } +fn js_check(r: Result<(), E>) +where + E: Into, +{ + if let Err(err) = r { + print_err_and_exit(err.into()); + } +} + fn main() { #[cfg(windows)] ansi_term::enable_ansi_support().ok(); // For Windows 10 @@ -94,39 +114,33 @@ fn main() { let should_prefetch = flags.prefetch || flags.info; let should_display_info = flags.info; - let state = Arc::new(isolate::IsolateState::new(flags, rest_argv, None)); + let state = Arc::new(IsolateState::new(flags, rest_argv, None)); + let state_ = state.clone(); let isolate_init = isolate_init::deno_isolate_init(); let permissions = permissions::DenoPermissions::from_flags(&state.flags); - let mut isolate = - isolate::Isolate::new(isolate_init, state, ops::dispatch, permissions); + let cli = Cli::new(isolate_init, state_, permissions); + let mut isolate = Isolate::new(cli); - tokio_util::init(|| { + let main_future = lazy(move || { // Setup runtime. - isolate - .execute("denoMain();") - .map_err(errors::RustOrJsError::from) - .unwrap_or_else(print_err_and_exit); + js_check(isolate.execute("denoMain()")); // Execute main module. - if let Some(main_module) = isolate.state.main_module() { + if let Some(main_module) = state.main_module() { debug!("main_module {}", main_module); - isolate - .execute_mod(&main_module, should_prefetch) - .unwrap_or_else(print_err_and_exit); + js_check(isolate.execute_mod(&main_module, should_prefetch)); if should_display_info { // Display file info and exit. Do not run file - modules::print_file_info( - &isolate.modules.borrow(), - &isolate.state.dir, - main_module, - ); + isolate.print_file_info(&main_module); std::process::exit(0); } } - isolate - .event_loop() - .map_err(errors::RustOrJsError::from) - .unwrap_or_else(print_err_and_exit); + isolate.then(|result| { + js_check(result); + Ok(()) + }) }); + + tokio_util::run(main_future); } diff --git a/src/modules.rs b/src/modules.rs index 67be47dd4d2bf1..908c31b6df542b 100644 --- a/src/modules.rs +++ b/src/modules.rs @@ -1,8 +1,8 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. use crate::ansi; use crate::deno_dir::DenoDir; -use crate::libdeno::deno_mod; use crate::msg; +use deno_core::deno_mod; use std::collections::HashMap; use std::collections::HashSet; use std::fmt; @@ -86,6 +86,44 @@ impl Modules { return 0; } } + + pub fn print_file_info(&self, deno_dir: &DenoDir, filename: String) { + let maybe_out = deno_dir.fetch_module_meta_data(&filename, "."); + if maybe_out.is_err() { + println!("{}", maybe_out.unwrap_err()); + return; + } + let out = maybe_out.unwrap(); + + println!("{} {}", ansi::bold("local:".to_string()), &(out.filename)); + println!( + "{} {}", + ansi::bold("type:".to_string()), + msg::enum_name_media_type(out.media_type) + ); + if out.maybe_output_code_filename.is_some() { + println!( + "{} {}", + ansi::bold("compiled:".to_string()), + out.maybe_output_code_filename.as_ref().unwrap(), + ); + } + if out.maybe_source_map_filename.is_some() { + println!( + "{} {}", + ansi::bold("map:".to_string()), + out.maybe_source_map_filename.as_ref().unwrap() + ); + } + + let deps = Deps::new(self, &out.module_name); + println!("{}{}", ansi::bold("deps:\n".to_string()), deps.name); + if let Some(ref depsdeps) = deps.deps { + for d in depsdeps { + println!("{}", d); + } + } + } } pub struct Deps { @@ -164,45 +202,3 @@ impl fmt::Display for Deps { Ok(()) } } - -pub fn print_file_info( - modules: &Modules, - deno_dir: &DenoDir, - filename: String, -) { - let maybe_out = deno_dir.fetch_module_meta_data(&filename, "."); - if maybe_out.is_err() { - println!("{}", maybe_out.unwrap_err()); - return; - } - let out = maybe_out.unwrap(); - - println!("{} {}", ansi::bold("local:".to_string()), &(out.filename)); - println!( - "{} {}", - ansi::bold("type:".to_string()), - msg::enum_name_media_type(out.media_type) - ); - if out.maybe_output_code_filename.is_some() { - println!( - "{} {}", - ansi::bold("compiled:".to_string()), - out.maybe_output_code_filename.as_ref().unwrap(), - ); - } - if out.maybe_source_map_filename.is_some() { - println!( - "{} {}", - ansi::bold("map:".to_string()), - out.maybe_source_map_filename.as_ref().unwrap() - ); - } - - let deps = Deps::new(modules, &out.module_name); - println!("{}{}", ansi::bold("deps:\n".to_string()), deps.name); - if let Some(ref depsdeps) = deps.deps { - for d in depsdeps { - println!("{}", d); - } - } -} diff --git a/src/msg.fbs b/src/msg.fbs index 1b57e72ef501b7..279264a4549ab9 100644 --- a/src/msg.fbs +++ b/src/msg.fbs @@ -1,74 +1,76 @@ union Any { - Start, - StartRes, - FormatError, - FormatErrorRes, - WorkerGetMessage, - WorkerGetMessageRes, - WorkerPostMessage, - FetchModuleMetaData, - FetchModuleMetaDataRes, - SetTimeout, - Exit, + Accept, + Chdir, + Chmod, + Close, + CopyFile, + Cwd, + CwdRes, + Dial, Environ, EnvironRes, - Permissions, - PermissionRevoke, - PermissionsRes, + Exit, Fetch, + FetchModuleMetaData, + FetchModuleMetaDataRes, FetchRes, + FormatError, + FormatErrorRes, + GlobalTimer, + GlobalTimerRes, + GlobalTimerStop, + IsTTY, + IsTTYRes, + Listen, + ListenRes, MakeTempDir, MakeTempDirRes, + Metrics, + MetricsRes, Mkdir, - Chmod, - Remove, - ReadFile, - ReadFileRes, + NewConn, + Now, + NowRes, + Open, + OpenRes, + PermissionRevoke, + Permissions, + PermissionsRes, + Read, ReadDir, ReadDirRes, - WriteFile, - CopyFile, - Rename, + ReadFile, + ReadFileRes, + ReadRes, Readlink, ReadlinkRes, - ReplStart, - ReplStartRes, + Remove, + Rename, ReplReadline, ReplReadlineRes, + ReplStart, + ReplStartRes, Resources, ResourcesRes, - Symlink, - Stat, - StatRes, - SetEnv, - Truncate, - Open, - OpenRes, - Read, - ReadRes, - Write, - WriteRes, - Close, - Shutdown, - Listen, - ListenRes, - Accept, - Dial, - NewConn, - Chdir, - Cwd, - CwdRes, - Metrics, - MetricsRes, Run, RunRes, RunStatus, RunStatusRes, - Now, - NowRes, - IsTTY, - IsTTYRes, Seek, + SetEnv, + Shutdown, + Start, + StartRes, + Stat, + StatRes, + Symlink, + Truncate, + WorkerGetMessage, + WorkerGetMessageRes, + WorkerPostMessage, + Write, + WriteFile, + WriteRes, } enum ErrorKind: byte { @@ -210,10 +212,14 @@ table Chdir { directory: string; } -table SetTimeout { +table GlobalTimer { timeout: int; } +table GlobalTimerRes { } + +table GlobalTimerStop { } + table Exit { code: int; } diff --git a/src/msg.rs b/src/msg.rs index 74027d3d655fec..080f39de81e69e 100644 --- a/src/msg.rs +++ b/src/msg.rs @@ -5,6 +5,7 @@ feature = "cargo-clippy", allow(clippy::all, clippy::pedantic) )] +use crate::isolate_state; use flatbuffers; use std::sync::atomic::Ordering; @@ -12,8 +13,8 @@ use std::sync::atomic::Ordering; // build_extra/rust/run.py (for the GN+Ninja build). include!(concat!(env!("GN_OUT_DIR"), "/gen/msg_generated.rs")); -impl<'a> From<&'a super::isolate::Metrics> for MetricsResArgs { - fn from(m: &'a super::isolate::Metrics) -> Self { +impl<'a> From<&'a isolate_state::Metrics> for MetricsResArgs { + fn from(m: &'a isolate_state::Metrics) -> Self { MetricsResArgs { ops_dispatched: m.ops_dispatched.load(Ordering::SeqCst) as u64, ops_completed: m.ops_completed.load(Ordering::SeqCst) as u64, diff --git a/src/msg_ring.rs b/src/msg_ring.rs new file mode 100644 index 00000000000000..874bc38c3906b7 --- /dev/null +++ b/src/msg_ring.rs @@ -0,0 +1,655 @@ +#![allow(dead_code)] + +use deno_core::deno_buf; +use std::marker; +use std::mem::{forget, size_of}; +use std::ops::{Add, BitAnd, Deref, DerefMut, Not, Sub}; +use std::slice; +use std::sync::Arc; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Counters { + pub message: usize, + pub acquire: usize, + pub release: usize, + pub spin: usize, + pub wait: usize, + pub notify: usize, + pub wrap: usize, +} + +#[derive(Clone, Copy)] +pub enum FillDirection { + TopDown, + BottomUp, +} + +trait MapOffset { + fn map_offset(&self, offset: usize, length: usize, end: usize) -> usize; +} + +impl MapOffset for FillDirection { + fn map_offset(&self, offset: usize, length: usize, end: usize) -> usize { + match self { + FillDirection::TopDown => offset, + FillDirection::BottomUp => end - length - offset, + } + } +} + +struct FrameAllocation; +#[allow(non_upper_case_globals)] +impl FrameAllocation { + pub const Alignment: usize = 4; + pub const HeaderByteLength: usize = 4; +} + +struct FrameHeader; +#[allow(non_upper_case_globals)] +impl FrameHeader { + // Using i32 since that's what's used on the JS side. + pub const None: i32 = 0x00_00_00_00; + pub const ByteLengthMask: i32 = 0x00_ff_ff_ff; + pub const EpochMask: i32 = 0x03_00_00_00; + pub const EpochInitSender: i32 = 0x00_00_00_00; + pub const EpochInitReceiver: i32 = 0x01_00_00_00; + pub const EpochIncrementPass: i32 = 0x01_00_00_00; + pub const EpochIncrementWrap: i32 = 0x02_00_00_00; + pub const HasMessageFlag: i32 = 0x04_00_00_00; + pub const HasWaitersFlag: i32 = 0x08_00_00_00; +} + +trait Align { + fn align(self, to: T) -> Self; + fn is_aligned(self, to: T) -> bool; +} + +impl Align for T +where + T: Copy + + From + + PartialEq + + Add + + Sub + + BitAnd + + Not, +{ + fn align(self, to: T) -> Self { + let mask = to - 1.into(); + (self + mask) & !mask + } + + fn is_aligned(self, to: T) -> bool { + self & (to - 1.into()) == 0.into() + } +} + +enum Dealloc { + Void, + Vec { + ptr: *mut u8, + len: usize, + cap: usize, + }, +} + +unsafe impl marker::Send for Dealloc {} + +impl Drop for Dealloc { + #[allow(clippy::match_ref_pats)] // Clippy is wrong, `&mut` is necessary. + fn drop(&mut self) { + if let &mut Dealloc::Vec { ptr, len, cap } = self { + unsafe { Vec::::from_raw_parts(ptr, len, cap) }; + } + } +} + +pub struct Buffer { + ptr: *mut u8, + byte_length: usize, + dealloc: Arc, +} + +unsafe impl marker::Send for Buffer {} +unsafe impl marker::Sync for Buffer {} + +impl Buffer { + pub fn new(byte_length: usize) -> Self { + assert!(byte_length > 0); + assert!(byte_length.is_aligned(FrameAllocation::Alignment)); + + let mut vec: Vec = Vec::new(); + vec.resize(byte_length, 0); + let ptr = vec.as_mut_ptr(); + let dealloc = Dealloc::Vec { + ptr, + len: vec.len(), + cap: vec.capacity(), + }; + forget(vec); + + Self { + ptr, + byte_length, + dealloc: Arc::new(dealloc), + } + } + + pub fn into_deno_buf(&self) -> deno_buf { + unsafe { deno_buf::from_raw_parts(self.ptr, self.byte_length) } + } + + pub unsafe fn from_raw_parts(ptr: *mut u8, byte_length: usize) -> Self { + assert!(byte_length.is_aligned(FrameAllocation::Alignment as usize)); + Self { + ptr, + byte_length, + dealloc: Arc::new(Dealloc::Void), + } + } + + pub fn split(self) -> (Self, Self) { + let half = self.byte_length() / 2; + assert!(half.is_aligned(FrameAllocation::Alignment)); + ( + Self { + ptr: self.ptr, + byte_length: half, + dealloc: self.dealloc.clone(), + }, + Self { + ptr: unsafe { self.ptr.add(half) }, + byte_length: half, + dealloc: self.dealloc.clone(), + }, + ) + } + + unsafe fn dup(&self) -> Self { + Self { + ptr: self.ptr, + byte_length: self.byte_length, + dealloc: self.dealloc.clone(), + } + } + + pub fn byte_length(&self) -> usize { + self.byte_length + } + + #[allow(dead_code)] + unsafe fn get(&self, byte_offset: usize) -> &T { + self.slice(byte_offset, size_of::()).get_unchecked(0) + } + + unsafe fn get_mut(&mut self, byte_offset: usize) -> &mut T { + self + .slice_mut(byte_offset, size_of::()) + .get_unchecked_mut(0) + } + + unsafe fn slice(&self, byte_offset: usize, byte_length: usize) -> &[T] { + let (offset, count) = self.map_bytes_to::(byte_offset, byte_length); + slice::from_raw_parts((self.ptr as *mut T).add(offset), count) + } + + unsafe fn slice_mut( + &mut self, + byte_offset: usize, + byte_length: usize, + ) -> &mut [T] { + let (offset, count) = self.map_bytes_to::(byte_offset, byte_length); + slice::from_raw_parts_mut((self.ptr as *mut T).add(offset), count) + } + + fn map_bytes_to( + &self, + byte_offset: usize, + byte_length: usize, + ) -> (usize, usize) { + let bytes_per_item = size_of::(); + assert!(byte_offset + byte_length <= self.byte_length); + debug_assert!(byte_offset.is_aligned(bytes_per_item)); + debug_assert!(byte_length.is_aligned(bytes_per_item)); + (byte_offset / bytes_per_item, byte_length / bytes_per_item) + } +} + +#[derive(Clone, Copy)] +pub struct Config { + pub fill_direction: FillDirection, + pub spin_count: u32, + // TODO maybe: + //pub spin_yield_cpu_time: u32, +} + +impl Default for Config { + fn default() -> Self { + Self { + fill_direction: FillDirection::TopDown, + spin_count: 10000, + // spin_yield_cpu_time: 0, + } + } +} + +// TODO: probably should use builder pattern. +pub struct MsgRing { + buffer: Buffer, + config: Config, +} + +impl MsgRing { + pub fn new(buffer: Buffer) -> Self { + Self { + buffer, + config: Default::default(), + } + } + + pub fn new_with(buffer: Buffer, config: Config) -> Self { + Self { buffer, config } + } + + pub fn split(self) -> (Sender, Receiver) { + let sender = Sender::new(unsafe { self.buffer.dup() }, self.config); + let receiver = Receiver::new(self.buffer, self.config); + (sender, receiver) + } +} + +struct Window { + pub buffer: Buffer, + pub config: Config, + pub counters: Counters, + + // Head and tail position are in bytes, but always starting at zero and + // not adjusted for buffer fill direction (see TypeScript implementation). + pub epoch: i32, + pub tail_position: usize, + pub head_position: usize, +} + +impl Window { + pub fn new(buffer: Buffer, config: Config, epoch: i32) -> Self { + let mut this = Self { + buffer, + config, + counters: Counters { + ..Default::default() + }, + epoch, + head_position: 0, + tail_position: 0, + }; + this.init(); + this + } + + #[inline] + pub fn byte_length(&self) -> usize { + self.head_position - self.tail_position + } + + #[inline] + fn is_at_end_of_buffer(&self) -> bool { + self.head_position == self.buffer.byte_length() + } + + #[inline] + fn header_byte_offset(&self, position: usize) -> usize { + self.config.fill_direction.map_offset( + position, + FrameAllocation::HeaderByteLength, + self.buffer.byte_length(), + ) + } + + #[inline] + fn get_message_byte_range(&self, frame_byte_length: usize) -> (usize, usize) { + let message_byte_length = + frame_byte_length - FrameAllocation::HeaderByteLength; + let message_byte_offset = self.config.fill_direction.map_offset( + self.tail_position + FrameAllocation::HeaderByteLength, + message_byte_length, + self.buffer.byte_length(), + ); + (message_byte_offset, message_byte_length) + } + + pub fn get_message_slice(&self, frame_byte_length: usize) -> &[u8] { + let (byte_offset, byte_length) = + self.get_message_byte_range(frame_byte_length); + unsafe { self.buffer.slice(byte_offset, byte_length) } + } + + pub fn get_message_slice_mut( + &mut self, + frame_byte_length: usize, + ) -> &mut [u8] { + let (byte_offset, byte_length) = + self.get_message_byte_range(frame_byte_length); + unsafe { self.buffer.slice_mut(byte_offset, byte_length) } + } + + fn init(&mut self) { + let target = self.buffer.byte_length() as i32; + let header_byte_offset = self.header_byte_offset(0); + let header_ref: &mut i32 = + unsafe { self.buffer.get_mut(header_byte_offset) }; + + if *header_ref == 0 { + *header_ref = target; + } + //let header_atomic: &mut Futex = + // unsafe { self.buffer.get_mut(header_byte_offset) }; + //header_atomic.compare_and_swap(0, target, Ordering::AcqRel); + } + + pub fn acquire_frame(&mut self, wait: bool) -> i32 { + if self.is_at_end_of_buffer() { + assert!(self.byte_length() == 0); + + self.epoch = + (self.epoch + FrameHeader::EpochIncrementWrap) & FrameHeader::EpochMask; + + self.head_position = 0; + self.tail_position = 0; + self.counters.wrap += 1; + } + + let header_byte_offset = self.header_byte_offset(self.head_position); + /* + let header_atomic: &mut Futex = + unsafe { self.buffer.get_mut(header_byte_offset) }; + */ + let header = *unsafe { self.buffer.get::(header_byte_offset) }; + + // let mut spin_count_remaining = self.config.spin_count; + // let mut sleep = false; + + if header & FrameHeader::EpochMask != self.epoch { + assert!(!wait); + return FrameHeader::None; + } + + // Note that operator precendece in Rust is different than in C and + // JavaScript (& has higher precedence than ==), so this is correct. + //while header & FrameHeader::EpochMask != self.epoch { + // if !wait { + // return FrameHeader::None; + // } + // + // if spin_count_remaining == 0 { + // let expect = header; + // let target = header | FrameHeader::HasWaitersFlag; + // header = + // header_atomic.compare_and_swap(expect, target, Ordering::AcqRel); + // if expect != header { + // continue; + // } + // header = target; + // sleep = true; + // self.counters.wait += 1; + // } else { + // spin_count_remaining -= 1; + // self.counters.spin += 1; + // } + // + // if sleep { + // header_atomic.wait(header, None); + // } else { + // std::thread::yield_now(); + // } + // header = header_atomic.load(Ordering::Acquire); + //} + + let byte_length = header & FrameHeader::ByteLengthMask; + let byte_length = byte_length as usize; + assert!(byte_length <= self.buffer.byte_length() - self.head_position); + + self.head_position += byte_length; + self.counters.acquire += 1; + + header + } + + pub fn release_frame(&mut self, byte_length: usize, flags: i32) { + assert!(byte_length >= FrameAllocation::HeaderByteLength); + assert!(byte_length <= self.byte_length()); + + if byte_length < self.byte_length() { + // Place a temporary header for which what will become the next message + // right after the message. + let next_header_byte_offset = + self.header_byte_offset(self.tail_position + byte_length); + let next_header_ref: &mut i32 = + unsafe { self.buffer.get_mut(next_header_byte_offset) }; + // TODO: Do away with epoch concept, just use zero always. + *next_header_ref = self.epoch; + } + + let tail_epoch = self.epoch + FrameHeader::EpochIncrementPass; + let new_header = byte_length as i32 | flags | tail_epoch; + + let header_byte_offset = self.header_byte_offset(self.tail_position); + let header_ref: &mut i32 = + unsafe { self.buffer.get_mut(header_byte_offset) }; + let old_header = *header_ref; + *header_ref = new_header; + assert!(old_header & FrameHeader::HasWaitersFlag == 0); + + // let header_atomic: &mut Futex = + // unsafe { self.buffer.get_mut(header_byte_offset) }; + // + // let old_header = header_atomic.swap(new_header, Ordering::AcqRel); + // + // if old_header & FrameHeader::HasWaitersFlag != 0 { + // header_atomic.notify_one(); + // self.counters.notify += 1; + // } + + self.tail_position += byte_length; + self.counters.release += 1; + } +} + +pub struct Sender { + window: Window, +} + +impl Sender { + pub fn new(buffer: Buffer, config: Config) -> Self { + Self { + window: Window::new(buffer, config, FrameHeader::EpochInitSender), + } + } + + pub fn compose(&mut self, byte_length: usize) -> Option { + Send::maybe_new(&mut self.window, byte_length) + } + + pub fn counters(&self) -> Counters { + self.window.counters + } +} + +pub struct Send<'msg> { + window: &'msg mut Window, + allocation_byte_length: usize, +} + +impl<'msg> Send<'msg> { + fn maybe_new( + window: &'msg mut Window, + message_byte_length: usize, + ) -> Option { + let mut r = Self { + window, + allocation_byte_length: 0, + }; + if r.allocate(message_byte_length) { + Some(r) + } else { + None + } + } + + // pub fn resize(&mut self, message_byte_length: usize) { + // self.allocate(message_byte_length) + // } + + pub fn send(self) { + self.window.counters.message += 1; + self + .window + .release_frame(self.allocation_byte_length, FrameHeader::HasMessageFlag); + } + + pub fn dispose(self) {} + + fn allocate(&mut self, byte_length: usize) -> bool { + self.allocation_byte_length = FrameAllocation::HeaderByteLength as usize + + byte_length.align(FrameAllocation::Alignment as usize); + assert!( + self.allocation_byte_length <= FrameHeader::ByteLengthMask as usize + ); + while self.window.byte_length() < self.allocation_byte_length { + if self.window.is_at_end_of_buffer() && self.window.byte_length() > 0 { + self + .window + .release_frame(self.window.byte_length(), FrameHeader::None); + } + let header = self.window.acquire_frame(false); + if header == FrameHeader::None { + return false; + } + } + true + } +} + +impl<'msg> Deref for Send<'msg> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + self.window.get_message_slice(self.allocation_byte_length) + } +} + +impl<'msg> DerefMut for Send<'msg> { + fn deref_mut(&mut self) -> &mut [u8] { + self + .window + .get_message_slice_mut(self.allocation_byte_length) + } +} + +pub struct Receiver { + window: Window, +} + +// TODO: validate this. +// I suspect Receiver is Send but not Sync. +unsafe impl marker::Send for Receiver {} + +impl Receiver { + pub fn new(buffer: Buffer, config: Config) -> Self { + Self { + window: Window::new(buffer, config, FrameHeader::EpochInitReceiver), + } + } + + pub fn receive(&mut self) -> Option { + Receive::maybe_new(&mut self.window) + } + + pub fn counters(&self) -> Counters { + self.window.counters + } +} + +pub struct Receive<'msg: 'msg> { + window: &'msg mut Window, +} + +impl<'msg> Receive<'msg> { + fn maybe_new(window: &'msg mut Window) -> Option { + let mut r = Self { window }; + if r.acquire() { + Some(r) + } else { + None + } + } + + fn acquire(&mut self) -> bool { + debug_assert_eq!(self.window.byte_length(), 0); + let mut skipped = false; + loop { + let header = self.window.acquire_frame(false); + if header == FrameHeader::None { + return false; + } + if header & FrameHeader::HasMessageFlag != 0 { + self.window.counters.message += 1; + return true; + } + self + .window + .release_frame(self.window.byte_length(), FrameHeader::None); + assert!(skipped == false); + skipped = true; + } + } + + fn release(&mut self) { + if self.window.byte_length() == 0 { + // No message was available and maybe_new() returned None. + return; + } + self + .window + .release_frame(self.window.byte_length(), FrameHeader::None); + } + + pub fn dispose(self) {} +} + +impl<'msg> Deref for Receive<'msg> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + self.window.get_message_slice(self.window.byte_length()) + } +} + +impl<'msg> Drop for Receive<'msg> { + fn drop(&mut self) { + self.release(); + } +} + +#[cfg(test)] +mod test { + use super::{Buffer, MsgRing}; + + #[test] + fn msg_ring_single_threaded() { + let buffer = Buffer::new(240); + let ring = MsgRing::new(buffer); + let (mut sender, mut receiver) = ring.split(); + + for i in 0..100_000usize { + let mut msg = sender.compose(40).unwrap(); + msg[0] = (i % 7) as u8; + msg[39] = (i % 13) as u8; + msg.send(); + + let msg = receiver.receive().unwrap(); + assert!(msg.len() == 40); + assert!(msg[0] == (i % 7) as u8); + assert!(msg[39] == (i % 13) as u8); + } + } +} diff --git a/src/ops.rs b/src/ops.rs index 5c5fc67fc40aa9..c96dcb76b8079a 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -1,18 +1,16 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. - use atty; use crate::ansi; +use crate::cli::Buf; +use crate::cli::Cli; +use crate::cli::CliOp; use crate::errors; use crate::errors::{permission_denied, DenoError, DenoResult, ErrorKind}; use crate::fs as deno_fs; use crate::http_util; -use crate::isolate::Buf; -use crate::isolate::Isolate; -use crate::isolate::IsolateState; -use crate::isolate::Op; +use crate::isolate_state::IsolateState; use crate::js_errors::apply_source_map; use crate::js_errors::JSErrorColor; -use crate::libdeno; use crate::msg; use crate::msg_util; use crate::repl; @@ -22,6 +20,7 @@ use crate::resources::table_entries; use crate::resources::Resource; use crate::tokio_util; use crate::version; +use deno_core::deno_buf; use deno_core::JSError; use flatbuffers::FlatBufferBuilder; use futures; @@ -55,11 +54,11 @@ use std::os::unix::process::ExitStatusExt; type OpResult = DenoResult; +pub type Op = dyn Future + Send; + // TODO Ideally we wouldn't have to box the Op being returned. // The box is just to make it easier to get a prototype refactor working. -type OpCreator = - fn(isolate: &Isolate, base: &msg::Base<'_>, data: libdeno::deno_buf) - -> Box; +type OpCreator = fn(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box; #[inline] fn empty_buf() -> Buf { @@ -71,45 +70,47 @@ fn empty_buf() -> Buf { /// control corresponds to the first argument of libdeno.send(). /// data corresponds to the second argument of libdeno.send(). pub fn dispatch( - isolate: &Isolate, - control: libdeno::deno_buf, - data: libdeno::deno_buf, -) -> (bool, Box) { + cli: &Cli, + control: Buf, + zero_copy: deno_buf, +) -> (bool, Box) { + let bytes_sent_control = control.len(); + let bytes_sent_zero_copy = zero_copy.len(); let base = msg::get_root_as_base(&control); let is_sync = base.sync(); let inner_type = base.inner_type(); let cmd_id = base.cmd_id(); - let op: Box = if inner_type == msg::Any::SetTimeout { - // SetTimeout is an exceptional op: the global timeout field is part of the - // Isolate state (not the IsolateState state) and it must be updated on the - // main thread. - assert_eq!(is_sync, true); - op_set_timeout(isolate, &base, data) - } else { + let op: Box = { // Handle regular ops. let op_creator: OpCreator = match inner_type { msg::Any::Accept => op_accept, msg::Any::Chdir => op_chdir, msg::Any::Chmod => op_chmod, msg::Any::Close => op_close, - msg::Any::FetchModuleMetaData => op_fetch_module_meta_data, msg::Any::CopyFile => op_copy_file, msg::Any::Cwd => op_cwd, msg::Any::Dial => op_dial, msg::Any::Environ => op_env, msg::Any::Exit => op_exit, msg::Any::Fetch => op_fetch, + msg::Any::FetchModuleMetaData => op_fetch_module_meta_data, msg::Any::FormatError => op_format_error, + msg::Any::GlobalTimer => op_global_timer, + msg::Any::GlobalTimerStop => op_global_timer_stop, + msg::Any::IsTTY => op_is_tty, msg::Any::Listen => op_listen, msg::Any::MakeTempDir => op_make_temp_dir, msg::Any::Metrics => op_metrics, msg::Any::Mkdir => op_mkdir, + msg::Any::Now => op_now, msg::Any::Open => op_open, + msg::Any::PermissionRevoke => op_revoke_permission, + msg::Any::Permissions => op_permissions, + msg::Any::Read => op_read, msg::Any::ReadDir => op_read_dir, msg::Any::ReadFile => op_read_file, msg::Any::Readlink => op_read_link, - msg::Any::Read => op_read, msg::Any::Remove => op_remove, msg::Any::Rename => op_rename, msg::Any::ReplReadline => op_repl_readline, @@ -117,6 +118,7 @@ pub fn dispatch( msg::Any::Resources => op_resources, msg::Any::Run => op_run, msg::Any::RunStatus => op_run_status, + msg::Any::Seek => op_seek, msg::Any::SetEnv => op_set_env, msg::Any::Shutdown => op_shutdown, msg::Any::Start => op_start, @@ -127,21 +129,21 @@ pub fn dispatch( msg::Any::WorkerPostMessage => op_worker_post_message, msg::Any::Write => op_write, msg::Any::WriteFile => op_write_file, - msg::Any::Now => op_now, - msg::Any::IsTTY => op_is_tty, - msg::Any::Seek => op_seek, - msg::Any::Permissions => op_permissions, - msg::Any::PermissionRevoke => op_revoke_permission, _ => panic!(format!( "Unhandled message {}", msg::enum_name_any(inner_type) )), }; - op_creator(&isolate, &base, data) + op_creator(&cli, &base, zero_copy) }; + cli + .state + .metrics_op_dispatched(bytes_sent_control, bytes_sent_zero_copy); + let state = cli.state.clone(); + let boxed_op = Box::new( - op.or_else(move |err: DenoError| -> DenoResult { + op.or_else(move |err: DenoError| -> Result { debug!("op err {}", err); // No matter whether we got an Err or Ok, we want a serialized message to // send back. So transform the DenoError into a deno_buf. @@ -156,7 +158,7 @@ pub fn dispatch( ..Default::default() }, )) - }).and_then(move |buf: Buf| -> DenoResult { + }).and_then(move |buf: Buf| -> Result { // Handle empty responses. For sync responses we just want // to send null. For async we want to send a small message // with the cmd_id. @@ -172,6 +174,7 @@ pub fn dispatch( }, ) }; + state.metrics_op_completed(buf.len()); Ok(buf) }), ); @@ -184,11 +187,7 @@ pub fn dispatch( (base.sync(), boxed_op) } -fn op_now( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_now(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let start = SystemTime::now(); let since_the_epoch = start.duration_since(UNIX_EPOCH).unwrap(); @@ -208,11 +207,7 @@ fn op_now( )) } -fn op_is_tty( - _isolate: &Isolate, - base: &msg::Base<'_>, - _data: libdeno::deno_buf, -) -> Box { +fn op_is_tty(_cli: &Cli, base: &msg::Base<'_>, _data: deno_buf) -> Box { let builder = &mut FlatBufferBuilder::new(); let inner = msg::IsTTYRes::create( builder, @@ -233,24 +228,16 @@ fn op_is_tty( )) } -fn op_exit( - _isolate: &Isolate, - base: &msg::Base<'_>, - _data: libdeno::deno_buf, -) -> Box { +fn op_exit(_cli: &Cli, base: &msg::Base<'_>, _data: deno_buf) -> Box { let inner = base.inner_as_exit().unwrap(); std::process::exit(inner.code()) } -fn op_start( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_start(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let mut builder = FlatBufferBuilder::new(); - let argv = isolate + let argv = cli .state .argv .iter() @@ -271,10 +258,7 @@ fn op_start( let deno_version = version::DENO; let deno_version_off = builder.create_string(deno_version); - let main_module = isolate - .state - .main_module() - .map(|m| builder.create_string(&m)); + let main_module = cli.state.main_module().map(|m| builder.create_string(&m)); let inner = msg::StartRes::create( &mut builder, @@ -283,9 +267,9 @@ fn op_start( pid: std::process::id(), argv: Some(argv_off), main_module, - debug_flag: isolate.state.flags.log_debug, - types_flag: isolate.state.flags.types, - version_flag: isolate.state.flags.version, + debug_flag: cli.state.flags.log_debug, + types_flag: cli.state.flags.types, + version_flag: cli.state.flags.version, v8_version: Some(v8_version_off), deno_version: Some(deno_version_off), no_color: !ansi::use_color(), @@ -305,17 +289,13 @@ fn op_start( )) } -fn op_format_error( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_format_error(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_format_error().unwrap(); let orig_error = String::from(inner.error().unwrap()); let js_error = JSError::from_v8_exception(&orig_error).unwrap(); - let js_error_mapped = apply_source_map(&js_error, &isolate.state.dir); + let js_error_mapped = apply_source_map(&js_error, &cli.state.dir); let js_error_string = JSErrorColor(&js_error_mapped).to_string(); let mut builder = FlatBufferBuilder::new(); @@ -366,9 +346,9 @@ pub fn odd_future(err: DenoError) -> Box { // https://github.com/denoland/deno/blob/golang/os.go#L100-L154 fn op_fetch_module_meta_data( - isolate: &Isolate, + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_fetch_module_meta_data().unwrap(); @@ -377,35 +357,32 @@ fn op_fetch_module_meta_data( let referrer = inner.referrer().unwrap(); // Check for allow read since this operation could be used to read from the file system. - if !isolate.permissions.allow_read.load(Ordering::SeqCst) { + if !cli.permissions.allow_read.load(Ordering::SeqCst) { debug!("No read permission for fetch_module_meta_data"); return odd_future(permission_denied()); } // Check for allow write since this operation could be used to write to the file system. - if !isolate.permissions.allow_write.load(Ordering::SeqCst) { + if !cli.permissions.allow_write.load(Ordering::SeqCst) { debug!("No network permission for fetch_module_meta_data"); return odd_future(permission_denied()); } // Check for allow net since this operation could be used to make https/http requests. - if !isolate.permissions.allow_net.load(Ordering::SeqCst) { + if !cli.permissions.allow_net.load(Ordering::SeqCst) { debug!("No network permission for fetch_module_meta_data"); return odd_future(permission_denied()); } assert_eq!( - isolate.state.dir.root.join("gen"), - isolate.state.dir.gen, + cli.state.dir.root.join("gen"), + cli.state.dir.gen, "Sanity check" ); Box::new(futures::future::result(|| -> OpResult { let builder = &mut FlatBufferBuilder::new(); - let out = isolate - .state - .dir - .fetch_module_meta_data(specifier, referrer)?; + let out = cli.state.dir.fetch_module_meta_data(specifier, referrer)?; let data_off = builder.create_vector(out.source_code.as_slice()); let msg_args = msg::FetchModuleMetaDataResArgs { module_name: Some(builder.create_string(&out.module_name)), @@ -426,11 +403,7 @@ fn op_fetch_module_meta_data( }())) } -fn op_chdir( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_chdir(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_chdir().unwrap(); let directory = inner.directory().unwrap(); @@ -440,48 +413,63 @@ fn op_chdir( }())) } -fn op_set_timeout( - isolate: &Isolate, +fn op_global_timer_stop( + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { + assert!(base.sync()); assert_eq!(data.len(), 0); - let inner = base.inner_as_set_timeout().unwrap(); - let val = inner.timeout(); - let timeout_due = if val >= 0 { - Some(Instant::now() + Duration::from_millis(val as u64)) - } else { - None - }; - isolate.set_timeout_due(timeout_due); + let mut t = cli.state.global_timer.lock().unwrap(); + t.cancel(); ok_future(empty_buf()) } -fn op_set_env( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_global_timer(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { + assert!(!base.sync()); + assert_eq!(data.len(), 0); + let cmd_id = base.cmd_id(); + let inner = base.inner_as_global_timer().unwrap(); + let val = inner.timeout(); + assert!(val >= 0); + + let mut t = cli.state.global_timer.lock().unwrap(); + let deadline = Instant::now() + Duration::from_millis(val as u64); + let f = t.new_timeout(deadline); + + Box::new(f.then(move |_| { + let builder = &mut FlatBufferBuilder::new(); + let inner = + msg::GlobalTimerRes::create(builder, &msg::GlobalTimerResArgs {}); + Ok(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(inner.as_union_value()), + inner_type: msg::Any::GlobalTimerRes, + ..Default::default() + }, + )) + })) +} + +fn op_set_env(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_set_env().unwrap(); let key = inner.key().unwrap(); let value = inner.value().unwrap(); - if let Err(e) = isolate.check_env() { + if let Err(e) = cli.check_env() { return odd_future(e); } std::env::set_var(key, value); ok_future(empty_buf()) } -fn op_env( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_env(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); - if let Err(e) = isolate.check_env() { + if let Err(e) = cli.check_env() { return odd_future(e); } @@ -505,22 +493,18 @@ fn op_env( )) } -fn op_permissions( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_permissions(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); let builder = &mut FlatBufferBuilder::new(); let inner = msg::PermissionsRes::create( builder, &msg::PermissionsResArgs { - run: isolate.permissions.allows_run(), - read: isolate.permissions.allows_read(), - write: isolate.permissions.allows_write(), - net: isolate.permissions.allows_net(), - env: isolate.permissions.allows_env(), + run: cli.permissions.allows_run(), + read: cli.permissions.allows_read(), + write: cli.permissions.allows_write(), + net: cli.permissions.allows_net(), + env: cli.permissions.allows_env(), }, ); ok_future(serialize_response( @@ -535,19 +519,19 @@ fn op_permissions( } fn op_revoke_permission( - isolate: &Isolate, + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_permission_revoke().unwrap(); let permission = inner.permission().unwrap(); let result = match permission { - "run" => isolate.permissions.revoke_run(), - "read" => isolate.permissions.revoke_read(), - "write" => isolate.permissions.revoke_write(), - "net" => isolate.permissions.revoke_net(), - "env" => isolate.permissions.revoke_env(), + "run" => cli.permissions.revoke_run(), + "read" => cli.permissions.revoke_read(), + "write" => cli.permissions.revoke_write(), + "net" => cli.permissions.revoke_net(), + "env" => cli.permissions.revoke_env(), _ => Ok(()), }; if let Err(e) = result { @@ -556,11 +540,7 @@ fn op_revoke_permission( ok_future(empty_buf()) } -fn op_fetch( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_fetch(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { let inner = base.inner_as_fetch().unwrap(); let cmd_id = base.cmd_id(); @@ -580,7 +560,7 @@ fn op_fetch( } let req = maybe_req.unwrap(); - if let Err(e) = isolate.check_net(url) { + if let Err(e) = cli.check_net(url) { return odd_future(e); } @@ -644,9 +624,9 @@ where } fn op_make_temp_dir( - isolate: &Isolate, + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { assert_eq!(data.len(), 0); let base = Box::new(*base); @@ -654,7 +634,7 @@ fn op_make_temp_dir( let cmd_id = base.cmd_id(); // FIXME - if let Err(e) = isolate.check_write("make_temp") { + if let Err(e) = cli.check_write("make_temp") { return odd_future(e); } @@ -692,18 +672,14 @@ fn op_make_temp_dir( }) } -fn op_mkdir( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_mkdir(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_mkdir().unwrap(); let path = String::from(inner.path().unwrap()); let recursive = inner.recursive(); let mode = inner.mode(); - if let Err(e) = isolate.check_write(&path) { + if let Err(e) = cli.check_write(&path) { return odd_future(e); } @@ -714,17 +690,13 @@ fn op_mkdir( }) } -fn op_chmod( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_chmod(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_chmod().unwrap(); let _mode = inner.mode(); let path = String::from(inner.path().unwrap()); - if let Err(e) = isolate.check_write(&path) { + if let Err(e) = cli.check_write(&path) { return odd_future(e); } @@ -753,11 +725,7 @@ fn op_chmod( }) } -fn op_open( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_open(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); let inner = base.inner_as_open().unwrap(); @@ -803,20 +771,20 @@ fn op_open( match mode { "r" => { - if let Err(e) = isolate.check_read(&filename_str) { + if let Err(e) = cli.check_read(&filename_str) { return odd_future(e); } } "w" | "a" | "x" => { - if let Err(e) = isolate.check_write(&filename_str) { + if let Err(e) = cli.check_write(&filename_str) { return odd_future(e); } } &_ => { - if let Err(e) = isolate.check_read(&filename_str) { + if let Err(e) = cli.check_read(&filename_str) { return odd_future(e); } - if let Err(e) = isolate.check_write(&filename_str) { + if let Err(e) = cli.check_write(&filename_str) { return odd_future(e); } } @@ -843,11 +811,7 @@ fn op_open( Box::new(op) } -fn op_close( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_close(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_close().unwrap(); let rid = inner.rid(); @@ -860,11 +824,7 @@ fn op_close( } } -fn op_shutdown( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_shutdown(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_shutdown().unwrap(); let rid = inner.rid(); @@ -886,11 +846,7 @@ fn op_shutdown( } } -fn op_read( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_read(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { let cmd_id = base.cmd_id(); let inner = base.inner_as_read().unwrap(); let rid = inner.rid(); @@ -924,11 +880,7 @@ fn op_read( } } -fn op_write( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_write(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { let cmd_id = base.cmd_id(); let inner = base.inner_as_write().unwrap(); let rid = inner.rid(); @@ -961,11 +913,7 @@ fn op_write( } } -fn op_seek( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_seek(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let _cmd_id = base.cmd_id(); let inner = base.inner_as_seek().unwrap(); @@ -983,18 +931,14 @@ fn op_seek( } } -fn op_remove( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_remove(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_remove().unwrap(); let path_ = inner.path().unwrap(); let path = PathBuf::from(path_); let recursive = inner.recursive(); - if let Err(e) = isolate.check_write(path.to_str().unwrap()) { + if let Err(e) = cli.check_write(path.to_str().unwrap()) { return odd_future(e); } @@ -1013,18 +957,14 @@ fn op_remove( } // Prototype https://github.com/denoland/deno/blob/golang/os.go#L171-L184 -fn op_read_file( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_read_file(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_read_file().unwrap(); let cmd_id = base.cmd_id(); let filename_ = inner.filename().unwrap(); let filename = PathBuf::from(filename_); debug!("op_read_file {}", filename.display()); - if let Err(e) = isolate.check_read(&filename_) { + if let Err(e) = cli.check_read(&filename_) { return odd_future(e); } blocking(base.sync(), move || { @@ -1051,11 +991,7 @@ fn op_read_file( }) } -fn op_copy_file( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_copy_file(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_copy_file().unwrap(); let from_ = inner.from().unwrap(); @@ -1063,10 +999,10 @@ fn op_copy_file( let to_ = inner.to().unwrap(); let to = PathBuf::from(to_); - if let Err(e) = isolate.check_read(&from_) { + if let Err(e) = cli.check_read(&from_) { return odd_future(e); } - if let Err(e) = isolate.check_write(&to_) { + if let Err(e) = cli.check_write(&to_) { return odd_future(e); } @@ -1107,11 +1043,7 @@ fn get_mode(_perm: &fs::Permissions) -> u32 { 0 } -fn op_cwd( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_cwd(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); Box::new(futures::future::result(|| -> OpResult { @@ -1133,11 +1065,7 @@ fn op_cwd( }())) } -fn op_stat( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_stat(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_stat().unwrap(); let cmd_id = base.cmd_id(); @@ -1145,7 +1073,7 @@ fn op_stat( let filename = PathBuf::from(filename_); let lstat = inner.lstat(); - if let Err(e) = isolate.check_read(&filename_) { + if let Err(e) = cli.check_read(&filename_) { return odd_future(e); } @@ -1185,17 +1113,13 @@ fn op_stat( }) } -fn op_read_dir( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_read_dir(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_read_dir().unwrap(); let cmd_id = base.cmd_id(); let path = String::from(inner.path().unwrap()); - if let Err(e) = isolate.check_read(&path) { + if let Err(e) = cli.check_read(&path) { return odd_future(e); } @@ -1246,11 +1170,7 @@ fn op_read_dir( }) } -fn op_write_file( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_write_file(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { let inner = base.inner_as_write_file().unwrap(); let filename = String::from(inner.filename().unwrap()); let update_perm = inner.update_perm(); @@ -1258,7 +1178,7 @@ fn op_write_file( let is_create = inner.is_create(); let is_append = inner.is_append(); - if let Err(e) = isolate.check_write(&filename) { + if let Err(e) = cli.check_write(&filename) { return odd_future(e); } @@ -1276,17 +1196,13 @@ fn op_write_file( }) } -fn op_rename( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_rename(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_rename().unwrap(); let oldpath = PathBuf::from(inner.oldpath().unwrap()); let newpath_ = inner.newpath().unwrap(); let newpath = PathBuf::from(newpath_); - if let Err(e) = isolate.check_write(&newpath_) { + if let Err(e) = cli.check_write(&newpath_) { return odd_future(e); } blocking(base.sync(), move || -> OpResult { @@ -1296,18 +1212,14 @@ fn op_rename( }) } -fn op_symlink( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_symlink(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_symlink().unwrap(); let oldname = PathBuf::from(inner.oldname().unwrap()); let newname_ = inner.newname().unwrap(); let newname = PathBuf::from(newname_); - if let Err(e) = isolate.check_write(&newname_) { + if let Err(e) = cli.check_write(&newname_) { return odd_future(e); } // TODO Use type for Windows. @@ -1325,18 +1237,14 @@ fn op_symlink( }) } -fn op_read_link( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_read_link(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_readlink().unwrap(); let cmd_id = base.cmd_id(); let name_ = inner.name().unwrap(); let name = PathBuf::from(name_); - if let Err(e) = isolate.check_read(&name_) { + if let Err(e) = cli.check_read(&name_) { return odd_future(e); } @@ -1363,18 +1271,14 @@ fn op_read_link( }) } -fn op_repl_start( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_repl_start(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_repl_start().unwrap(); let cmd_id = base.cmd_id(); let history_file = String::from(inner.history_file().unwrap()); debug!("op_repl_start {}", history_file); - let history_path = repl::history_path(&isolate.state.dir, &history_file); + let history_path = repl::history_path(&cli.state.dir, &history_file); let repl = repl::Repl::new(history_path); let resource = resources::add_repl(repl); @@ -1395,9 +1299,9 @@ fn op_repl_start( } fn op_repl_readline( - _isolate: &Isolate, + _cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_repl_readline().unwrap(); @@ -1430,18 +1334,14 @@ fn op_repl_readline( }) } -fn op_truncate( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_truncate(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let inner = base.inner_as_truncate().unwrap(); let filename = String::from(inner.name().unwrap()); let len = inner.len(); - if let Err(e) = isolate.check_write(&filename) { + if let Err(e) = cli.check_write(&filename) { return odd_future(e); } @@ -1453,13 +1353,9 @@ fn op_truncate( }) } -fn op_listen( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_listen(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); - if let Err(e) = isolate.check_net("listen") { + if let Err(e) = cli.check_net("listen") { return odd_future(e); } @@ -1515,13 +1411,9 @@ fn new_conn(cmd_id: u32, tcp_stream: TcpStream) -> OpResult { )) } -fn op_accept( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_accept(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); - if let Err(e) = isolate.check_net("accept") { + if let Err(e) = cli.check_net("accept") { return odd_future(e); } let cmd_id = base.cmd_id(); @@ -1541,13 +1433,9 @@ fn op_accept( } } -fn op_dial( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_dial(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); - if let Err(e) = isolate.check_net("dial") { + if let Err(e) = cli.check_net("dial") { return odd_future(e); } let cmd_id = base.cmd_id(); @@ -1567,18 +1455,14 @@ fn op_dial( Box::new(op) } -fn op_metrics( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_metrics(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); let builder = &mut FlatBufferBuilder::new(); let inner = msg::MetricsRes::create( builder, - &msg::MetricsResArgs::from(&isolate.state.metrics), + &msg::MetricsResArgs::from(&cli.state.metrics), ); ok_future(serialize_response( cmd_id, @@ -1591,11 +1475,7 @@ fn op_metrics( )) } -fn op_resources( - _isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_resources(_cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); @@ -1643,15 +1523,11 @@ fn subprocess_stdio_map(v: msg::ProcessStdio) -> std::process::Stdio { } } -fn op_run( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_run(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert!(base.sync()); let cmd_id = base.cmd_id(); - if let Err(e) = isolate.check_run() { + if let Err(e) = cli.check_run() { return odd_future(e); } @@ -1716,17 +1592,13 @@ fn op_run( )) } -fn op_run_status( - isolate: &Isolate, - base: &msg::Base<'_>, - data: libdeno::deno_buf, -) -> Box { +fn op_run_status(cli: &Cli, base: &msg::Base<'_>, data: deno_buf) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); let inner = base.inner_as_run_status().unwrap(); let rid = inner.rid(); - if let Err(e) = isolate.check_run() { + if let Err(e) = cli.check_run() { return odd_future(e); } @@ -1793,15 +1665,15 @@ impl Future for GetMessageFuture { } fn op_worker_get_message( - isolate: &Isolate, + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { assert_eq!(data.len(), 0); let cmd_id = base.cmd_id(); let op = GetMessageFuture { - state: isolate.state.clone(), + state: cli.state.clone(), }; let op = op.map_err(move |_| -> DenoError { unimplemented!() }); let op = op.and_then(move |maybe_buf| -> DenoResult { @@ -1827,16 +1699,16 @@ fn op_worker_get_message( } fn op_worker_post_message( - isolate: &Isolate, + cli: &Cli, base: &msg::Base<'_>, - data: libdeno::deno_buf, + data: deno_buf, ) -> Box { let cmd_id = base.cmd_id(); let d = Vec::from(data.as_ref()).into_boxed_slice(); - assert!(isolate.state.worker_channels.is_some()); - let tx = match isolate.state.worker_channels { + assert!(cli.state.worker_channels.is_some()); + let tx = match cli.state.worker_channels { None => panic!("expected worker_channels"), Some(ref wc) => { let wc = wc.lock().unwrap(); @@ -1862,14 +1734,15 @@ fn op_worker_post_message( #[cfg(test)] mod tests { use super::*; - use crate::isolate::{Isolate, IsolateState}; + use crate::cli::Cli; use crate::isolate_init::IsolateInit; + use crate::isolate_state::IsolateState; use crate::permissions::DenoPermissions; use std::sync::atomic::AtomicBool; #[test] fn fetch_module_meta_fails_without_read() { - let state = IsolateState::mock(); + let state = Arc::new(IsolateState::mock()); let permissions = DenoPermissions { allow_read: AtomicBool::new(false), allow_write: AtomicBool::new(true), @@ -1877,13 +1750,12 @@ mod tests { allow_net: AtomicBool::new(true), allow_run: AtomicBool::new(true), }; - let isolate = Isolate::new( + let cli = Cli::new( IsolateInit { snapshot: None, init_script: None, }, state, - dispatch, permissions, ); let builder = &mut FlatBufferBuilder::new(); @@ -1901,11 +1773,8 @@ mod tests { msg::finish_base_buffer(builder, base); let data = builder.finished_data(); let final_msg = msg::get_root_as_base(&data); - let fetch_result = op_fetch_module_meta_data( - &isolate, - &final_msg, - libdeno::deno_buf::empty(), - ).wait(); + let fetch_result = + op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait(); match fetch_result { Ok(_) => assert!(true), Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()), @@ -1914,7 +1783,7 @@ mod tests { #[test] fn fetch_module_meta_fails_without_write() { - let state = IsolateState::mock(); + let state = Arc::new(IsolateState::mock()); let permissions = DenoPermissions { allow_read: AtomicBool::new(true), allow_write: AtomicBool::new(false), @@ -1922,13 +1791,12 @@ mod tests { allow_net: AtomicBool::new(true), allow_run: AtomicBool::new(true), }; - let isolate = Isolate::new( + let cli = Cli::new( IsolateInit { snapshot: None, init_script: None, }, state, - dispatch, permissions, ); let builder = &mut FlatBufferBuilder::new(); @@ -1946,11 +1814,8 @@ mod tests { msg::finish_base_buffer(builder, base); let data = builder.finished_data(); let final_msg = msg::get_root_as_base(&data); - let fetch_result = op_fetch_module_meta_data( - &isolate, - &final_msg, - libdeno::deno_buf::empty(), - ).wait(); + let fetch_result = + op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait(); match fetch_result { Ok(_) => assert!(true), Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()), @@ -1959,7 +1824,7 @@ mod tests { #[test] fn fetch_module_meta_fails_without_net() { - let state = IsolateState::mock(); + let state = Arc::new(IsolateState::mock()); let permissions = DenoPermissions { allow_read: AtomicBool::new(true), allow_write: AtomicBool::new(true), @@ -1967,13 +1832,12 @@ mod tests { allow_net: AtomicBool::new(false), allow_run: AtomicBool::new(true), }; - let isolate = Isolate::new( + let cli = Cli::new( IsolateInit { snapshot: None, init_script: None, }, state, - dispatch, permissions, ); let builder = &mut FlatBufferBuilder::new(); @@ -1991,11 +1855,8 @@ mod tests { msg::finish_base_buffer(builder, base); let data = builder.finished_data(); let final_msg = msg::get_root_as_base(&data); - let fetch_result = op_fetch_module_meta_data( - &isolate, - &final_msg, - libdeno::deno_buf::empty(), - ).wait(); + let fetch_result = + op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait(); match fetch_result { Ok(_) => assert!(true), Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()), @@ -2004,7 +1865,7 @@ mod tests { #[test] fn fetch_module_meta_not_permission_denied_with_permissions() { - let state = IsolateState::mock(); + let state = Arc::new(IsolateState::mock()); let permissions = DenoPermissions { allow_read: AtomicBool::new(true), allow_write: AtomicBool::new(true), @@ -2012,13 +1873,12 @@ mod tests { allow_net: AtomicBool::new(true), allow_run: AtomicBool::new(false), }; - let isolate = Isolate::new( + let cli = Cli::new( IsolateInit { snapshot: None, init_script: None, }, state, - dispatch, permissions, ); let builder = &mut FlatBufferBuilder::new(); @@ -2036,11 +1896,8 @@ mod tests { msg::finish_base_buffer(builder, base); let data = builder.finished_data(); let final_msg = msg::get_root_as_base(&data); - let fetch_result = op_fetch_module_meta_data( - &isolate, - &final_msg, - libdeno::deno_buf::empty(), - ).wait(); + let fetch_result = + op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait(); match fetch_result { Ok(_) => assert!(true), Err(e) => assert!(e.to_string() != permission_denied().to_string()), diff --git a/src/resources.rs b/src/resources.rs index 2d617265ea6a54..e962bce8e6134e 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -8,6 +8,7 @@ // descriptors". This module implements a global resource table. Ops (AKA // handlers) look up resources by their integer id here. +use crate::cli::Buf; #[cfg(unix)] use crate::eager_unix as eager; use crate::errors; @@ -15,8 +16,7 @@ use crate::errors::bad_resource; use crate::errors::DenoError; use crate::errors::DenoResult; use crate::http_body::HttpBody; -use crate::isolate::Buf; -use crate::isolate::WorkerChannels; +use crate::isolate_state::WorkerChannels; use crate::repl::Repl; use crate::tokio_util; use crate::tokio_write; @@ -175,50 +175,12 @@ impl Resource { } } - /// Track the current task (for TcpListener resource). - /// Throws an error if another task is already tracked. - pub fn track_task(&mut self) -> Result<(), std::io::Error> { - let mut table = RESOURCE_TABLE.lock().unwrap(); - // Only track if is TcpListener. - if let Some(Repr::TcpListener(_, t)) = table.get_mut(&self.rid) { - // Currently, we only allow tracking a single accept task for a listener. - // This might be changed in the future with multiple workers. - // Caveat: TcpListener by itself also only tracks an accept task at a time. - // See https://github.com/tokio-rs/tokio/issues/846#issuecomment-454208883 - if t.is_some() { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "Another accept task is ongoing", - )); - } - t.replace(futures::task::current()); - } - Ok(()) - } - - /// Stop tracking a task (for TcpListener resource). - /// Happens when the task is done and thus no further tracking is needed. - pub fn untrack_task(&mut self) { - let mut table = RESOURCE_TABLE.lock().unwrap(); - // Only untrack if is TcpListener. - if let Some(Repr::TcpListener(_, t)) = table.get_mut(&self.rid) { - // DO NOT assert is_some here. - // See reasoning in Accept::poll(). - t.take(); - } - } - // close(2) is done by dropping the value. Therefore we just need to remove // the resource from the RESOURCE_TABLE. pub fn close(&self) { let mut table = RESOURCE_TABLE.lock().unwrap(); let r = table.remove(&self.rid); assert!(r.is_some()); - // If TcpListener, we must kill all pending accepts! - if let Repr::TcpListener(_, Some(t)) = r.unwrap() { - // Call notify on the tracked task, so that they would error out. - t.notify(); - } } pub fn shutdown(&mut self, how: Shutdown) -> Result<(), DenoError> { diff --git a/src/tokio_util.rs b/src/tokio_util.rs index 3dddff9c2f2696..ed71c63a49a430 100644 --- a/src/tokio_util.rs +++ b/src/tokio_util.rs @@ -1,6 +1,6 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +#![allow(dead_code)] use crate::resources::Resource; - use futures; use futures::Future; use futures::Poll; @@ -11,6 +11,14 @@ use tokio; use tokio::net::TcpStream; use tokio_executor; +pub fn run(future: F) +where + F: Future + Send + 'static, +{ + // tokio::runtime::current_thread::run(future) + tokio::run(future) +} + pub fn block_on(future: F) -> Result where F: Send + 'static + Future, @@ -63,29 +71,7 @@ impl Future for Accept { fn poll(&mut self) -> Poll { let (stream, addr) = match self.state { - // Similar to try_ready!, but also track/untrack accept task - // in TcpListener resource. - // In this way, when the listener is closed, the task can be - // notified to error out (instead of stuck forever). - AcceptState::Pending(ref mut r) => match r.poll_accept() { - Ok(futures::prelude::Async::Ready(t)) => { - // Notice: it is possible to be Ready on the first poll. - // When eager accept fails due to WouldBlock, - // a next poll() might still be immediately Ready. - // See https://github.com/denoland/deno/issues/1756. - r.untrack_task(); - t - } - Ok(futures::prelude::Async::NotReady) => { - // Would error out if another accept task is being tracked. - r.track_task()?; - return Ok(futures::prelude::Async::NotReady); - } - Err(e) => { - r.untrack_task(); - return Err(e); - } - }, + AcceptState::Pending(ref mut r) => try_ready!(r.poll_accept()), AcceptState::Empty => panic!("poll Accept after it's done"), }; @@ -122,3 +108,11 @@ where f() } } + +pub fn panic_on_error(f: F) -> impl Future +where + F: Future, + E: std::fmt::Debug, +{ + f.map_err(|err| panic!("Future got unexpected error: {:?}", err)) +} diff --git a/src/workers.rs b/src/workers.rs index 3a3b690b2751ef..f9d7c53a4e5a48 100644 --- a/src/workers.rs +++ b/src/workers.rs @@ -1,19 +1,21 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -use crate::isolate::Buf; +use crate::cli::Buf; +use crate::cli::Cli; +use crate::flags::DenoFlags; use crate::isolate::Isolate; -use crate::isolate::IsolateState; -use crate::isolate::WorkerChannels; use crate::isolate_init::IsolateInit; +use crate::isolate_state::IsolateState; +use crate::isolate_state::WorkerChannels; use crate::js_errors::JSErrorColor; -use crate::ops; use crate::permissions::DenoPermissions; use crate::resources; use crate::tokio_util; use deno_core::JSError; - +use futures::future::lazy; use futures::sync::mpsc; use futures::sync::oneshot; use futures::Future; +use futures::Poll; use std::sync::Arc; use std::thread; @@ -25,7 +27,8 @@ pub struct Worker { impl Worker { pub fn new( init: IsolateInit, - parent_state: &Arc, + flags: DenoFlags, + argv: Vec, permissions: DenoPermissions, ) -> (Self, WorkerChannels) { let (worker_in_tx, worker_in_rx) = mpsc::channel::(1); @@ -34,13 +37,11 @@ impl Worker { let internal_channels = (worker_out_tx, worker_in_rx); let external_channels = (worker_in_tx, worker_out_rx); - let state = Arc::new(IsolateState::new( - parent_state.flags.clone(), - parent_state.argv.clone(), - Some(internal_channels), - )); + let state = + Arc::new(IsolateState::new(flags, argv, Some(internal_channels))); - let isolate = Isolate::new(init, state, ops::dispatch, permissions); + let cli = Cli::new(init, state, permissions); + let isolate = Isolate::new(cli); let worker = Worker { isolate }; (worker, external_channels) @@ -49,15 +50,20 @@ impl Worker { pub fn execute(&self, js_source: &str) -> Result<(), JSError> { self.isolate.execute(js_source) } +} - pub fn event_loop(&self) -> Result<(), JSError> { - self.isolate.event_loop() +impl Future for Worker { + type Item = (); + type Error = JSError; + + fn poll(&mut self) -> Poll<(), JSError> { + self.isolate.poll() } } pub fn spawn( init: IsolateInit, - state: Arc, + state: &IsolateState, js_source: String, permissions: DenoPermissions, ) -> resources::Resource { @@ -67,27 +73,38 @@ pub fn spawn( // let (js_error_tx, js_error_rx) = oneshot::channel::(); let (p, c) = oneshot::channel::(); let builder = thread::Builder::new().name("worker".to_string()); - let _tid = builder - .spawn(move || { - let (worker, external_channels) = Worker::new(init, &state, permissions); - let resource = resources::add_worker(external_channels); - p.send(resource.clone()).unwrap(); + let flags = state.flags.clone(); + let argv = state.argv.clone(); - tokio_util::init(|| { - (|| -> Result<(), JSError> { - worker.execute("denoMain()")?; - worker.execute("workerMain()")?; - worker.execute(&js_source)?; - worker.event_loop()?; + let _tid = builder + .spawn(move || { + tokio_util::run(lazy(move || { + let (worker, external_channels) = + Worker::new(init, flags, argv, permissions); + let resource = resources::add_worker(external_channels); + p.send(resource.clone()).unwrap(); + + worker + .execute("denoMain()") + .expect("worker denoMain failed"); + worker + .execute("workerMain()") + .expect("worker workerMain failed"); + worker.execute(&js_source).expect("worker js_source failed"); + + worker.then(move |r| -> Result<(), ()> { + resource.close(); + debug!("workers.rs after resource close"); + if let Err(err) = r { + eprintln!("{}", JSErrorColor(&err).to_string()); + std::process::exit(1); + } Ok(()) - })().or_else(|err: JSError| -> Result<(), JSError> { - eprintln!("{}", JSErrorColor(&err).to_string()); - std::process::exit(1) - }).unwrap(); - }); + }) + })); - resource.close(); + debug!("workers.rs after spawn"); }).unwrap(); c.wait().unwrap() @@ -103,7 +120,7 @@ mod tests { let isolate_init = isolate_init::compiler_isolate_init(); let resource = spawn( isolate_init, - IsolateState::mock(), + &IsolateState::mock(), r#" onmessage = function(e) { let s = new TextDecoder().decode(e.data);; @@ -140,7 +157,7 @@ mod tests { let isolate_init = isolate_init::compiler_isolate_init(); let resource = spawn( isolate_init, - IsolateState::mock(), + &IsolateState::mock(), "onmessage = () => close();".into(), DenoPermissions::default(), );