feat(compiler): implement remaining serialization features

.vscode/settings.json

@@ -17,5 +17,6 @@
     "**/bower_components": true,
     "**/lib": true,
     "**/lib-test": true
-  }
+  },
+  "typescript.tsdk": "node_modules/typescript/lib"
 }

gulpfile.js

@@ -20,7 +20,8 @@ const CAPNP_WHITELIST = [
   'capnp-ts/test/integration/list-mania.capnp',
   'capnp-ts/test/integration/upgrade-v1.capnp',
   'capnpc-ts/test/integration/import-bar.capnp',
-  'capnpc-ts/test/integration/import-foo.capnp'
+  'capnpc-ts/test/integration/import-foo.capnp',
+  'capnpc-ts/test/integration/test.capnp'
 ];
 
 
@@ -38,7 +39,8 @@ function compileCapnp() {
     files.filter(function (file) {
       return file.path.endsWith('.capnp') && CAPNP_WHITELIST.some((p) => file.path.endsWith(p));
     }).forEach(function (file) {
-      var options = ['-o./packages/capnpc-ts/bin/capnpc-ts.js', '-I', path.join(__dirname, 'src', 'std'), file.path];
+      gutil.log('Compiling', "'" + gutil.colors.cyan(path.relative(__dirname, file.path)) + "'...");
+      var options = ['-o./packages/capnpc-ts/bin/capnpc-ts.ts', '-I', path.join(__dirname, 'src', 'std'), file.path];
       var result = spawnSync('capnpc', options, { stdio: 'inherit' });
       if (result.status !== 0) {
         throw new Error('Process exited with non-zero status: ' + result.status);

packages/capnp-ts/src/errors.ts

@@ -138,3 +138,8 @@ export const TYPE_GET_GENERIC_LIST =
   'CAPNP-TS041 Attempted to call get() on a generic list.';
 export const TYPE_SET_GENERIC_LIST =
   'CAPNP-TS042 Attempted to call set() on a generic list.';
+
+export const PTR_WRITE_CONST_LIST =
+  'CAPNP-TS043 Attempted to write to a const list.';
+export const PTR_WRITE_CONST_STRUCT =
+  'CAPNP-TS044 Attempted to write to a const struct.';

packages/capnp-ts/src/index.ts

@@ -7,6 +7,7 @@ export {
   ListElementSize,
   Message,
   ObjectSize,
+  readRawPointer,
 
   AnyPointerList,
   BoolList,

packages/capnp-ts/src/serialization/index.ts

@@ -4,6 +4,6 @@
 
 export * from './mask';
 export { ListElementSize } from './list-element-size';
-export { Message } from './message';
+export { Message, readRawPointer } from './message';
 export { ObjectSize } from './object-size';
 export * from './pointers';

packages/capnp-ts/src/serialization/mask.ts

@@ -2,7 +2,9 @@
  * @author jdiaz5513
  */
 
-import {Int64, Uint64} from '../types';
+import { Int64, Uint64 } from '../types';
+import { Struct } from './pointers';
+import { ObjectSize } from './object-size';
 
 type DataViewSetter = (byteOffset: number, value: number, littleEndian?: boolean) => void;
 

packages/capnp-ts/src/serialization/message.ts

@@ -10,7 +10,7 @@ import {
 import { dumpBuffer, format, padToWord } from '../util';
 import { AnyArena, Arena, MultiSegmentArena, SingleSegmentArena } from './arena';
 import { pack, unpack } from './packing';
-import { StructCtor, PointerType, Struct } from './pointers';
+import { Pointer, StructCtor, PointerType, Struct } from './pointers';
 import { Segment } from './segment';
 import { getTargetStructSize, validate } from './pointers/pointer';
 import { resize, initStruct } from './pointers/struct';
@@ -32,6 +32,7 @@ export class Message {
   static readonly getRoot = getRoot;
   static readonly getSegment = getSegment;
   static readonly initRoot = initRoot;
+  static readonly readRawPointer = readRawPointer;
   static readonly toArrayBuffer = toArrayBuffer;
   static readonly toPackedArrayBuffer = toPackedArrayBuffer;
 
@@ -120,6 +121,16 @@ export class Message {
 
   initRoot<T extends Struct>(RootStruct: StructCtor<T>) { return initRoot(RootStruct, this); }
 
+  /**
+   * Set the root of the message to a copy of the given pointer. Used internally
+   * to make copies of pointers for default values.
+   *
+   * @param {Pointer} src The source pointer to copy.
+   * @returns {void}
+   */
+
+   setRoot(src: Pointer) { return setRoot(src, this); }
+
   /**
    * Combine the contents of this message's segments into a single array buffer and prepend a stream framing header
    * containing information about the following segment data.
@@ -401,6 +412,30 @@ export function initRoot<T extends Struct>(RootStruct: StructCtor<T>, m: Message
 
 }
 
+/**
+ * Read a pointer in raw form (a packed message with framing headers). Does not
+ * care or attempt to validate the input beyond parsing the message
+ * segments.
+ *
+ * This is typically used by the compiler to load default values, but can be
+ * useful to work with messages with an unknown schema.
+ *
+ * @param {ArrayBuffer} data The raw data to read.
+ * @returns {Pointer} A root pointer.
+ */
+
+export function readRawPointer(data: ArrayBuffer): Pointer {
+
+  return new Pointer(new Message(data).getSegment(0), 0);
+
+}
+
+export function setRoot(src: Pointer, m: Message): void {
+
+  Pointer.copyFrom(src, new Pointer(m.getSegment(0), 0));
+
+}
+
 export function toArrayBuffer(m: Message): ArrayBuffer {
 
   const streamFrame = getStreamFrame(m);

packages/capnp-ts/src/serialization/pointers/list.ts

@@ -47,7 +47,9 @@ export class List<T> extends Pointer {
     displayName: 'List<Generic>' as string,
     size: ListElementSize.VOID,
   };
+  static readonly get = get;
   static readonly initList = initList;
+  static readonly set = set;
 
   static toString(): string {
 
@@ -172,20 +174,32 @@ export class List<T> extends Pointer {
 
   }
 
-  forEach(callbackfn: (this: void, value: T, index: number) => void): void {
+  findIndex(callbackfn: FilterCallback<T>): number {
 
     const length = this.getLength();
 
-    for (let i = 0; i < length; i++) callbackfn(this.get(i), i);
+    for (let i = 0; i < length; i++) {
+
+      const value = this.get(i);
+
+      if (callbackfn(value, i)) return i;
+
+    }
+
+    return -1;
 
   }
 
-  get(_index: number): T {
+  forEach(callbackfn: (this: void, value: T, index: number) => void): void {
+
+    const length = this.getLength();
 
-    throw new TypeError();
+    for (let i = 0; i < length; i++) callbackfn(this.get(i), i);
 
   }
 
+  get(_index: number): T { return get(_index, this); }
+
   /**
    * Get the length of this list.
    *
@@ -266,11 +280,7 @@ export class List<T> extends Pointer {
 
   }
 
-  set(_index: number, _value: T): void {
-
-    throw new TypeError();
-
-  }
+  set(_index: number, _value: T): void { return set(_index, _value, this); }
 
   slice(start = 0, end?: number): T[] {
 
@@ -407,3 +417,14 @@ export function initList<T>(
 
 }
 
+export function get<T>(_index: number, _l: List<T>): T {
+
+  throw new TypeError();
+
+}
+
+export function set<T>(_index: number, _value: T, _l: List<T>): void {
+
+  throw new TypeError();
+
+}

packages/capnp-ts/src/serialization/pointers/pointer.ts

@@ -1032,7 +1032,7 @@ export function copyFromList(src: Pointer, dst: Pointer): void {
 
   if (srcElementSize === ListElementSize.POINTER) {
 
-    dstContent = dst.segment.allocate(getByteLength(getTargetCompositeListSize(src)) * srcLength);
+    dstContent = dst.segment.allocate(srcLength << 3);
 
     // Recursively copy each pointer in the list.
 

packages/capnp-ts/src/serialization/pointers/struct.ts

@@ -22,7 +22,7 @@ import { PointerType } from './pointer-type';
 import { Text } from './text';
 import {
   PTR_INIT_COMPOSITE_STRUCT, PTR_ADOPT_COMPOSITE_STRUCT, PTR_DISOWN_COMPOSITE_STRUCT, PTR_INVALID_UNION_ACCESS,
-  PTR_STRUCT_DATA_OUT_OF_BOUNDS, PTR_STRUCT_POINTER_OUT_OF_BOUNDS,
+  PTR_STRUCT_DATA_OUT_OF_BOUNDS, PTR_STRUCT_POINTER_OUT_OF_BOUNDS, INVARIANT_UNREACHABLE_CODE,
 } from '../../errors';
 
 const trace = initTrace('capnp:struct');
@@ -289,7 +289,7 @@ export function getBit(bitOffset: number, s: Struct, defaultMask?: DataView): bo
 
 }
 
-export function getData(index: number, s: Struct): Data {
+export function getData(index: number, s: Struct, defaultValue?: Pointer): Data {
 
   checkPointerBounds(index, s);
 
@@ -301,7 +301,15 @@ export function getData(index: number, s: Struct): Data {
 
   if (isNull(l)) {
 
-    List.initList(ListElementSize.BYTE, 0, l);
+    if (defaultValue) {
+
+      Pointer.copyFrom(defaultValue, l);
+
+    } else {
+
+      List.initList(ListElementSize.BYTE, 0, l);
+
+    }
 
   }
 
@@ -460,7 +468,7 @@ export function getInt8(byteOffset: number, s: Struct, defaultMask?: DataView):
 
 }
 
-export function getList<T>(index: number, ListClass: ListCtor<T>, s: Struct): List<T> {
+export function getList<T>(index: number, ListClass: ListCtor<T>, s: Struct, defaultValue?: Pointer): List<T> {
 
   checkPointerBounds(index, s);
 
@@ -472,7 +480,15 @@ export function getList<T>(index: number, ListClass: ListCtor<T>, s: Struct): Li
 
   if (isNull(l)) {
 
-    List.initList(ListClass._capnp.size, 0, l, ListClass._capnp.compositeSize);
+    if (defaultValue) {
+
+      Pointer.copyFrom(defaultValue, l);
+
+    } else {
+
+      List.initList(ListClass._capnp.size, 0, l, ListClass._capnp.compositeSize);
+
+    }
 
   } else if (ListClass._capnp.compositeSize !== undefined) {
 
@@ -612,13 +628,23 @@ export function getSize(s: Struct): ObjectSize {
 
 }
 
-export function getStruct<T extends Struct>(index: number, StructClass: StructCtor<T>, s: Struct): T {
+export function getStruct<T extends Struct>(
+  index: number, StructClass: StructCtor<T>, s: Struct, defaultValue?: Pointer,
+): T {
 
   const t = getPointerAs(index, StructClass, s);
 
   if (isNull(t)) {
 
-    initStruct(StructClass._capnp.size, t);
+    if (defaultValue) {
+
+      Pointer.copyFrom(defaultValue, t);
+
+    } else {
+
+      initStruct(StructClass._capnp.size, t);
+
+    }
 
   } else {
 
@@ -646,9 +672,14 @@ export function getStruct<T extends Struct>(index: number, StructClass: StructCt
 
 }
 
-export function getText(index: number, s: Struct): string {
+export function getText(index: number, s: Struct, defaultValue?: string): string {
+
+  const t = Text.fromPointer(getPointer(index, s));
+
+  // FIXME: This will perform an unnecessary string<>ArrayBuffer roundtrip.
+  if (isNull(t) && defaultValue) t.set(0, defaultValue);
 
-  return Text.fromPointer(getPointer(index, s)).get(0);
+  return t.get(0);
 
 }
 

packages/capnp-ts/src/std/schema.capnp.ts

@@ -1,9 +1,9 @@
+/* tslint:disable */
+
 /**
  * This file has been automatically generated by the [capnpc-ts utility](https://github.com/jdiaz5513/capnp-ts).
  */
 
-/* tslint:disable */
-
 import * as capnp from "../index";
 import { ObjectSize as __O, Struct as __S } from '../index';
 export const _capnpFileId = "a93fc509624c72d9";
@@ -279,12 +279,11 @@ export enum Field_Which {
     SLOT = 0,
     GROUP = 1
 }
-const FIELD_DISCRIMINANT_VALUE_DEFAULT = capnp.getUint16Mask(65535);
 export class Field extends __S {
     static readonly NO_DISCRIMINANT = 65535;
     static readonly SLOT = Field_Which.SLOT;
     static readonly GROUP = Field_Which.GROUP;
-    static readonly _capnp = { displayName: "Field", id: "9aad50a41f4af45f", size: new __O(24, 4) };
+    static readonly _capnp = { displayName: "Field", id: "9aad50a41f4af45f", size: new __O(24, 4), defaultDiscriminantValue: capnp.getUint16Mask(65535) };
     static _Annotations: capnp.ListCtor<Annotation>;
     getName(): string { return __S.getText(0, this); }
     setName(value: string): void { __S.setText(0, value, this); }
@@ -296,7 +295,7 @@ export class Field extends __S {
     hasAnnotations(): boolean { return !__S.isNull(__S.getPointer(1, this)); }
     initAnnotations(length: number): capnp.List<Annotation> { return __S.initList(1, Field._Annotations, length, this); }
     setAnnotations(value: capnp.List<Annotation>): void { __S.copyFrom(value, __S.getPointer(1, this)); }
-    getDiscriminantValue(): number { return __S.getUint16(2, this, FIELD_DISCRIMINANT_VALUE_DEFAULT); }
+    getDiscriminantValue(): number { return __S.getUint16(2, this, Field._capnp.defaultDiscriminantValue); }
     setDiscriminantValue(value: number): void { __S.setUint16(2, value, this); }
     getSlot(): Field_Slot {
         __S.testWhich("slot", __S.getUint16(8, this), 0, this);

packages/capnp-ts/src/util.ts

@@ -6,8 +6,8 @@
 /* tslint:disable:no-any no-unsafe-any */
 import initTrace from 'debug';
 
-import {MAX_BUFFER_DUMP_BYTES, MAX_INT32, MAX_UINT32} from './constants';
-import {RANGE_INT32_OVERFLOW, RANGE_INVALID_UTF8, RANGE_UINT32_OVERFLOW} from './errors';
+import { MAX_BUFFER_DUMP_BYTES, MAX_INT32, MAX_UINT32 } from './constants';
+import { RANGE_INT32_OVERFLOW, RANGE_INVALID_UTF8, RANGE_UINT32_OVERFLOW } from './errors';
 
 const trace = initTrace('capnp:util');
 trace('load');

packages/capnp-ts/test/integration/foo-new.capnp.ts

@@ -1,9 +1,9 @@
+/* tslint:disable */
+
 /**
  * This file has been automatically generated by the [capnpc-ts utility](https://github.com/jdiaz5513/capnp-ts).
  */
 
-/* tslint:disable */
-
 import * as capnp from "../../lib/index";
 import { ObjectSize as __O, Struct as __S } from '../../lib/index';
 export const _capnpFileId = "e0b7ff464fbc7ee1";

packages/capnp-ts/test/integration/foo.capnp.ts

@@ -1,9 +1,9 @@
+/* tslint:disable */
+
 /**
  * This file has been automatically generated by the [capnpc-ts utility](https://github.com/jdiaz5513/capnp-ts).
  */
 
-/* tslint:disable */
-
 import * as capnp from "../../lib/index";
 import { ObjectSize as __O, Struct as __S } from '../../lib/index';
 export const _capnpFileId = "e0b7ff464fbc7ee1";