implemented experimental compressed encoder/decoders

decoding.js

@@ -28,6 +28,7 @@
 
 import * as buffer from './buffer.js'
 import * as binary from './binary.js'
+import * as math from './math.js'
 
 /**
  * A Decoder handles the decoding of an Uint8Array.
@@ -414,3 +415,149 @@ const readAnyLookupTable = [
  * @param {Decoder} decoder
  */
 export const readAny = decoder => readAnyLookupTable[127 - readUint8(decoder)](decoder)
+
+/**
+ * T must not be null.
+ *
+ * @template T
+ */
+export class RleDecoder extends Decoder {
+  /**
+   * @param {Uint8Array} uint8Array
+   * @param {function(Decoder):T} reader
+   */
+  constructor (uint8Array, reader) {
+    super(uint8Array)
+    /**
+     * The reader
+     */
+    this.reader = reader
+    /**
+     * Current state
+     * @type {T|null}
+     */
+    this.s = null
+    this.count = 0
+  }
+
+  read () {
+    if (this.count === 0) {
+      this.s = this.reader(this)
+      if (hasContent(this)) {
+        this.count = readVarUint(this) + 1 // see encoder implementation for the reason why this is incremented
+      } else {
+        this.count = -1 // read the current value forever
+      }
+    }
+    this.count--
+    return this.s
+  }
+}
+
+export class IntDiffDecoder extends Decoder {
+  /**
+   * @param {Uint8Array} uint8Array
+   * @param {number} start
+   */
+  constructor (uint8Array, start) {
+    super(uint8Array)
+    /**
+     * Current state
+     * @type {number}
+     */
+    this.s = start
+  }
+
+  /**
+   * @return {number}
+   */
+  read () {
+    this.s += readVarInt(this)
+    return this.s
+  }
+}
+
+export class RleIntDiffDecoder extends Decoder {
+  /**
+   * @param {Uint8Array} uint8Array
+   * @param {number} start
+   */
+  constructor (uint8Array, start) {
+    super(uint8Array)
+    /**
+     * Current state
+     * @type {number}
+     */
+    this.s = start
+    this.count = 0
+  }
+
+  /**
+   * @return {number}
+   */
+  read () {
+    if (this.count === 0) {
+      this.s += readVarInt(this)
+      if (hasContent(this)) {
+        this.count = readVarUint(this) + 1 // see encoder implementation for the reason why this is incremented
+      } else {
+        this.count = -1 // read the current value forever
+      }
+    }
+    this.count--
+    return this.s
+  }
+}
+
+export class UintOptRleDecoder extends Decoder {
+  /**
+   * @param {Uint8Array} uint8Array
+   */
+  constructor (uint8Array) {
+    super(uint8Array)
+    /**
+     * @type {number}
+     */
+    this.s = 0
+    this.count = 0
+  }
+
+  read () {
+    if (this.count === 0) {
+      this.s = readVarInt(this)
+      // if the sign is negative, we read the count too, otherwise count is 1
+      const isNegative = math.isNegativeZero(this.s)
+      this.count = 1
+      if (isNegative) {
+        this.s = -this.s
+        this.count = readVarUint(this) + 2
+      }
+    }
+    this.count--
+    return this.s
+  }
+}
+
+export class StringDecoder {
+  /**
+   * @param {Uint8Array} uint8Array
+   */
+  constructor (uint8Array) {
+    this.decoder = new UintOptRleDecoder(uint8Array)
+    this.str = readVarString(this.decoder)
+    /**
+     * @type {number}
+     */
+    this.spos = 0
+  }
+
+  /**
+   * @return {string}
+   */
+  read () {
+    const end = this.spos + this.decoder.read()
+    const res = this.str.slice(this.spos, end)
+    this.spos = end
+    return res
+  }
+}

encoding.js

@@ -239,15 +239,19 @@ export const writeVarUint = (encoder, num) => {
  *
  * Encodes integers in the range from [-2147483648, -2147483647].
  *
+ * We don't use zig-zag encoding because we want to keep the option open
+ * to use the same function for BigInt and 53bit integers (doubles).
+ *
+ * We use the 7th bit instead for signaling that this is a negative number.
+ *
  * @function
  * @param {Encoder} encoder
  * @param {number} num The number that is to be encoded.
  */
 export const writeVarInt = (encoder, num) => {
-  let isNegative = false
-  if (num < 0) {
+  const isNegative = math.isNegativeZero(num)
+  if (isNegative) {
     num = -num
-    isNegative = true
   }
   //             |- whether to continue reading         |- whether is negative     |- number
   write(encoder, (num > binary.BITS6 ? binary.BIT8 : 0) | (isNegative ? binary.BIT7 : 0) | (binary.BITS6 & num))
@@ -485,3 +489,176 @@ export const writeAny = (encoder, data) => {
       write(encoder, 127)
   }
 }
+
+/**
+ * T must not be null.
+ *
+ * @template T
+ */
+export class RleEncoder extends Encoder {
+  /**
+   * @param {function(Encoder, T):void} writer
+   */
+  constructor (writer) {
+    super()
+    /**
+     * The writer
+     */
+    this.w = writer
+    /**
+     * Current state
+     * @type {T|null}
+     */
+    this.s = null
+    this.count = 0
+  }
+
+  /**
+   * @param {T} v
+   */
+  write (v) {
+    if (this.s === v) {
+      this.count++
+    } else {
+      if (this.count > 0) {
+        // flush counter, unless this is the first value (count = 0)
+        writeVarUint(this, this.count - 1) // since count is always > 0, we can decrement by one. non-standard encoding ftw
+      }
+      this.count = 1
+      // write first value
+      this.w(this, v)
+      this.s = v
+    }
+  }
+}
+
+export class IntDiffEncoder extends Encoder {
+  /**
+   * @param {number} start
+   */
+  constructor (start) {
+    super()
+    /**
+     * Current state
+     * @type {number}
+     */
+    this.s = start
+  }
+
+  /**
+   * @param {number} v
+   */
+  write (v) {
+    writeVarInt(this, v - this.s)
+    this.s = v
+  }
+}
+
+export class RleIntDiffEncoder extends Encoder {
+  /**
+   * @param {number} start
+   */
+  constructor (start) {
+    super()
+    /**
+     * Current state
+     * @type {number}
+     */
+    this.s = start
+    this.count = 0
+  }
+
+  /**
+   * @param {number} v
+   */
+  write (v) {
+    if (this.s === v && this.count > 0) {
+      this.count++
+    } else {
+      if (this.count > 0) {
+        // flush counter, unless this is the first value (count = 0)
+        writeVarUint(this, this.count - 1) // since count is always > 0, we can decrement by one. non-standard encoding ftw
+      }
+      this.count = 1
+      // write first value
+      writeVarInt(this, v - this.s)
+      this.s = v
+    }
+  }
+}
+
+/**
+ * @param {UintOptRleEncoder} encoder
+ */
+const flushUintOptRleEncoder = encoder => {
+  if (encoder.count > 0) {
+    // flush counter, unless this is the first value (count = 0)
+    // case 1: just a single value. set sign to positive
+    // case 2: write several values. set sign to negative to indicate that there is a length coming
+    writeVarInt(encoder.encoder, encoder.count === 1 ? encoder.s : -encoder.s)
+    if (encoder.count > 1) {
+      writeVarUint(encoder.encoder, encoder.count - 2) // since count is always > 1, we can decrement by one. non-standard encoding ftw
+    }
+  }
+}
+
+export class UintOptRleEncoder {
+  constructor () {
+    this.encoder = new Encoder()
+    /**
+     * @type {number}
+     */
+    this.s = 0
+    this.count = 0
+  }
+
+  /**
+   * @param {number} v
+   */
+  write (v) {
+    if (this.s === v) {
+      this.count++
+    } else {
+      flushUintOptRleEncoder(this)
+      this.count = 1
+      this.s = v
+    }
+  }
+
+  toUint8Array () {
+    flushUintOptRleEncoder(this)
+    return toUint8Array(this.encoder)
+  }
+}
+
+export class StringEncoder {
+  constructor () {
+    /**
+     * @type {Array<string>}
+     */
+    this.sarr = []
+    this.s = ''
+    this.lensE = new UintOptRleEncoder()
+  }
+
+  /**
+   * @param {string} string
+   */
+  write (string) {
+    this.s += string
+    if (this.s.length > 19) {
+      this.sarr.push(this.s)
+      this.s = ''
+    }
+    this.lensE.write(string.length)
+  }
+
+  toUint8Array () {
+    const encoder = new Encoder()
+    this.sarr.push(this.s)
+    this.s = ''
+    writeVarString(encoder, this.sarr.join(''))
+    writeUint8Array(encoder, this.lensE.toUint8Array())
+    return toUint8Array(encoder)
+  }
+}