From 60d53084798c89de2b061aebcc675e3179b67f18 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Mon, 6 Oct 2014 12:17:08 -0400 Subject: [PATCH 01/11] refactoring, introducing IO interfaces, starting to modify the pickling --- .../scala/pickling/binary/BinaryInput.scala | 166 ++++ .../scala/pickling/binary/BinaryOutput.scala | 162 ++++ .../scala/pickling/binary/BinaryPickle.scala | 770 ++++++++++++++++++ .../pickling/binary/BinaryPickleFormat.scala | 530 +----------- .../src/main/scala/pickling/binary/Util.scala | 758 +++++++++-------- 5 files changed, 1479 insertions(+), 907 deletions(-) create mode 100644 core/src/main/scala/pickling/binary/BinaryInput.scala create mode 100644 core/src/main/scala/pickling/binary/BinaryOutput.scala create mode 100644 core/src/main/scala/pickling/binary/BinaryPickle.scala diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala new file mode 100644 index 0000000000..0d83ebc25d --- /dev/null +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -0,0 +1,166 @@ +package scala.pickling.binary + +import scala.reflect.ClassTag + +abstract class BinaryInput { + + def getBoolean(): Boolean + + def getByte(): Byte + + def getChar(): Char + + def getShort(): Short + + def getInt(): Int + + def getLong(): Long + + def getFloat(): Float + + def getDouble(): Double + + def getString(): String + + //generic method when the performance is not an issue + @inline private def getArray[T: ClassTag](get: () => T): Array[T] = { + val size = getInt + val array = Array.ofDim[T](size) + for(i <- 0 until size) { + array(i) = get() + } + array + } + + def getBooleanArray(): Unit = getArray(getBoolean) + def getByteArray(): Unit = getArray(getByte) + def getCharArray(): Unit = getArray(getChar) + def getDoubleArray(): Unit = getArray(getDouble) + def getFloatArray(): Unit = getArray(getFloat) + def getIntArray(): Unit = getArray(getInt) + def getLongArray(): Unit = getArray(getLong) + def getShortArray(): Unit = getArray(getShort) +} + +class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { + + def getBoolean() = buffer.get.asInstanceOf[Boolean] + + def getByte() = buffer.get + + def getChar() = buffer.getChar + + def getShort() = buffer.getShort + + def getInt() = buffer.getInt + + def getLong() = buffer.getLong + + def getFloat() = buffer.getFloat + + def getDouble() = buffer.getDouble + + def getString() = { + val size = getInt + val bytes = Array.ofDim[Byte](size) + buffer.get(bytes) + new String(bytes, "UTF-8") + } + +} + +class ByteArrayInput(data: Array[Byte]) extends BinaryInput { + + private var idx = 0 + + def getBoolean() = { + val res = (data(idx) != 0) + idx += 1 + res + } + + def getByte() = { + val res = data(idx) + idx += 1 + res + } + + def getChar() = { + var res = 0 + res |= data(idx ) << 8 + res |= data(idx+1) + idx += 2 + res.asInstanceOf[Char] + } + + def getShort() = { + var res = 0 + res |= data(idx ) << 8 + res |= data(idx+1) + idx += 2 + res.asInstanceOf[Short] + } + + def getInt() = { + var res = (0: Int) + res |= data(idx ) << 24 + res |= data(idx+1) << 26 + res |= data(idx+2) << 8 + res |= data(idx+3) + idx += 4 + res + } + + def getLong() = { + var res = (0: Long) + res |= data(idx ) << 56 + res |= data(idx+1) << 48 + res |= data(idx+2) << 40 + res |= data(idx+3) << 32 + res |= data(idx+4) << 24 + res |= data(idx+5) << 26 + res |= data(idx+6) << 8 + res |= data(idx+7) + idx += 8 + res + } + + def getFloat() = { + val r = getInt() + java.lang.Float.intBitsToFloat(r) + } + + def getDouble() = { + val r = getLong() + java.lang.Double.longBitsToDouble(r) + } + + def getString() = { + val size = getInt + val bytes = data.slice(idx, idx + size) + idx += size + new String(bytes, "UTF-8") + } +} + +class DataStreamInput(stream: java.io.DataInputStream) extends BinaryInput { + + def getBoolean() = stream.readBoolean() + + def getByte() = stream.readByte() + + def getChar() = stream.readChar() + + def getShort() = stream.readShort() + + def getInt() = stream.readInt() + + def getLong() = stream.readLong() + + def getFloat() = stream.readFloat() + + def getDouble() = stream.readDouble() + + def getString() = stream.readUTF() + +} diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala new file mode 100644 index 0000000000..dbb8f743dc --- /dev/null +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -0,0 +1,162 @@ +package scala.pickling.binary + +abstract class BinaryOutput { + + def putBoolean(value: Boolean): Unit + + def putByte(value: Byte): Unit + + def putChar(value: Char): Unit + + def putShort(value: Short): Unit + + def putInt(value: Int): Unit + + def putLong(value: Long): Unit + + def putFloat(value: Float): Unit + + def putDouble(value: Double): Unit + + def putString(value: String): Unit + + //generic method when the performance is not an issue + @inline private def putArray[T](array: Array[T], put: T => Unit): Unit = { + putInt(array.size) + for(elt <- array) put(elt) + } + + def putBooleanArray(value: Array[Boolean]): Unit = putArray(value, putBoolean) + def putByteArray(value: Array[Byte]): Unit = putArray(value, putByte) + def putCharArray(value: Array[Char]): Unit = putArray(value, putChar) + def putDoubleArray(value: Array[Double]): Unit = putArray(value, putDouble) + def putFloatArray(value: Array[Float]): Unit = putArray(value, putFloat) + def putIntArray(value: Array[Int]): Unit = putArray(value, putInt) + def putLongArray(value: Array[Long]): Unit = putArray(value, putLong) + def putShortArray(value: Array[Short]): Unit = putArray(value, putShort) + +} + +//TODO how to deal with capacity issue ? +class ByteBufferOutput(buffer: java.nio.ByteBuffer) extends BinaryOutput { + + def putBoolean(value: Boolean) = buffer.put(value.asInstanceOf[Byte]) + + def putByte(value: Byte) = buffer.put(value) + + def putChar(value: Char) = buffer.putChar(value) + + def putShort(value: Short) = buffer.putShort(value) + + def putInt(value: Int) = buffer.putInt(value) + + def putLong(value: Long) = buffer.putLong(value) + + def putFloat(value: Float) = buffer.putFloat(value) + + def putDouble(value: Double) = buffer.putDouble(value) + + def putString(value: String) { + val bytes = value.getBytes("UTF-8") + putInt(bytes.length) + buffer.put(bytes) + } + +} + +class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { + + val buffer = new java.io.ByteArrayOutputStream(initialCapacity) + + def putBoolean(value: Boolean) { + if (value) buffer.write(1) + else buffer.write(0) + } + + def putByte(value: Byte) { + buffer.write(value) + } + + def putChar(value: Char) { + val fst = value >>> 8 & 0xff + val snd = value & 0xff + buffer.write(fst) + buffer.write(snd) + } + + def putShort(value: Short) { + val fst = value >>> 8 & 0xff + val snd = value & 0xff + buffer.write(fst) + buffer.write(snd) + } + + def putInt(value: Int) { + val fst = value >>> 24 + val snd = value >>> 16 & 0xff + val thrd = value >>> 8 & 0xff + val frth = value & 0xff + buffer.write(fst) + buffer.write(snd) + buffer.write(thrd) + buffer.write(frth) + } + + def putLong(value: Long) { + val elem1 = (value >>> 56 & 0xff).asInstanceOf[Int] + val elem2 = (value >>> 48 & 0xff).asInstanceOf[Int] + val elem3 = (value >>> 40 & 0xff).asInstanceOf[Int] + val elem4 = (value >>> 32 & 0xff).asInstanceOf[Int] + val elem5 = (value >>> 24 & 0xff).asInstanceOf[Int] + val elem6 = (value >>> 16 & 0xff).asInstanceOf[Int] + val elem7 = (value >>> 8 & 0xff).asInstanceOf[Int] + val elem8 = (value & 0xff).asInstanceOf[Int] + buffer.write(elem1) + buffer.write(elem2) + buffer.write(elem3) + buffer.write(elem4) + buffer.write(elem5) + buffer.write(elem6) + buffer.write(elem7) + buffer.write(elem8) + } + + def putFloat(value: Float) { + val intValue = java.lang.Float.floatToRawIntBits(value) + putInt(intValue) + } + + def putDouble(value: Double) { + val longValue = java.lang.Double.doubleToRawLongBits(value) + putLong(longValue) + } + + def putString(value: String) { + val bytes = value.getBytes("UTF-8") + putInt(bytes.length) + buffer.write(bytes) + } + +} + +class DataStreamOutput(stream: java.io.DataOutputStream) extends BinaryOutput { + + def putBoolean(value: Boolean) = stream.writeBoolean(value) + + def putByte(value: Byte) = stream.writeByte(value) + + def putChar(value: Char) = stream.writeChar(value) + + def putShort(value: Short) = stream.writeShort(value) + + def putInt(value: Int) = stream.writeInt(value) + + def putLong(value: Long) = stream.writeLong(value) + + def putFloat(value: Float) = stream.writeFloat(value) + + def putDouble(value: Double) = stream.writeDouble(value) + + def putString(value: String) = stream.writeUTF(value) + +} diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala new file mode 100644 index 0000000000..45e63b1936 --- /dev/null +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -0,0 +1,770 @@ +package scala.pickling.binary + +import scala.pickling._ +import scala.pickling.internal._ +import scala.language.implicitConversions +import scala.reflect.runtime.universe.Mirror + +import java.io.InputStream + +abstract class BinaryPickle extends Pickle { + type PickleFormatType = BinaryPickleFormat + type ValueType = Array[Byte] + + val value: Array[Byte] + + def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader +} + +case class BinaryPickleArray(data: Array[Byte]) extends BinaryPickle { + val value: Array[Byte] = data + + def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = + new BinaryPickleReader(data, mirror, format) + + override def toString = s"""BinaryPickle(${value.mkString("[", ",", "]")})""" +} + +case class BinaryPickleStream(input: InputStream) extends BinaryPickle { + val value: Array[Byte] = Array.ofDim[Byte](0) + + def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = + new BinaryInputStreamReader(input, mirror, format) + + /* Do not override def toString to avoid traversing the input stream. */ +} + +object BinaryPickle { + def apply(a: Array[Byte]): BinaryPickle = + new BinaryPickleArray(a) +} + +class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) extends PBuilder with PickleTools { + import format._ + + @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => + //mkOutput(hints.knownSize) + + if (picklee == null) { + output.putByte( NULL_TAG) + } else if (hints.oid != -1) { + output.putByte( REF_TAG) + output.putInt( hints.oid) + } else { + if (!hints.isElidedType) { + // quickly decide whether we should use picklee.getClass instead + val ts = + if (hints.tag.key.contains("anonfun$")) picklee.getClass.getName + else hints.tag.key + output.putString( ts) + } + + // NOTE: it looks like we don't have to write object ids at all + // traversals employed by pickling and unpickling are exactly the same + // hence when unpickling it's enough to just increment the nextUnpicklee counter + // and everything will work out automatically! + + hints.tag.key match { // PERF: should store typestring once in hints. + case KEY_UNIT => + output.putByte(UNIT_TAG) + case KEY_NULL => + output.putByte(NULL_TAG) + case KEY_BYTE => + output.putByte(picklee.asInstanceOf[Byte]) + case KEY_SHORT => + output.putShort(picklee.asInstanceOf[Short]) + case KEY_CHAR => + output.putChar(picklee.asInstanceOf[Char]) + case KEY_INT => + output.putInt(picklee.asInstanceOf[Int]) + case KEY_LONG => + output.putLong(picklee.asInstanceOf[Long]) + case KEY_BOOLEAN => + output.putBoolean(picklee.asInstanceOf[Boolean]) + case KEY_FLOAT => + output.putFloat(picklee.asInstanceOf[Float]) + case KEY_DOUBLE => + output.putDouble(picklee.asInstanceOf[Double]) + case KEY_STRING => + output.putString(picklee.asInstanceOf[String]) + case KEY_ARRAY_BYTE => + output.putByteArray(picklee.asInstanceOf[Array[Byte]]) + case KEY_ARRAY_CHAR => + output.putCharArray(picklee.asInstanceOf[Array[Char]]) + case KEY_ARRAY_SHORT => + output.putShortArray(picklee.asInstanceOf[Array[Short]]) + case KEY_ARRAY_INT => + output.putIntArray(picklee.asInstanceOf[Array[Int]]) + case KEY_ARRAY_LONG => + output.putLongArray(picklee.asInstanceOf[Array[Long]]) + case KEY_ARRAY_BOOLEAN => + output.putBooleanArray(picklee.asInstanceOf[Array[Boolean]]) + case KEY_ARRAY_FLOAT => + output.putFloatArray(picklee.asInstanceOf[Array[Float]]) + case KEY_ARRAY_DOUBLE => + output.putDoubleArray(picklee.asInstanceOf[Array[Double]]) + case _ => + if (hints.isElidedType) output.putByte(ELIDED_TAG) + } + } + this + } + + @inline def putField(name: String, pickler: PBuilder => Unit): PBuilder = { + // can skip writing name if we pickle/unpickle in the same order + pickler(this) + this + } + + @inline def endEntry(): Unit = { /* do nothing */ } + + @inline def beginCollection(length: Int): PBuilder = { + output.putInt(length) + this + } + + @inline def putElement(pickler: PBuilder => Unit): PBuilder = { + pickler(this) + this + } + + @inline def endCollection(): Unit = { + } + + @inline def result() = { + ??? + //BinaryPickle(output.result()) + } + +} + +class BinaryPickleBuilder(format: BinaryPickleFormat, out: ArrayOutput[Byte]) extends PBuilder with PickleTools { + import format._ + + private var output: ArrayOutput[Byte] = out + + @inline private[this] def mkOutput(knownSize: Int): Unit = + if (output == null) + output = if (knownSize != -1) new scala.pickling.ByteArrayOutput(knownSize) + else new ByteArrayBufferOutput + + @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => + mkOutput(hints.knownSize) + + if (picklee == null) { + Util.encodeByte(output, NULL_TAG) + } else if (hints.oid != -1) { + Util.encodeByte(output, REF_TAG) + Util.encodeInt(output, hints.oid) + } else { + if (!hints.isElidedType) { + // quickly decide whether we should use picklee.getClass instead + val ts = + if (hints.tag.key.contains("anonfun$")) picklee.getClass.getName + else hints.tag.key + Util.encodeString(output, ts) + } + + // NOTE: it looks like we don't have to write object ids at all + // traversals employed by pickling and unpickling are exactly the same + // hence when unpickling it's enough to just increment the nextUnpicklee counter + // and everything will work out automatically! + + hints.tag.key match { // PERF: should store typestring once in hints. + case KEY_UNIT => + Util.encodeByte(output, UNIT_TAG) + case KEY_NULL => + Util.encodeByte(output, NULL_TAG) + case KEY_BYTE => + Util.encodeByte(output, picklee.asInstanceOf[Byte]) + case KEY_SHORT => + Util.encodeShort(output, picklee.asInstanceOf[Short]) + case KEY_CHAR => + Util.encodeChar(output, picklee.asInstanceOf[Char]) + case KEY_INT => + Util.encodeInt(output, picklee.asInstanceOf[Int]) + case KEY_LONG => + Util.encodeLong(output, picklee.asInstanceOf[Long]) + case KEY_BOOLEAN => + Util.encodeBoolean(output, picklee.asInstanceOf[Boolean]) + case KEY_FLOAT => + val intValue = java.lang.Float.floatToRawIntBits(picklee.asInstanceOf[Float]) + Util.encodeInt(output, intValue) + case KEY_DOUBLE => + val longValue = java.lang.Double.doubleToRawLongBits(picklee.asInstanceOf[Double]) + Util.encodeLong(output, longValue) + case KEY_STRING => + Util.encodeString(output, picklee.asInstanceOf[String]) + case KEY_ARRAY_BYTE => + Util.encodeByteArray(output, picklee.asInstanceOf[Array[Byte]]) + case KEY_ARRAY_CHAR => + Util.encodeCharArray(output, picklee.asInstanceOf[Array[Char]]) + case KEY_ARRAY_SHORT => + Util.encodeShortArray(output, picklee.asInstanceOf[Array[Short]]) + case KEY_ARRAY_INT => + Util.encodeIntArray(output, picklee.asInstanceOf[Array[Int]]) + case KEY_ARRAY_LONG => + Util.encodeLongArray(output, picklee.asInstanceOf[Array[Long]]) + case KEY_ARRAY_BOOLEAN => + Util.encodeBooleanArray(output, picklee.asInstanceOf[Array[Boolean]]) + case KEY_ARRAY_FLOAT => + Util.encodeFloatArray(output, picklee.asInstanceOf[Array[Float]]) + case KEY_ARRAY_DOUBLE => + Util.encodeDoubleArray(output, picklee.asInstanceOf[Array[Double]]) + case _ => + if (hints.isElidedType) Util.encodeByte(output, ELIDED_TAG) + } + } + this + } + + @inline def putField(name: String, pickler: PBuilder => Unit): PBuilder = { + // can skip writing name if we pickle/unpickle in the same order + pickler(this) + this + } + + @inline def endEntry(): Unit = { /* do nothing */ } + + @inline def beginCollection(length: Int): PBuilder = { + Util.encodeInt(output, length) + this + } + + @inline def putElement(pickler: PBuilder => Unit): PBuilder = { + pickler(this) + this + } + + @inline def endCollection(): Unit = { + } + + @inline def result() = { + BinaryPickle(output.result()) + } +} + +abstract class AbstractBinaryReader(val mirror: Mirror) { + protected var _lastTagRead: FastTypeTag[_] = null + protected var _lastTypeStringRead: String = null + + protected def lastTagRead: FastTypeTag[_] = + if (_lastTagRead != null) + _lastTagRead + else { + // assume _lastTypeStringRead != null + _lastTagRead = FastTypeTag(mirror, _lastTypeStringRead) + _lastTagRead + } +} + +class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { + import format._ + + def nextByte(): Byte = ??? +//{ +// val b = in.read() +// if (b == -1) throw new EndOfStreamException +// b.asInstanceOf[Byte] +//} + + def decodeStringWithLookahead(la: Byte): String = ??? +//{ +// // read 3 more bytes +// val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) +// val len = { +// val len0 = Util.decodeIntFrom(buf, 0) +// if (len0 > 1000) +// throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") +// else if (len0 < 0) +// throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") +// else +// len0 +// } +// val bytes = Array.ofDim[Byte](len) +// var num = in.read(bytes) +// while (num < len) { +// val readMore = in.read(bytes, num, len - num) +// num += readMore +// } +// new String(bytes, "UTF-8") +//} + + var gla: Option[Byte] = None + + def beginEntryNoTag(): String = + beginEntryNoTagDebug(false) + + def beginEntryNoTagDebug(debugOn: Boolean): String = { + val res: Any = withHints { hints => + // if (debugOn) + // debug(s"hints: $hints") + + if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { + val lookahead = nextByte() + lookahead match { + case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null + case REF_TAG => FastTypeTag.Ref + case _ => gla = Some(lookahead); hints.tag + } + } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { + hints.tag + } else { + val lookahead = nextByte() + // if (debugOn) + // debug(s"checking lookahead: $lookahead") + lookahead match { + case NULL_TAG => + FastTypeTag.Null + case ELIDED_TAG => + hints.tag + case REF_TAG => + FastTypeTag.Ref + case _ => + // do not consume lookahead byte + val res = try { + decodeStringWithLookahead(lookahead) + } catch { + case PicklingException(msg) => + val primInfo = if (hints.tag == null) "" + else s"\nnullable prim: ${nullablePrimitives.contains(hints.tag.key)}\nprim: ${primitives.contains(hints.tag.key)}" + throw PicklingException(s"error decoding type string. debug info: $hints$primInfo\ncause:$msg") + } + // if (debugOn) + // debug(s"decodeStringWithLookahead: $res") + res + } + } + } + if (res.isInstanceOf[String]) { + // if (debugOn) + // debug(s"replacing tag with last type string read: ${res.asInstanceOf[String]}") + _lastTagRead = null + _lastTypeStringRead = res.asInstanceOf[String] + _lastTypeStringRead + } else { + _lastTagRead = res.asInstanceOf[FastTypeTag[_]] + _lastTagRead.key + } + } + + def beginEntry(): FastTypeTag[_] = { + beginEntryNoTag() + lastTagRead + } + + def atPrimitive: Boolean = primitives.contains(lastTagRead.key) + + def readPrimitive(): Any = { + val res = lastTagRead.key match { + case KEY_NULL => null + case KEY_REF => lookupUnpicklee(in.getInt) + case KEY_BYTE => in.getByte + case KEY_SHORT => in.getShort + case KEY_CHAR => in.getChar + case KEY_INT => in.getInt + case KEY_LONG => in.getLong + case KEY_BOOLEAN => in.getBoolean + case KEY_FLOAT => in.getFloat + case KEY_DOUBLE => in.getDouble + + case KEY_STRING => in.getString + + case KEY_ARRAY_BYTE => in.getByteArray + case KEY_ARRAY_SHORT => in.getShortArray + case KEY_ARRAY_CHAR => in.getCharArray + case KEY_ARRAY_INT => in.getIntArray + case KEY_ARRAY_LONG => in.getLongArray + case KEY_ARRAY_BOOLEAN => in.getBooleanArray + case KEY_ARRAY_FLOAT => in.getFloatArray + case KEY_ARRAY_DOUBLE => in.getDoubleArray + } + res + } + + def atObject: Boolean = !atPrimitive + + def readField(name: String): BinaryPickleReader2 = + this + + def endEntry(): Unit = { /* do nothing */ } + + def beginCollection(): PReader = this + + def readLength(): Int = in.getInt + + def readElement(): PReader = this + + def endCollection(): Unit = { /* do nothing */ } + +} + +class BinaryInputStreamReader(in: InputStream, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { + import format._ + + def nextByte(): Byte = { + val b = in.read() + if (b == -1) throw new EndOfStreamException + b.asInstanceOf[Byte] + } + + def decodeStringWithLookahead(la: Byte): String = { + // read 3 more bytes + val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) + val len = { + val len0 = Util.decodeIntFrom(buf, 0) + if (len0 > 1000) + throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") + else if (len0 < 0) + throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") + else + len0 + } + val bytes = Array.ofDim[Byte](len) + var num = in.read(bytes) + while (num < len) { + val readMore = in.read(bytes, num, len - num) + num += readMore + } + new String(bytes, "UTF-8") + } + + var gla: Option[Byte] = None + + def beginEntryNoTag(): String = + beginEntryNoTagDebug(false) + + def beginEntryNoTagDebug(debugOn: Boolean): String = { + val res: Any = withHints { hints => + // if (debugOn) + // debug(s"hints: $hints") + + if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { + val lookahead = nextByte() + lookahead match { + case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null + case REF_TAG => FastTypeTag.Ref + case _ => gla = Some(lookahead); hints.tag + } + } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { + hints.tag + } else { + val lookahead = nextByte() + // if (debugOn) + // debug(s"checking lookahead: $lookahead") + lookahead match { + case NULL_TAG => + FastTypeTag.Null + case ELIDED_TAG => + hints.tag + case REF_TAG => + FastTypeTag.Ref + case _ => + // do not consume lookahead byte + val res = try { + decodeStringWithLookahead(lookahead) + } catch { + case PicklingException(msg) => + val primInfo = if (hints.tag == null) "" + else s"\nnullable prim: ${nullablePrimitives.contains(hints.tag.key)}\nprim: ${primitives.contains(hints.tag.key)}" + throw PicklingException(s"error decoding type string. debug info: $hints$primInfo\ncause:$msg") + } + // if (debugOn) + // debug(s"decodeStringWithLookahead: $res") + res + } + } + } + if (res.isInstanceOf[String]) { + // if (debugOn) + // debug(s"replacing tag with last type string read: ${res.asInstanceOf[String]}") + _lastTagRead = null + _lastTypeStringRead = res.asInstanceOf[String] + _lastTypeStringRead + } else { + _lastTagRead = res.asInstanceOf[FastTypeTag[_]] + _lastTagRead.key + } + } + + def beginEntry(): FastTypeTag[_] = { + beginEntryNoTag() + lastTagRead + } + + def atPrimitive: Boolean = primitives.contains(lastTagRead.key) + + def decodeInt(): Int = { + val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte()) + Util.decodeIntFrom(buf, 0) + } + + def decodeIntWithLookahead(): Int = gla match { + case Some(fstByte) => + gla = None // clear global lookahead + val buf = Array[Byte](fstByte, nextByte(), nextByte(), nextByte()) + Util.decodeIntFrom(buf, 0) + case None => + decodeInt() + } + + def decodeShort(): Short = { + val buf = Array[Byte](nextByte(), nextByte()) + val fst = ((buf(0) << 8) & 0xFFFF).toShort + val snd = (buf(1) & 0x00FF).toShort + (fst | snd).toShort + } + + def decodeChar(): Char = { + val buf = Array[Byte](nextByte(), nextByte()) + val fst = ((buf(0) << 8) & 0xFFFF).toChar + val snd = (buf(1) & 0x00FF).toChar + (fst | snd).toChar + } + + def decodeLong(): Long = { + val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte(), + nextByte(), nextByte(), nextByte(), nextByte()) + val elem1 = ((buf(0).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong + val elem2 = ((buf(1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong + val elem3 = ((buf(2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong + val elem4 = ((buf(3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong + val elem5 = ((buf(4).toLong << 24) & 0x00000000FFFFFFFFL).toLong + val elem6 = ((buf(5).toLong << 16) & 0x0000000000FFFFFFL).toLong + val elem7 = ((buf(6).toLong << 8) & 0x000000000000FFFFL).toLong + val elem8 = (buf(7).toLong & 0x00000000000000FFL).toLong + elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 + } + + def decodeBoolean(): Boolean = { + nextByte() != 0 + } + + def decodeString(): String = { + val len = decodeIntWithLookahead() + val bytes = Array.ofDim[Byte](len) + if (len > 0) { + val num = in.read(bytes) + if (num < len) throw new Exception("Could not read enough bytes from input stream") + } + new String(bytes, "UTF-8") + } + + def decodeByteArray(): Array[Byte] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len) + in.read(arr) + Util.decodeByteArray(arr, 0, len) + } + + def decodeShortArray(): Array[Short] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 2) + in.read(arr) + Util.decodeShortArray(arr, 0, len) + } + + def decodeCharArray(): Array[Char] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 2) + in.read(arr) + Util.decodeCharArray(arr, 0, len) + } + + def decodeIntArray(): Array[Int] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 4) + in.read(arr) + Util.decodeIntArray(arr, 0, len) + } + + // Consider a macro such as Util.decodeArray[Long](in, len) + def decodeLongArray(): Array[Long] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 8) + in.read(arr) + Util.decodeLongArray(arr, 0, len) + } + + def decodeBooleanArray(): Array[Boolean] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len) + in.read(arr) + Util.decodeBooleanArray(arr, 0, len) + } + + def decodeFloatArray(): Array[Float] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 4) + in.read(arr) + Util.decodeFloatArray(arr, 0, len) + } + + def decodeDoubleArray(): Array[Double] = { + val len = decodeIntWithLookahead() + val arr = Array.ofDim[Byte](len * 8) + in.read(arr) + Util.decodeDoubleArray(arr, 0, len) + } + + def readPrimitive(): Any = { + val res = lastTagRead.key match { + case KEY_NULL => null + case KEY_REF => lookupUnpicklee(decodeInt()) + case KEY_BYTE => nextByte() + case KEY_SHORT => decodeShort() + case KEY_CHAR => decodeChar() + case KEY_INT => decodeInt() + case KEY_LONG => decodeLong() + case KEY_BOOLEAN => decodeBoolean() + case KEY_FLOAT => + val r = decodeInt() + java.lang.Float.intBitsToFloat(r) + case KEY_DOUBLE => + val r = decodeLong() + java.lang.Double.longBitsToDouble(r) + + case KEY_STRING => decodeString() + + case KEY_ARRAY_BYTE => decodeByteArray() + case KEY_ARRAY_SHORT => decodeShortArray() + case KEY_ARRAY_CHAR => decodeCharArray() + case KEY_ARRAY_INT => decodeIntArray() + case KEY_ARRAY_LONG => decodeLongArray() + case KEY_ARRAY_BOOLEAN => decodeBooleanArray() + case KEY_ARRAY_FLOAT => decodeFloatArray() + case KEY_ARRAY_DOUBLE => decodeDoubleArray() + } + res + } + + def atObject: Boolean = !atPrimitive + + def readField(name: String): BinaryInputStreamReader = + this + + def endEntry(): Unit = { /* do nothing */ } + + def beginCollection(): PReader = this + + def readLength(): Int = { + decodeInt() + } + + def readElement(): PReader = this + + def endCollection(): Unit = { /* do nothing */ } +} + +class BinaryPickleReader(arr: Array[Byte], mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { + import format._ + + private var pos = 0 + + def beginEntryNoTag(): String = + beginEntryNoTagDebug(false) + + def beginEntryNoTagDebug(debugOn: Boolean): String = { + val res: Any = withHints { hints => + if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { + val lookahead = arr(pos) + lookahead match { + case UNIT_TAG => pos += 1; FastTypeTag.Unit + case NULL_TAG => pos += 1; FastTypeTag.Null + case REF_TAG => pos += 1; FastTypeTag.Ref + case _ => hints.tag + } + } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { + hints.tag + } else { + val lookahead = arr(pos) + lookahead match { + case NULL_TAG => + pos += 1 + FastTypeTag.Null + case ELIDED_TAG => + pos += 1 + hints.tag + case REF_TAG => + pos += 1 + FastTypeTag.Ref + case _ => + val (typeString, newpos) = Util.decodeStringFrom(arr, pos) + pos = newpos + typeString + } + } + } + if (res.isInstanceOf[String]) { + _lastTagRead = null + _lastTypeStringRead = res.asInstanceOf[String] + _lastTypeStringRead + } else { + _lastTagRead = res.asInstanceOf[FastTypeTag[_]] + _lastTagRead.key + } + } + + def beginEntry(): FastTypeTag[_] = { + beginEntryNoTag() + lastTagRead + } + + def atPrimitive: Boolean = primitives.contains(lastTagRead.key) + + def readPrimitive(): Any = { + var newpos = pos + val res = lastTagRead.key match { + case KEY_UNIT => () + case KEY_NULL => null + case KEY_REF => newpos = pos+4 ; lookupUnpicklee(Util.decodeIntFrom(arr, pos)) + case KEY_BYTE => newpos = pos+1 ; arr(pos) + case KEY_SHORT => newpos = pos+2 ; Util.decodeShortFrom(arr, pos) + case KEY_CHAR => newpos = pos+2 ; Util.decodeCharFrom(arr, pos) + case KEY_INT => newpos = pos+4 ; Util.decodeIntFrom(arr, pos) + case KEY_LONG => newpos = pos+8 ; Util.decodeLongFrom(arr, pos) + case KEY_BOOLEAN => newpos = pos+1 ; Util.decodeBooleanFrom(arr, pos) + case KEY_FLOAT => + val r = Util.decodeIntFrom(arr, pos) + newpos = pos+4 + java.lang.Float.intBitsToFloat(r) + case KEY_DOUBLE => + val r = Util.decodeLongFrom(arr, pos) + newpos = pos+8 + java.lang.Double.longBitsToDouble(r) + + case KEY_STRING => val r = Util.decodeStringFrom(arr, pos); newpos = r._2 ; r._1 + + case KEY_ARRAY_BYTE => val r = Util.decodeByteArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_SHORT => val r = Util.decodeShortArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_CHAR => val r = Util.decodeCharArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_INT => val r = Util.decodeIntArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_LONG => val r = Util.decodeLongArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_BOOLEAN => val r = Util.decodeBooleanArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_FLOAT => val r = Util.decodeFloatArrayFrom(arr, pos); newpos = r._2 ; r._1 + case KEY_ARRAY_DOUBLE => val r = Util.decodeDoubleArrayFrom(arr, pos); newpos = r._2 ; r._1 + } + + pos = newpos + res + } + + def atObject: Boolean = !atPrimitive + + def readField(name: String): BinaryPickleReader = + this + + def endEntry(): Unit = { /* do nothing */ } + + def beginCollection(): PReader = this + + def readLength(): Int = { + val length = Util.decodeIntFrom(arr, pos) + pos += 4 + length + } + + def readElement(): PReader = this + + def endCollection(): Unit = { /* do nothing */ } +} diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index 4ac98ccc51..5bc270a6eb 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -4,536 +4,9 @@ import scala.pickling.internal._ import scala.language.implicitConversions import scala.reflect.runtime.universe.Mirror -import java.io.InputStream - package object binary { implicit val pickleFormat = new BinaryPickleFormat implicit def toBinaryPickle(value: Array[Byte]): BinaryPickle = BinaryPickle(value) -} - -package binary { - - abstract class BinaryPickle extends Pickle { - type PickleFormatType = BinaryPickleFormat - type ValueType = Array[Byte] - - val value: Array[Byte] - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader - } - - case class BinaryPickleArray(data: Array[Byte]) extends BinaryPickle { - val value: Array[Byte] = data - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryPickleReader(data, mirror, format) - - override def toString = s"""BinaryPickle(${value.mkString("[", ",", "]")})""" - } - - case class BinaryPickleStream(input: InputStream) extends BinaryPickle { - val value: Array[Byte] = Array.ofDim[Byte](0) - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryInputStreamReader(input, mirror, format) - - /* Do not override def toString to avoid traversing the input stream. */ - } - - object BinaryPickle { - def apply(a: Array[Byte]): BinaryPickle = - new BinaryPickleArray(a) - } - - final class BinaryPickleBuilder(format: BinaryPickleFormat, out: ArrayOutput[Byte]) extends PBuilder with PickleTools { - import format._ - - private var output: ArrayOutput[Byte] = out - - @inline private[this] def mkOutput(knownSize: Int): Unit = - if (output == null) - output = if (knownSize != -1) new ByteArrayOutput(knownSize) - else new ByteArrayBufferOutput - - @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => - mkOutput(hints.knownSize) - - if (picklee == null) { - Util.encodeByte(output, NULL_TAG) - } else if (hints.oid != -1) { - Util.encodeByte(output, REF_TAG) - Util.encodeInt(output, hints.oid) - } else { - if (!hints.isElidedType) { - // quickly decide whether we should use picklee.getClass instead - val ts = - if (hints.tag.key.contains("anonfun$")) picklee.getClass.getName - else hints.tag.key - Util.encodeString(output, ts) - } - - // NOTE: it looks like we don't have to write object ids at all - // traversals employed by pickling and unpickling are exactly the same - // hence when unpickling it's enough to just increment the nextUnpicklee counter - // and everything will work out automatically! - - hints.tag.key match { // PERF: should store typestring once in hints. - case KEY_UNIT => - Util.encodeByte(output, UNIT_TAG) - case KEY_NULL => - Util.encodeByte(output, NULL_TAG) - case KEY_BYTE => - Util.encodeByte(output, picklee.asInstanceOf[Byte]) - case KEY_SHORT => - Util.encodeShort(output, picklee.asInstanceOf[Short]) - case KEY_CHAR => - Util.encodeChar(output, picklee.asInstanceOf[Char]) - case KEY_INT => - Util.encodeInt(output, picklee.asInstanceOf[Int]) - case KEY_LONG => - Util.encodeLong(output, picklee.asInstanceOf[Long]) - case KEY_BOOLEAN => - Util.encodeBoolean(output, picklee.asInstanceOf[Boolean]) - case KEY_FLOAT => - val intValue = java.lang.Float.floatToRawIntBits(picklee.asInstanceOf[Float]) - Util.encodeInt(output, intValue) - case KEY_DOUBLE => - val longValue = java.lang.Double.doubleToRawLongBits(picklee.asInstanceOf[Double]) - Util.encodeLong(output, longValue) - case KEY_STRING => - Util.encodeString(output, picklee.asInstanceOf[String]) - case KEY_ARRAY_BYTE => - Util.encodeByteArray(output, picklee.asInstanceOf[Array[Byte]]) - case KEY_ARRAY_CHAR => - Util.encodeCharArray(output, picklee.asInstanceOf[Array[Char]]) - case KEY_ARRAY_SHORT => - Util.encodeShortArray(output, picklee.asInstanceOf[Array[Short]]) - case KEY_ARRAY_INT => - Util.encodeIntArray(output, picklee.asInstanceOf[Array[Int]]) - case KEY_ARRAY_LONG => - Util.encodeLongArray(output, picklee.asInstanceOf[Array[Long]]) - case KEY_ARRAY_BOOLEAN => - Util.encodeBooleanArray(output, picklee.asInstanceOf[Array[Boolean]]) - case KEY_ARRAY_FLOAT => - Util.encodeFloatArray(output, picklee.asInstanceOf[Array[Float]]) - case KEY_ARRAY_DOUBLE => - Util.encodeDoubleArray(output, picklee.asInstanceOf[Array[Double]]) - case _ => - if (hints.isElidedType) Util.encodeByte(output, ELIDED_TAG) - } - } - this - } - - @inline def putField(name: String, pickler: PBuilder => Unit): PBuilder = { - // can skip writing name if we pickle/unpickle in the same order - pickler(this) - this - } - - @inline def endEntry(): Unit = { /* do nothing */ } - - @inline def beginCollection(length: Int): PBuilder = { - Util.encodeInt(output, length) - this - } - - @inline def putElement(pickler: PBuilder => Unit): PBuilder = { - pickler(this) - this - } - - @inline def endCollection(): Unit = { - } - - @inline def result() = { - BinaryPickle(output.result()) - } - } - - abstract class AbstractBinaryReader(val mirror: Mirror) { - protected var _lastTagRead: FastTypeTag[_] = null - protected var _lastTypeStringRead: String = null - - protected def lastTagRead: FastTypeTag[_] = - if (_lastTagRead != null) - _lastTagRead - else { - // assume _lastTypeStringRead != null - _lastTagRead = FastTypeTag(mirror, _lastTypeStringRead) - _lastTagRead - } - } - - class BinaryInputStreamReader(in: InputStream, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - def nextByte(): Byte = { - val b = in.read() - if (b == -1) throw new EndOfStreamException - b.asInstanceOf[Byte] - } - - def decodeStringWithLookahead(la: Byte): String = { - // read 3 more bytes - val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) - val len = { - val len0 = Util.decodeIntFrom(buf, 0) - if (len0 > 1000) - throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") - else if (len0 < 0) - throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") - else - len0 - } - val bytes = Array.ofDim[Byte](len) - var num = in.read(bytes) - while (num < len) { - val readMore = in.read(bytes, num, len - num) - num += readMore - } - new String(bytes, "UTF-8") - } - - var gla: Option[Byte] = None - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - // if (debugOn) - // debug(s"hints: $hints") - - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = nextByte() - lookahead match { - case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null - case REF_TAG => FastTypeTag.Ref - case _ => gla = Some(lookahead); hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = nextByte() - // if (debugOn) - // debug(s"checking lookahead: $lookahead") - lookahead match { - case NULL_TAG => - FastTypeTag.Null - case ELIDED_TAG => - hints.tag - case REF_TAG => - FastTypeTag.Ref - case _ => - // do not consume lookahead byte - val res = try { - decodeStringWithLookahead(lookahead) - } catch { - case PicklingException(msg) => - val primInfo = if (hints.tag == null) "" - else s"\nnullable prim: ${nullablePrimitives.contains(hints.tag.key)}\nprim: ${primitives.contains(hints.tag.key)}" - throw PicklingException(s"error decoding type string. debug info: $hints$primInfo\ncause:$msg") - } - // if (debugOn) - // debug(s"decodeStringWithLookahead: $res") - res - } - } - } - if (res.isInstanceOf[String]) { - // if (debugOn) - // debug(s"replacing tag with last type string read: ${res.asInstanceOf[String]}") - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def decodeInt(): Int = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - } - - def decodeIntWithLookahead(): Int = gla match { - case Some(fstByte) => - gla = None // clear global lookahead - val buf = Array[Byte](fstByte, nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - case None => - decodeInt() - } - - def decodeShort(): Short = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toShort - val snd = (buf(1) & 0x00FF).toShort - (fst | snd).toShort - } - - def decodeChar(): Char = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toChar - val snd = (buf(1) & 0x00FF).toChar - (fst | snd).toChar - } - - def decodeLong(): Long = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte(), - nextByte(), nextByte(), nextByte(), nextByte()) - val elem1 = ((buf(0).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong - val elem2 = ((buf(1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong - val elem3 = ((buf(2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong - val elem4 = ((buf(3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong - val elem5 = ((buf(4).toLong << 24) & 0x00000000FFFFFFFFL).toLong - val elem6 = ((buf(5).toLong << 16) & 0x0000000000FFFFFFL).toLong - val elem7 = ((buf(6).toLong << 8) & 0x000000000000FFFFL).toLong - val elem8 = (buf(7).toLong & 0x00000000000000FFL).toLong - elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 - } - - def decodeBoolean(): Boolean = { - nextByte() != 0 - } - - def decodeString(): String = { - val len = decodeIntWithLookahead() - val bytes = Array.ofDim[Byte](len) - if (len > 0) { - val num = in.read(bytes) - if (num < len) throw new Exception("Could not read enough bytes from input stream") - } - new String(bytes, "UTF-8") - } - - def decodeByteArray(): Array[Byte] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeByteArray(arr, 0, len) - } - - def decodeShortArray(): Array[Short] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeShortArray(arr, 0, len) - } - - def decodeCharArray(): Array[Char] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeCharArray(arr, 0, len) - } - - def decodeIntArray(): Array[Int] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeIntArray(arr, 0, len) - } - - // Consider a macro such as Util.decodeArray[Long](in, len) - def decodeLongArray(): Array[Long] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeLongArray(arr, 0, len) - } - - def decodeBooleanArray(): Array[Boolean] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeBooleanArray(arr, 0, len) - } - - def decodeFloatArray(): Array[Float] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeFloatArray(arr, 0, len) - } - - def decodeDoubleArray(): Array[Double] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeDoubleArray(arr, 0, len) - } - - def readPrimitive(): Any = { - val res = lastTagRead.key match { - case KEY_NULL => null - case KEY_REF => lookupUnpicklee(decodeInt()) - case KEY_BYTE => nextByte() - case KEY_SHORT => decodeShort() - case KEY_CHAR => decodeChar() - case KEY_INT => decodeInt() - case KEY_LONG => decodeLong() - case KEY_BOOLEAN => decodeBoolean() - case KEY_FLOAT => - val r = decodeInt() - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = decodeLong() - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => decodeString() - - case KEY_ARRAY_BYTE => decodeByteArray() - case KEY_ARRAY_SHORT => decodeShortArray() - case KEY_ARRAY_CHAR => decodeCharArray() - case KEY_ARRAY_INT => decodeIntArray() - case KEY_ARRAY_LONG => decodeLongArray() - case KEY_ARRAY_BOOLEAN => decodeBooleanArray() - case KEY_ARRAY_FLOAT => decodeFloatArray() - case KEY_ARRAY_DOUBLE => decodeDoubleArray() - } - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryInputStreamReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - decodeInt() - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } - } - - class BinaryPickleReader(arr: Array[Byte], mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - private var pos = 0 - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = arr(pos) - lookahead match { - case UNIT_TAG => pos += 1; FastTypeTag.Unit - case NULL_TAG => pos += 1; FastTypeTag.Null - case REF_TAG => pos += 1; FastTypeTag.Ref - case _ => hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = arr(pos) - lookahead match { - case NULL_TAG => - pos += 1 - FastTypeTag.Null - case ELIDED_TAG => - pos += 1 - hints.tag - case REF_TAG => - pos += 1 - FastTypeTag.Ref - case _ => - val (typeString, newpos) = Util.decodeStringFrom(arr, pos) - pos = newpos - typeString - } - } - } - if (res.isInstanceOf[String]) { - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def readPrimitive(): Any = { - var newpos = pos - val res = lastTagRead.key match { - case KEY_UNIT => () - case KEY_NULL => null - case KEY_REF => newpos = pos+4 ; lookupUnpicklee(Util.decodeIntFrom(arr, pos)) - case KEY_BYTE => newpos = pos+1 ; arr(pos) - case KEY_SHORT => newpos = pos+2 ; Util.decodeShortFrom(arr, pos) - case KEY_CHAR => newpos = pos+2 ; Util.decodeCharFrom(arr, pos) - case KEY_INT => newpos = pos+4 ; Util.decodeIntFrom(arr, pos) - case KEY_LONG => newpos = pos+8 ; Util.decodeLongFrom(arr, pos) - case KEY_BOOLEAN => newpos = pos+1 ; Util.decodeBooleanFrom(arr, pos) - case KEY_FLOAT => - val r = Util.decodeIntFrom(arr, pos) - newpos = pos+4 - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = Util.decodeLongFrom(arr, pos) - newpos = pos+8 - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => val r = Util.decodeStringFrom(arr, pos); newpos = r._2 ; r._1 - - case KEY_ARRAY_BYTE => val r = Util.decodeByteArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_SHORT => val r = Util.decodeShortArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_CHAR => val r = Util.decodeCharArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_INT => val r = Util.decodeIntArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_LONG => val r = Util.decodeLongArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_BOOLEAN => val r = Util.decodeBooleanArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_FLOAT => val r = Util.decodeFloatArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_DOUBLE => val r = Util.decodeDoubleArrayFrom(arr, pos); newpos = r._2 ; r._1 - } - - pos = newpos - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryPickleReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - val length = Util.decodeIntFrom(arr, pos) - pos += 4 - length - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } - } trait Constants { val NULL_TAG : Byte = -2 @@ -576,4 +49,7 @@ package binary { def createBuilder(out: ArrayOutput[Byte]): PBuilder = new BinaryPickleBuilder(this, out) def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) } + } + + diff --git a/core/src/main/scala/pickling/binary/Util.scala b/core/src/main/scala/pickling/binary/Util.scala index 24000e8e1f..10c00e70f6 100644 --- a/core/src/main/scala/pickling/binary/Util.scala +++ b/core/src/main/scala/pickling/binary/Util.scala @@ -1,383 +1,381 @@ -package scala.pickling - -package binary { - import collection.mutable.Buffer - import UnsafeMemory._ - - object Util { - - val SizeOfByte = 1 - val SizeOfShort = 2 - val SizeOfInt = 4 - val SizeOfLong = 8 - val SizeOfFloat = 4 - val SizeOfDouble = 8 - val SizeOfChar = 2 - val SizeOfBoolean = 1 - - // Decoding functions - - def decodeShortFrom(arr: Array[Byte], i: Int): Short = { - val fst = ((arr(i) << 8) & 0xFFFF).toShort - val snd = (arr(i+1) & 0x00FF).toShort - (fst | snd).toShort - } - - def decodeIntFrom(arr: Array[Byte], i: Int): Int = { - val fst = (arr(i) << 24).toInt - val snd = ((arr(i+1) << 16) & 0x00FFFFFF).toInt - val thrd = ((arr(i+2) << 8) & 0x0000FFFF).toInt - val frth = (arr(i+3) & 0x000000FF).toInt - fst | snd | thrd | frth - } - - def decodeLongFrom(arr: Array[Byte], i: Int): Long = { - val elem1 = ((arr(i).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong - val elem2 = ((arr(i+1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong - val elem3 = ((arr(i+2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong - val elem4 = ((arr(i+3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong - val elem5 = ((arr(i+4).toLong << 24) & 0x00000000FFFFFFFFL).toLong - val elem6 = ((arr(i+5).toLong << 16) & 0x0000000000FFFFFFL).toLong - val elem7 = ((arr(i+6).toLong << 8) & 0x000000000000FFFFL).toLong - val elem8 = (arr(i+7).toLong & 0x00000000000000FFL).toLong - elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 - } - - def decodeCharFrom(arr: Array[Byte], i: Int): Char = { - val fst = ((arr(i) << 8) & 0xFFFF).toChar - val snd = (arr(i+1) & 0x00FF).toChar - (fst | snd).toChar - } - - def decodeStringFrom(arr: Array[Byte], i: Int): (String, Int) = { - val len = decodeIntFrom(arr, i) - val bytes = Array.ofDim[Byte](len) - Array.copy(arr, i + SizeOfInt, bytes, 0, len) - (new String(bytes, "UTF-8"), i + SizeOfInt + len) - } - - def decodeBooleanFrom(arr: Array[Byte], i: Int): Boolean = { - arr(i) != 0 - } - - def decodeByteArray(arr: Array[Byte], offset: Int, len: Int): Array[Byte] = { - val newArr = Array.ofDim[Byte](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfByte) - newArr - } - - def decodeByteArrayFrom(arr: Array[Byte], i: Int): (Array[Byte], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeByteArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfByte) - } - - def decodeShortArray(arr: Array[Byte], offset: Int, len: Int): Array[Short] = { - val newArr = Array.ofDim[Short](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.shortArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfShort) - newArr - } - - def decodeShortArrayFrom(arr: Array[Byte], i: Int): (Array[Short], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeShortArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfShort) - } - - def decodeCharArray(arr: Array[Byte], offset: Int, len: Int): Array[Char] = { - val newArr = Array.ofDim[Char](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.charArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfChar) - newArr - } - - def decodeCharArrayFrom(arr: Array[Byte], i: Int): (Array[Char], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeCharArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfChar) - } - - def decodeIntArray(arr: Array[Byte], offset: Int, len: Int): Array[Int] = { - val newArr = Array.ofDim[Int](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.intArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfInt) - newArr - } - - def decodeIntArrayFrom(arr: Array[Byte], i: Int): (Array[Int], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeIntArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfInt) - } - - def decodeLongArray(arr: Array[Byte], offset: Int, len: Int): Array[Long] = { - val newArr = Array.ofDim[Long](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.longArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfLong) - newArr - } - - def decodeLongArrayFrom(arr: Array[Byte], i: Int): (Array[Long], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeLongArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfLong) - } - - def decodeFloatArray(arr: Array[Byte], offset: Int, len: Int): Array[Float] = { - val newArr = Array.ofDim[Float](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.floatArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfFloat) - newArr - } - - def decodeFloatArrayFrom(arr: Array[Byte], i: Int): (Array[Float], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeFloatArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfFloat) - } - - def decodeDoubleArray(arr: Array[Byte], offset: Int, len: Int): Array[Double] = { - val newArr = Array.ofDim[Double](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.doubleArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfDouble) - newArr - } - - def decodeDoubleArrayFrom(arr: Array[Byte], i: Int): (Array[Double], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeDoubleArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfDouble) - } - - def decodeBooleanArray(arr: Array[Byte], offset: Int, len: Int): Array[Boolean] = { - val newArr = Array.ofDim[Boolean](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.booleanArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfBoolean) - newArr - } - - def decodeBooleanArrayFrom(arr: Array[Byte], i: Int): (Array[Boolean], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeBooleanArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfBoolean) - } - - // Encoding functions - - def encodeByte(arr: ArrayOutput[Byte], value: Byte): Unit = { - arr += value - } - - def encodeShort(arr: ArrayOutput[Byte], value: Short): Unit = { - val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] - val snd = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - } - - def encodeInt(arr: ArrayOutput[Byte], value: Int): Unit = { - val fst = (value >>> 24).asInstanceOf[Byte] - val snd = (value >>> 16 & 0xff).asInstanceOf[Byte] - val thrd = (value >>> 8 & 0xff).asInstanceOf[Byte] - val frth = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - arr += thrd - arr += frth - } - - - def encodeLong(arr: ArrayOutput[Byte], value: Long): Unit = { - val elem1 = (value >>> 56 & 0xff).asInstanceOf[Byte] - val elem2 = (value >>> 48 & 0xff).asInstanceOf[Byte] - val elem3 = (value >>> 40 & 0xff).asInstanceOf[Byte] - val elem4 = (value >>> 32 & 0xff).asInstanceOf[Byte] - val elem5 = (value >>> 24 & 0xff).asInstanceOf[Byte] - val elem6 = (value >>> 16 & 0xff).asInstanceOf[Byte] - val elem7 = (value >>> 8 & 0xff).asInstanceOf[Byte] - val elem8 = (value & 0xff).asInstanceOf[Byte] - arr += elem1 - arr += elem2 - arr += elem3 - arr += elem4 - arr += elem5 - arr += elem6 - arr += elem7 - arr += elem8 - } - - def encodeChar(arr: ArrayOutput[Byte], value: Char): Unit = { - val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] - val snd = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - } - - def encodeString(arr: ArrayOutput[Byte], value: String): Unit = { - val bytes = value.getBytes("UTF-8") - encodeInt(arr, bytes.length) - val (buf, pos) = arr.target(bytes.length * SizeOfByte) - encodeByteArrayTo(buf, pos, bytes) - arr.flush(buf) - } - - def encodeBoolean(arr: ArrayOutput[Byte], value: Boolean): Unit = { - arr += (if (value) 1 else 0) - } - - def encodeByteArray(arr: ArrayOutput[Byte], value: Array[Byte]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfByte) - encodeByteArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeShortArray(arr: ArrayOutput[Byte], value: Array[Short]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfShort) - encodeShortArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeIntArray(arr: ArrayOutput[Byte], value: Array[Int]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfInt) - encodeIntArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeLongArray(arr: ArrayOutput[Byte], value: Array[Long]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfLong) - encodeLongArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeFloatArray(arr: ArrayOutput[Byte], value: Array[Float]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfFloat) - encodeFloatArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeDoubleArray(arr: ArrayOutput[Byte], value: Array[Double]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfDouble) - encodeDoubleArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeCharArray(arr: ArrayOutput[Byte], value: Array[Char]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfChar) - encodeCharArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeBooleanArray(arr: ArrayOutput[Byte], value: Array[Boolean]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfBoolean) - encodeBooleanArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeByteArrayTo(arr: Array[Byte], i: Int, value: Array[Byte]): Int = { - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfByte) - i + value.length * SizeOfByte - } - - def encodeShortArrayTo(arr: Array[Byte], i: Int, value: Array[Short]): Int = { - val srcOffset = UnsafeMemory.shortArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfShort) - i + value.length * SizeOfShort - } - - def encodeIntArrayTo(arr: Array[Byte], i: Int, value: Array[Int]): Int = { - val srcOffset = UnsafeMemory.intArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfInt) - i + value.length * SizeOfInt - } - - def encodeLongArrayTo(arr: Array[Byte], i: Int, value: Array[Long]): Int = { - val srcOffset = UnsafeMemory.longArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfLong) - i + value.length * SizeOfLong - } - - def encodeFloatArrayTo(arr: Array[Byte], i: Int, value: Array[Float]): Int = { - val srcOffset = UnsafeMemory.floatArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfFloat) - i + value.length * SizeOfFloat - } - - def encodeDoubleArrayTo(arr: Array[Byte], i: Int, value: Array[Double]): Int = { - val srcOffset = UnsafeMemory.doubleArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfDouble) - i + value.length * SizeOfDouble - } - - def encodeCharArrayTo(arr: Array[Byte], i: Int, value: Array[Char]): Int = { - val srcOffset = UnsafeMemory.charArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfChar) - i + value.length * SizeOfChar - } - - def encodeBooleanArrayTo(arr: Array[Byte], i: Int, value: Array[Boolean]): Int = { - val srcOffset = UnsafeMemory.booleanArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfBoolean) - i + value.length * SizeOfBoolean - } - } - - object UnsafeMemory { - import sun.misc.Unsafe - - private[pickling] val unsafe: Unsafe = - scala.concurrent.util.Unsafe.instance - - private[pickling] val byteArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Byte]]) - private[pickling] val shortArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Short]]) - private[pickling] val intArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Int]]) - private[pickling] val longArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Long]]) - private[pickling] val floatArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Float]]) - private[pickling] val doubleArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Double]]) - private[pickling] val charArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Char]]) - private[pickling] val booleanArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Boolean]]) - - def putInt(arr: Array[Byte], i: Int, value: Int): Unit = { - unsafe.putInt(arr, byteArrayOffset + i, value) - } - - def getInt(arr: Array[Byte], i: Int): Int = { - unsafe.getInt(arr, byteArrayOffset + i) - } +package scala.pickling.binary + +import scala.pickling._ +import collection.mutable.Buffer +import UnsafeMemory._ + +object Util { + + val SizeOfByte = 1 + val SizeOfShort = 2 + val SizeOfInt = 4 + val SizeOfLong = 8 + val SizeOfFloat = 4 + val SizeOfDouble = 8 + val SizeOfChar = 2 + val SizeOfBoolean = 1 + + // Decoding functions + + def decodeShortFrom(arr: Array[Byte], i: Int): Short = { + val fst = ((arr(i) << 8) & 0xFFFF).toShort + val snd = (arr(i+1) & 0x00FF).toShort + (fst | snd).toShort + } + + def decodeIntFrom(arr: Array[Byte], i: Int): Int = { + val fst = (arr(i) << 24).toInt + val snd = ((arr(i+1) << 16) & 0x00FFFFFF).toInt + val thrd = ((arr(i+2) << 8) & 0x0000FFFF).toInt + val frth = (arr(i+3) & 0x000000FF).toInt + fst | snd | thrd | frth + } + + def decodeLongFrom(arr: Array[Byte], i: Int): Long = { + val elem1 = ((arr(i).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong + val elem2 = ((arr(i+1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong + val elem3 = ((arr(i+2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong + val elem4 = ((arr(i+3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong + val elem5 = ((arr(i+4).toLong << 24) & 0x00000000FFFFFFFFL).toLong + val elem6 = ((arr(i+5).toLong << 16) & 0x0000000000FFFFFFL).toLong + val elem7 = ((arr(i+6).toLong << 8) & 0x000000000000FFFFL).toLong + val elem8 = (arr(i+7).toLong & 0x00000000000000FFL).toLong + elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 + } + + def decodeCharFrom(arr: Array[Byte], i: Int): Char = { + val fst = ((arr(i) << 8) & 0xFFFF).toChar + val snd = (arr(i+1) & 0x00FF).toChar + (fst | snd).toChar + } + + def decodeStringFrom(arr: Array[Byte], i: Int): (String, Int) = { + val len = decodeIntFrom(arr, i) + val bytes = Array.ofDim[Byte](len) + Array.copy(arr, i + SizeOfInt, bytes, 0, len) + (new String(bytes, "UTF-8"), i + SizeOfInt + len) + } + + def decodeBooleanFrom(arr: Array[Byte], i: Int): Boolean = { + arr(i) != 0 + } + + def decodeByteArray(arr: Array[Byte], offset: Int, len: Int): Array[Byte] = { + val newArr = Array.ofDim[Byte](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfByte) + newArr + } + + def decodeByteArrayFrom(arr: Array[Byte], i: Int): (Array[Byte], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeByteArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfByte) + } + + def decodeShortArray(arr: Array[Byte], offset: Int, len: Int): Array[Short] = { + val newArr = Array.ofDim[Short](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.shortArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfShort) + newArr + } + + def decodeShortArrayFrom(arr: Array[Byte], i: Int): (Array[Short], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeShortArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfShort) + } + + def decodeCharArray(arr: Array[Byte], offset: Int, len: Int): Array[Char] = { + val newArr = Array.ofDim[Char](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.charArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfChar) + newArr + } + + def decodeCharArrayFrom(arr: Array[Byte], i: Int): (Array[Char], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeCharArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfChar) + } + + def decodeIntArray(arr: Array[Byte], offset: Int, len: Int): Array[Int] = { + val newArr = Array.ofDim[Int](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.intArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfInt) + newArr + } + + def decodeIntArrayFrom(arr: Array[Byte], i: Int): (Array[Int], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeIntArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfInt) + } + + def decodeLongArray(arr: Array[Byte], offset: Int, len: Int): Array[Long] = { + val newArr = Array.ofDim[Long](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.longArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfLong) + newArr + } + + def decodeLongArrayFrom(arr: Array[Byte], i: Int): (Array[Long], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeLongArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfLong) + } + + def decodeFloatArray(arr: Array[Byte], offset: Int, len: Int): Array[Float] = { + val newArr = Array.ofDim[Float](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.floatArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfFloat) + newArr + } + + def decodeFloatArrayFrom(arr: Array[Byte], i: Int): (Array[Float], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeFloatArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfFloat) + } + + def decodeDoubleArray(arr: Array[Byte], offset: Int, len: Int): Array[Double] = { + val newArr = Array.ofDim[Double](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.doubleArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfDouble) + newArr + } + + def decodeDoubleArrayFrom(arr: Array[Byte], i: Int): (Array[Double], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeDoubleArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfDouble) + } + + def decodeBooleanArray(arr: Array[Byte], offset: Int, len: Int): Array[Boolean] = { + val newArr = Array.ofDim[Boolean](len) + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.booleanArrayOffset + UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfBoolean) + newArr + } + + def decodeBooleanArrayFrom(arr: Array[Byte], i: Int): (Array[Boolean], Int) = { + val len = decodeIntFrom(arr, i) + val nextPos = i + SizeOfInt + val ia = decodeBooleanArray(arr, nextPos, len) + (ia, nextPos + len * SizeOfBoolean) + } + + // Encoding functions + + def encodeByte(arr: ArrayOutput[Byte], value: Byte): Unit = { + arr += value + } + + def encodeShort(arr: ArrayOutput[Byte], value: Short): Unit = { + val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] + val snd = (value & 0xff).asInstanceOf[Byte] + arr += fst + arr += snd + } + + def encodeInt(arr: ArrayOutput[Byte], value: Int): Unit = { + val fst = (value >>> 24).asInstanceOf[Byte] + val snd = (value >>> 16 & 0xff).asInstanceOf[Byte] + val thrd = (value >>> 8 & 0xff).asInstanceOf[Byte] + val frth = (value & 0xff).asInstanceOf[Byte] + arr += fst + arr += snd + arr += thrd + arr += frth + } + + + def encodeLong(arr: ArrayOutput[Byte], value: Long): Unit = { + val elem1 = (value >>> 56 & 0xff).asInstanceOf[Byte] + val elem2 = (value >>> 48 & 0xff).asInstanceOf[Byte] + val elem3 = (value >>> 40 & 0xff).asInstanceOf[Byte] + val elem4 = (value >>> 32 & 0xff).asInstanceOf[Byte] + val elem5 = (value >>> 24 & 0xff).asInstanceOf[Byte] + val elem6 = (value >>> 16 & 0xff).asInstanceOf[Byte] + val elem7 = (value >>> 8 & 0xff).asInstanceOf[Byte] + val elem8 = (value & 0xff).asInstanceOf[Byte] + arr += elem1 + arr += elem2 + arr += elem3 + arr += elem4 + arr += elem5 + arr += elem6 + arr += elem7 + arr += elem8 + } + + def encodeChar(arr: ArrayOutput[Byte], value: Char): Unit = { + val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] + val snd = (value & 0xff).asInstanceOf[Byte] + arr += fst + arr += snd + } + + def encodeString(arr: ArrayOutput[Byte], value: String): Unit = { + val bytes = value.getBytes("UTF-8") + encodeInt(arr, bytes.length) + val (buf, pos) = arr.target(bytes.length * SizeOfByte) + encodeByteArrayTo(buf, pos, bytes) + arr.flush(buf) + } + + def encodeBoolean(arr: ArrayOutput[Byte], value: Boolean): Unit = { + arr += (if (value) 1 else 0) + } + + def encodeByteArray(arr: ArrayOutput[Byte], value: Array[Byte]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfByte) + encodeByteArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeShortArray(arr: ArrayOutput[Byte], value: Array[Short]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfShort) + encodeShortArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeIntArray(arr: ArrayOutput[Byte], value: Array[Int]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfInt) + encodeIntArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeLongArray(arr: ArrayOutput[Byte], value: Array[Long]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfLong) + encodeLongArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeFloatArray(arr: ArrayOutput[Byte], value: Array[Float]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfFloat) + encodeFloatArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeDoubleArray(arr: ArrayOutput[Byte], value: Array[Double]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfDouble) + encodeDoubleArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeCharArray(arr: ArrayOutput[Byte], value: Array[Char]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfChar) + encodeCharArrayTo(buf, pos, value) + arr.flush(buf) } + def encodeBooleanArray(arr: ArrayOutput[Byte], value: Array[Boolean]): Unit = { + encodeInt(arr, value.length) + val (buf, pos) = arr.target(value.length * SizeOfBoolean) + encodeBooleanArrayTo(buf, pos, value) + arr.flush(buf) + } + + def encodeByteArrayTo(arr: Array[Byte], i: Int, value: Array[Byte]): Int = { + val srcOffset = UnsafeMemory.byteArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfByte) + i + value.length * SizeOfByte + } + + def encodeShortArrayTo(arr: Array[Byte], i: Int, value: Array[Short]): Int = { + val srcOffset = UnsafeMemory.shortArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfShort) + i + value.length * SizeOfShort + } + + def encodeIntArrayTo(arr: Array[Byte], i: Int, value: Array[Int]): Int = { + val srcOffset = UnsafeMemory.intArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfInt) + i + value.length * SizeOfInt + } + + def encodeLongArrayTo(arr: Array[Byte], i: Int, value: Array[Long]): Int = { + val srcOffset = UnsafeMemory.longArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfLong) + i + value.length * SizeOfLong + } + + def encodeFloatArrayTo(arr: Array[Byte], i: Int, value: Array[Float]): Int = { + val srcOffset = UnsafeMemory.floatArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfFloat) + i + value.length * SizeOfFloat + } + + def encodeDoubleArrayTo(arr: Array[Byte], i: Int, value: Array[Double]): Int = { + val srcOffset = UnsafeMemory.doubleArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfDouble) + i + value.length * SizeOfDouble + } + + def encodeCharArrayTo(arr: Array[Byte], i: Int, value: Array[Char]): Int = { + val srcOffset = UnsafeMemory.charArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfChar) + i + value.length * SizeOfChar + } + + def encodeBooleanArrayTo(arr: Array[Byte], i: Int, value: Array[Boolean]): Int = { + val srcOffset = UnsafeMemory.booleanArrayOffset + val destOffset = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfBoolean) + i + value.length * SizeOfBoolean + } +} + +object UnsafeMemory { + import sun.misc.Unsafe + + private[pickling] val unsafe: Unsafe = + scala.concurrent.util.Unsafe.instance + + private[pickling] val byteArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Byte]]) + private[pickling] val shortArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Short]]) + private[pickling] val intArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Int]]) + private[pickling] val longArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Long]]) + private[pickling] val floatArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Float]]) + private[pickling] val doubleArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Double]]) + private[pickling] val charArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Char]]) + private[pickling] val booleanArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Boolean]]) + + def putInt(arr: Array[Byte], i: Int, value: Int): Unit = { + unsafe.putInt(arr, byteArrayOffset + i, value) + } + + def getInt(arr: Array[Byte], i: Int): Int = { + unsafe.getInt(arr, byteArrayOffset + i) + } } From eb49bdeb1c28545052888708d26e9b40bd58a1f3 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Mon, 6 Oct 2014 22:35:05 -0400 Subject: [PATCH 02/11] keeping compatibility with lookahead --- .../scala/pickling/binary/BinaryInput.scala | 81 ++++++--- .../scala/pickling/binary/BinaryOutput.scala | 156 +++++++++++------- .../scala/pickling/binary/BinaryPickle.scala | 48 ++---- 3 files changed, 165 insertions(+), 120 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala index 0d83ebc25d..52d7f47327 100644 --- a/core/src/main/scala/pickling/binary/BinaryInput.scala +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -32,14 +32,44 @@ abstract class BinaryInput { array } - def getBooleanArray(): Unit = getArray(getBoolean) - def getByteArray(): Unit = getArray(getByte) - def getCharArray(): Unit = getArray(getChar) - def getDoubleArray(): Unit = getArray(getDouble) - def getFloatArray(): Unit = getArray(getFloat) - def getIntArray(): Unit = getArray(getInt) - def getLongArray(): Unit = getArray(getLong) - def getShortArray(): Unit = getArray(getShort) + def getBooleanArray(): Array[Boolean] = getArray(getBoolean) + def getByteArray(): Array[Byte] = getArray(getByte) + def getCharArray(): Array[Char] = getArray(getChar) + def getDoubleArray(): Array[Double] = getArray(getDouble) + def getFloatArray(): Array[Float] = getArray(getFloat) + def getIntArray(): Array[Int] = getArray(getInt) + def getLongArray(): Array[Long] = getArray(getLong) + def getShortArray(): Array[Short] = getArray(getShort) + + //TODO lookahead + protected var lookahead: Option[Byte] = None + + def setLookahead(b: Byte) { + lookahead = Some(b) + } + + def getIntWithLookahead() = { + lookahead match { + case Some(b) => + var i = b << 24 + i |= getByte << 16 + i |= getByte << 8 + i |= getByte + lookahead = None + i + case None => + getInt + } + } + + def getBytes(array: Array[Byte]) + + def getStringWithLookahead() { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) + getBytes(array) + } + } class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { @@ -67,6 +97,10 @@ class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { new String(bytes, "UTF-8") } + def getBytes(array: Array[Byte]) { + buffer.get(array) + } + } class ByteArrayInput(data: Array[Byte]) extends BinaryInput { @@ -141,26 +175,35 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { idx += size new String(bytes, "UTF-8") } + + def getBytes(array: Array[Byte]) { + val size = array.size + data.copyToArray(array, idx, size) + idx += size + } + + //TODO override array for faster copy + } -class DataStreamInput(stream: java.io.DataInputStream) extends BinaryInput { +// class DataStreamInput(stream: java.io.DataInputStream) extends BinaryInput { - def getBoolean() = stream.readBoolean() +// def getBoolean() = stream.readBoolean() - def getByte() = stream.readByte() +// def getByte() = stream.readByte() - def getChar() = stream.readChar() +// def getChar() = stream.readChar() - def getShort() = stream.readShort() +// def getShort() = stream.readShort() - def getInt() = stream.readInt() +// def getInt() = stream.readInt() - def getLong() = stream.readLong() +// def getLong() = stream.readLong() - def getFloat() = stream.readFloat() +// def getFloat() = stream.readFloat() - def getDouble() = stream.readDouble() +// def getDouble() = stream.readDouble() - def getString() = stream.readUTF() +// def getString() = stream.readUTF() -} +// } diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index dbb8f743dc..fd95e6d44a 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -2,6 +2,8 @@ package scala.pickling.binary abstract class BinaryOutput { + def result: Option[Array[Byte]] + def putBoolean(value: Boolean): Unit def putByte(value: Byte): Unit @@ -18,7 +20,11 @@ abstract class BinaryOutput { def putDouble(value: Double): Unit - def putString(value: String): Unit + def putString(value: String) { + val bytes = value.getBytes("UTF-8") + putByteArray(bytes) + } + //generic method when the performance is not an issue @inline private def putArray[T](array: Array[T], put: T => Unit): Unit = { @@ -37,29 +43,66 @@ abstract class BinaryOutput { } -//TODO how to deal with capacity issue ? -class ByteBufferOutput(buffer: java.nio.ByteBuffer) extends BinaryOutput { +class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { - def putBoolean(value: Boolean) = buffer.put(value.asInstanceOf[Byte]) + import java.nio.ByteOrder + import java.nio.ByteBuffer + + def result = None + + private var buffer = _buffer + assert(buffer.order == ByteOrder.BIG_ENDIAN) + + private def grow { + val newSize = 2*buffer.capacity + assert(newSize > 0) + val newBuffer = + if (buffer.isDirect) ByteBuffer.allocateDirect(newSize) + else ByteBuffer.allocate(newSize) + //copy the content + val pos = buffer.position + buffer.limit(pos) + buffer.position(0) + newBuffer.put(buffer) + buffer = newBuffer + } - def putByte(value: Byte) = buffer.put(value) + @inline private def withReallocate[A](op: A => ByteBuffer, value: A) { + while(true) { + buffer.mark + try { + op(value) + return + } catch { + case _: java.nio.BufferOverflowException => + buffer.reset + grow + } + } + } - def putChar(value: Char) = buffer.putChar(value) + def putBoolean(value: Boolean) = withReallocate[Byte](buffer.put, value.asInstanceOf[Byte]) - def putShort(value: Short) = buffer.putShort(value) + def putByte(value: Byte) = withReallocate[Byte](buffer.put, value) - def putInt(value: Int) = buffer.putInt(value) + def putChar(value: Char) = withReallocate(buffer.putChar, value) - def putLong(value: Long) = buffer.putLong(value) + def putShort(value: Short) = withReallocate(buffer.putShort, value) - def putFloat(value: Float) = buffer.putFloat(value) + def putInt(value: Int) = withReallocate(buffer.putInt, value) - def putDouble(value: Double) = buffer.putDouble(value) + def putLong(value: Long) = withReallocate(buffer.putLong, value) - def putString(value: String) { - val bytes = value.getBytes("UTF-8") - putInt(bytes.length) - buffer.put(bytes) + def putFloat(value: Float) = withReallocate(buffer.putFloat, value) + + def putDouble(value: Double) = withReallocate(buffer.putDouble, value) + + override def putByteArray(value: Array[Byte]): Unit = { + def process(value: Array[Byte]) = { + putInt(value.size) + buffer.put(value) + } + withReallocate(process, value) } } @@ -68,6 +111,8 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { val buffer = new java.io.ByteArrayOutputStream(initialCapacity) + def result = Some(buffer.toByteArray) + def putBoolean(value: Boolean) { if (value) buffer.write(1) else buffer.write(0) @@ -78,47 +123,31 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { } def putChar(value: Char) { - val fst = value >>> 8 & 0xff - val snd = value & 0xff - buffer.write(fst) - buffer.write(snd) + buffer.write(value >>> 8 & 0xff) + buffer.write(value & 0xff) } def putShort(value: Short) { - val fst = value >>> 8 & 0xff - val snd = value & 0xff - buffer.write(fst) - buffer.write(snd) + buffer.write(value >>> 8 & 0xff) + buffer.write(value & 0xff) } def putInt(value: Int) { - val fst = value >>> 24 - val snd = value >>> 16 & 0xff - val thrd = value >>> 8 & 0xff - val frth = value & 0xff - buffer.write(fst) - buffer.write(snd) - buffer.write(thrd) - buffer.write(frth) + buffer.write(value >>> 24) + buffer.write(value >>> 16 & 0xff) + buffer.write(value >>> 8 & 0xff) + buffer.write(value & 0xff) } def putLong(value: Long) { - val elem1 = (value >>> 56 & 0xff).asInstanceOf[Int] - val elem2 = (value >>> 48 & 0xff).asInstanceOf[Int] - val elem3 = (value >>> 40 & 0xff).asInstanceOf[Int] - val elem4 = (value >>> 32 & 0xff).asInstanceOf[Int] - val elem5 = (value >>> 24 & 0xff).asInstanceOf[Int] - val elem6 = (value >>> 16 & 0xff).asInstanceOf[Int] - val elem7 = (value >>> 8 & 0xff).asInstanceOf[Int] - val elem8 = (value & 0xff).asInstanceOf[Int] - buffer.write(elem1) - buffer.write(elem2) - buffer.write(elem3) - buffer.write(elem4) - buffer.write(elem5) - buffer.write(elem6) - buffer.write(elem7) - buffer.write(elem8) + buffer.write((value >>> 56 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 48 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 40 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 32 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 24 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 16 & 0xff).asInstanceOf[Int]) + buffer.write((value >>> 8 & 0xff).asInstanceOf[Int]) + buffer.write((value & 0xff).asInstanceOf[Int]) } def putFloat(value: Float) { @@ -131,32 +160,33 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { putLong(longValue) } - def putString(value: String) { - val bytes = value.getBytes("UTF-8") - putInt(bytes.length) - buffer.write(bytes) + override def putByteArray(value: Array[Byte]): Unit = { + putInt(value.length) + buffer.write(value) } + + //TODO override array } -class DataStreamOutput(stream: java.io.DataOutputStream) extends BinaryOutput { - - def putBoolean(value: Boolean) = stream.writeBoolean(value) +// class DataStreamOutput(stream: java.io.DataOutputStream) extends BinaryOutput { +// +// def putBoolean(value: Boolean) = stream.writeBoolean(value) - def putByte(value: Byte) = stream.writeByte(value) +// def putByte(value: Byte) = stream.writeByte(value) - def putChar(value: Char) = stream.writeChar(value) +// def putChar(value: Char) = stream.writeChar(value) - def putShort(value: Short) = stream.writeShort(value) +// def putShort(value: Short) = stream.writeShort(value) - def putInt(value: Int) = stream.writeInt(value) +// def putInt(value: Int) = stream.writeInt(value) - def putLong(value: Long) = stream.writeLong(value) +// def putLong(value: Long) = stream.writeLong(value) - def putFloat(value: Float) = stream.writeFloat(value) +// def putFloat(value: Float) = stream.writeFloat(value) - def putDouble(value: Double) = stream.writeDouble(value) +// def putDouble(value: Double) = stream.writeDouble(value) - def putString(value: String) = stream.writeUTF(value) +// override def putString(value: String) = stream.writeUTF(value) -} +// } diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index 45e63b1936..f8a2213256 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -132,8 +132,10 @@ class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) ext } @inline def result() = { - ??? - //BinaryPickle(output.result()) + output.result match { + case Some(b) => BinaryPickle(b) + case None => null + } } } @@ -261,37 +263,6 @@ abstract class AbstractBinaryReader(val mirror: Mirror) { class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { import format._ - def nextByte(): Byte = ??? -//{ -// val b = in.read() -// if (b == -1) throw new EndOfStreamException -// b.asInstanceOf[Byte] -//} - - def decodeStringWithLookahead(la: Byte): String = ??? -//{ -// // read 3 more bytes -// val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) -// val len = { -// val len0 = Util.decodeIntFrom(buf, 0) -// if (len0 > 1000) -// throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") -// else if (len0 < 0) -// throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") -// else -// len0 -// } -// val bytes = Array.ofDim[Byte](len) -// var num = in.read(bytes) -// while (num < len) { -// val readMore = in.read(bytes, num, len - num) -// num += readMore -// } -// new String(bytes, "UTF-8") -//} - - var gla: Option[Byte] = None - def beginEntryNoTag(): String = beginEntryNoTagDebug(false) @@ -301,16 +272,16 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF // debug(s"hints: $hints") if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = nextByte() + val lookahead = in.getByte() lookahead match { - case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null + case NULL_TAG => in.setLookahead(lookahead); FastTypeTag.Null case REF_TAG => FastTypeTag.Ref - case _ => gla = Some(lookahead); hints.tag + case _ => in.setLookahead(lookahead); hints.tag } } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { hints.tag } else { - val lookahead = nextByte() + val lookahead = in.getByte() // if (debugOn) // debug(s"checking lookahead: $lookahead") lookahead match { @@ -323,7 +294,8 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF case _ => // do not consume lookahead byte val res = try { - decodeStringWithLookahead(lookahead) + in.setLookahead(lookahead) + in.getStringWithLookahead } catch { case PicklingException(msg) => val primInfo = if (hints.tag == null) "" From e186b0ccecc892a587d10562caf6c9966e481e3a Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Tue, 7 Oct 2014 11:42:44 -0400 Subject: [PATCH 03/11] integration --- .../scala/pickling/binary/BinaryOutput.scala | 92 ++++++++++++++++++- .../scala/pickling/binary/BinaryPickle.scala | 13 ++- .../pickling/binary/BinaryPickleFormat.scala | 8 +- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index fd95e6d44a..2534ff22f3 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -4,6 +4,8 @@ abstract class BinaryOutput { def result: Option[Array[Byte]] + def ensureCapacity(capacity: Int): Unit + def putBoolean(value: Boolean): Unit def putByte(value: Byte): Unit @@ -53,8 +55,7 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { private var buffer = _buffer assert(buffer.order == ByteOrder.BIG_ENDIAN) - private def grow { - val newSize = 2*buffer.capacity + private def growTo(newSize: Int) { assert(newSize > 0) val newBuffer = if (buffer.isDirect) ByteBuffer.allocateDirect(newSize) @@ -67,6 +68,19 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { buffer = newBuffer } + + private def grow { + val newSize = 2*buffer.capacity + growTo(newSize) + } + + def ensureCapacity(capacity: Int) { + val left = buffer.remaining + if (left < capacity) { + growTo(buffer.capacity + (capacity - left)) + } + } + @inline private def withReallocate[A](op: A => ByteBuffer, value: A) { while(true) { buffer.mark @@ -112,6 +126,10 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { val buffer = new java.io.ByteArrayOutputStream(initialCapacity) def result = Some(buffer.toByteArray) + + def ensureCapacity(capacity: Int) { + //TODO ignore for the moment, not sure if it is worth creating a new buffer and copying the old one + } def putBoolean(value: Boolean) { if (value) buffer.write(1) @@ -169,6 +187,76 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { } +class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends BinaryOutput { + + def result = { + val res = buffer.result + if (res == null) None + else Some(res) + } + + def ensureCapacity(capacity: Int) { + //TODO ignore for the moment, not sure if it is worth creating a new buffer and copying the old one + } + + @inline private def write(i: Int) { + buffer += i.asInstanceOf[Byte] + } + + def putBoolean(value: Boolean) { + if (value) write(1) + else write(0) + } + + def putByte(value: Byte) { + write(value) + } + + def putChar(value: Char) { + write(value >>> 8 & 0xff) + write(value & 0xff) + } + + def putShort(value: Short) { + write(value >>> 8 & 0xff) + write(value & 0xff) + } + + def putInt(value: Int) { + write(value >>> 24) + write(value >>> 16 & 0xff) + write(value >>> 8 & 0xff) + write(value & 0xff) + } + + def putLong(value: Long) { + write((value >>> 56 & 0xff).asInstanceOf[Int]) + write((value >>> 48 & 0xff).asInstanceOf[Int]) + write((value >>> 40 & 0xff).asInstanceOf[Int]) + write((value >>> 32 & 0xff).asInstanceOf[Int]) + write((value >>> 24 & 0xff).asInstanceOf[Int]) + write((value >>> 16 & 0xff).asInstanceOf[Int]) + write((value >>> 8 & 0xff).asInstanceOf[Int]) + write((value & 0xff).asInstanceOf[Int]) + } + + def putFloat(value: Float) { + val intValue = java.lang.Float.floatToRawIntBits(value) + putInt(intValue) + } + + def putDouble(value: Double) { + val longValue = java.lang.Double.doubleToRawLongBits(value) + putLong(longValue) + } + + override def putByteArray(value: Array[Byte]): Unit = { + putInt(value.length) + buffer.put(value) + } + +} + // class DataStreamOutput(stream: java.io.DataOutputStream) extends BinaryOutput { // // def putBoolean(value: Boolean) = stream.writeBoolean(value) diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index f8a2213256..1a488819cc 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -34,7 +34,17 @@ case class BinaryPickleStream(input: InputStream) extends BinaryPickle { /* Do not override def toString to avoid traversing the input stream. */ } +case class BinaryInputPickle(input: BinaryInput) extends BinaryPickle { + val value: Array[Byte] = Array.ofDim[Byte](0) + + def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = + new BinaryPickleReader2(input, mirror, format) + + /* Do not override def toString to avoid traversing the input stream. */ +} + object BinaryPickle { + //TODO override that stuff def apply(a: Array[Byte]): BinaryPickle = new BinaryPickleArray(a) } @@ -42,8 +52,9 @@ object BinaryPickle { class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) extends PBuilder with PickleTools { import format._ + @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => - //mkOutput(hints.knownSize) + output.ensureCapacity(hints.knownSize) if (picklee == null) { output.putByte( NULL_TAG) diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index 5bc270a6eb..912ed19264 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -45,8 +45,12 @@ package object binary { class BinaryPickleFormat extends PickleFormat with Constants { type PickleType = BinaryPickle type OutputType = ArrayOutput[Byte] - def createBuilder() = new BinaryPickleBuilder(this, null) - def createBuilder(out: ArrayOutput[Byte]): PBuilder = new BinaryPickleBuilder(this, out) + //def createBuilder() = new BinaryPickleBuilder(this, null) + def createBuilder(): PBuilder = createBuilder(new ByteArrayOutput) + //def createBuilder(out: ArrayOutput[Byte]): PBuilder = new BinaryPickleBuilder(this, out) + def createBuilder(out: ArrayOutput[Byte]): PBuilder = createBuilder(new PickleArrayOutput(out)) + def createBuilder(out: BinaryOutput): PBuilder = new BinaryPickleBuilder2(this, out) + def createBuilder(out: java.nio.ByteBuffer): PBuilder = createBuilder(new ByteBufferOutput(out)) def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) } From 6fedbeb2b791a4f5207fb59abb4f4cfccc1e4a13 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Tue, 7 Oct 2014 19:18:48 -0400 Subject: [PATCH 04/11] started debuging. left todo: lower-level array cpy --- .../scala/pickling/binary/BinaryInput.scala | 70 ++++++------- .../scala/pickling/binary/BinaryOutput.scala | 4 +- .../scala/pickling/binary/BinaryPickle.scala | 15 +-- .../pickling/binary/BinaryPickleFormat.scala | 98 +++++++++---------- .../main/scala/pickling/binary/package.scala | 8 ++ .../pickling/run/binary-list-t-new.scala | 7 +- .../scala/pickling/run/share-binary.scala | 4 +- .../test/scala/pickling/run/vector-int.scala | 4 +- 8 files changed, 115 insertions(+), 95 deletions(-) create mode 100644 core/src/main/scala/pickling/binary/package.scala diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala index 52d7f47327..f24de0cb69 100644 --- a/core/src/main/scala/pickling/binary/BinaryInput.scala +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -20,11 +20,16 @@ abstract class BinaryInput { def getDouble(): Double - def getString(): String + def getString(): String = { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) + getBytes(array) + new String(array, "UTF-8") + } //generic method when the performance is not an issue @inline private def getArray[T: ClassTag](get: () => T): Array[T] = { - val size = getInt + val size = getIntWithLookahead val array = Array.ofDim[T](size) for(i <- 0 until size) { array(i) = get() @@ -41,7 +46,6 @@ abstract class BinaryInput { def getLongArray(): Array[Long] = getArray(getLong) def getShortArray(): Array[Short] = getArray(getShort) - //TODO lookahead protected var lookahead: Option[Byte] = None def setLookahead(b: Byte) { @@ -64,10 +68,12 @@ abstract class BinaryInput { def getBytes(array: Array[Byte]) - def getStringWithLookahead() { - val size = getIntWithLookahead - val array = Array.ofDim[Byte](size) - getBytes(array) + def getStringWithLookahead(la: Byte): String = { + val oldLa = lookahead + setLookahead(la) + val res = getString + lookahead = oldLa + res } } @@ -90,12 +96,12 @@ class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { def getDouble() = buffer.getDouble - def getString() = { - val size = getInt - val bytes = Array.ofDim[Byte](size) - buffer.get(bytes) - new String(bytes, "UTF-8") - } +//def getString() = { +// val size = getInt +// val bytes = Array.ofDim[Byte](size) +// buffer.get(bytes) +// new String(bytes, "UTF-8") +//} def getBytes(array: Array[Byte]) { buffer.get(array) @@ -108,11 +114,9 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { private var idx = 0 def getBoolean() = { - val res = (data(idx) != 0) - idx += 1 - res + getByte() != 0 } - + def getByte() = { val res = data(idx) idx += 1 @@ -138,7 +142,7 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { def getInt() = { var res = (0: Int) res |= data(idx ) << 24 - res |= data(idx+1) << 26 + res |= data(idx+1) << 16 res |= data(idx+2) << 8 res |= data(idx+3) idx += 4 @@ -147,14 +151,14 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { def getLong() = { var res = (0: Long) - res |= data(idx ) << 56 - res |= data(idx+1) << 48 - res |= data(idx+2) << 40 - res |= data(idx+3) << 32 - res |= data(idx+4) << 24 - res |= data(idx+5) << 26 - res |= data(idx+6) << 8 - res |= data(idx+7) + res |= (data(idx ).toLong << 56) & 0xFFFFFFFFFFFFFFFFL + res |= (data(idx+1).toLong << 48) & 0x00FFFFFFFFFFFFFFL + res |= (data(idx+2).toLong << 40) & 0x0000FFFFFFFFFFFFL + res |= (data(idx+3).toLong << 32) & 0x000000FFFFFFFFFFL + res |= (data(idx+4).toLong << 24) & 0x00000000FFFFFFFFL + res |= (data(idx+5).toLong << 16) & 0x0000000000FFFFFFL + res |= (data(idx+6).toLong << 8 ) & 0x000000000000FFFFL + res |= (data(idx+7).toLong ) & 0x00000000000000FFL idx += 8 res } @@ -169,16 +173,16 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { java.lang.Double.longBitsToDouble(r) } - def getString() = { - val size = getInt - val bytes = data.slice(idx, idx + size) - idx += size - new String(bytes, "UTF-8") - } +//def getString() = { +// val size = getInt +// val bytes = data.slice(idx, idx + size) +// idx += size +// new String(bytes, "UTF-8") +//} def getBytes(array: Array[Byte]) { val size = array.size - data.copyToArray(array, idx, size) + data.view(idx, idx+size).copyToArray(array, 0, size) idx += size } diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index 2534ff22f3..e0faf4504e 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -132,8 +132,8 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { } def putBoolean(value: Boolean) { - if (value) buffer.write(1) - else buffer.write(0) + if (value) putByte(1) + else putByte(0) } def putByte(value: Byte) { diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index 1a488819cc..fad019b627 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -20,7 +20,8 @@ case class BinaryPickleArray(data: Array[Byte]) extends BinaryPickle { val value: Array[Byte] = data def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryPickleReader(data, mirror, format) + new BinaryPickleReader2(new ByteArrayInput(data), mirror, format) + //new BinaryPickleReader(data, mirror, format) override def toString = s"""BinaryPickle(${value.mkString("[", ",", "]")})""" } @@ -49,7 +50,7 @@ object BinaryPickle { new BinaryPickleArray(a) } -class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) extends PBuilder with PickleTools { +class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) extends BinaryPBuilder with PickleTools { import format._ @@ -145,13 +146,13 @@ class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) ext @inline def result() = { output.result match { case Some(b) => BinaryPickle(b) - case None => null + case None => BinaryPickle(null) //TODO is that safe ? } } } -class BinaryPickleBuilder(format: BinaryPickleFormat, out: ArrayOutput[Byte]) extends PBuilder with PickleTools { +class BinaryPickleBuilder(format: BinaryPickleFormat, out: ArrayOutput[Byte]) extends BinaryPBuilder with PickleTools { import format._ private var output: ArrayOutput[Byte] = out @@ -285,7 +286,8 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { val lookahead = in.getByte() lookahead match { - case NULL_TAG => in.setLookahead(lookahead); FastTypeTag.Null + case UNIT_TAG => FastTypeTag.Unit + case NULL_TAG => FastTypeTag.Null case REF_TAG => FastTypeTag.Ref case _ => in.setLookahead(lookahead); hints.tag } @@ -305,8 +307,7 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF case _ => // do not consume lookahead byte val res = try { - in.setLookahead(lookahead) - in.getStringWithLookahead + in.getStringWithLookahead(lookahead) } catch { case PicklingException(msg) => val primInfo = if (hints.tag == null) "" diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index 912ed19264..23410e1a4c 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -1,59 +1,57 @@ -package scala.pickling +package scala.pickling.binary +import scala.pickling._ import scala.pickling.internal._ import scala.language.implicitConversions import scala.reflect.runtime.universe.Mirror -package object binary { - implicit val pickleFormat = new BinaryPickleFormat - implicit def toBinaryPickle(value: Array[Byte]): BinaryPickle = BinaryPickle(value) - - trait Constants { - val NULL_TAG : Byte = -2 - val REF_TAG : Byte = -3 - val UNIT_TAG : Byte = -4 - val ELIDED_TAG: Byte = -5 - - val KEY_NULL = FastTypeTag.Null.key - val KEY_BYTE = FastTypeTag.Byte.key - val KEY_SHORT = FastTypeTag.Short.key - val KEY_CHAR = FastTypeTag.Char.key - val KEY_INT = FastTypeTag.Int.key - val KEY_LONG = FastTypeTag.Long.key - val KEY_BOOLEAN = FastTypeTag.Boolean.key - val KEY_FLOAT = FastTypeTag.Float.key - val KEY_DOUBLE = FastTypeTag.Double.key - val KEY_UNIT = FastTypeTag.Unit.key - - val KEY_STRING = FastTypeTag.String.key - - val KEY_ARRAY_BYTE = FastTypeTag.ArrayByte.key - val KEY_ARRAY_SHORT = FastTypeTag.ArrayShort.key - val KEY_ARRAY_CHAR = FastTypeTag.ArrayChar.key - val KEY_ARRAY_INT = FastTypeTag.ArrayInt.key - val KEY_ARRAY_LONG = FastTypeTag.ArrayLong.key - val KEY_ARRAY_BOOLEAN = FastTypeTag.ArrayBoolean.key - val KEY_ARRAY_FLOAT = FastTypeTag.ArrayFloat.key - val KEY_ARRAY_DOUBLE = FastTypeTag.ArrayDouble.key - - val KEY_REF = FastTypeTag.Ref.key - - val primitives = Set(KEY_NULL, KEY_REF, KEY_BYTE, KEY_SHORT, KEY_CHAR, KEY_INT, KEY_LONG, KEY_BOOLEAN, KEY_FLOAT, KEY_DOUBLE, KEY_UNIT, KEY_STRING, KEY_ARRAY_BYTE, KEY_ARRAY_SHORT, KEY_ARRAY_CHAR, KEY_ARRAY_INT, KEY_ARRAY_LONG, KEY_ARRAY_BOOLEAN, KEY_ARRAY_FLOAT, KEY_ARRAY_DOUBLE) - val nullablePrimitives = Set(KEY_NULL, KEY_STRING, KEY_ARRAY_BYTE, KEY_ARRAY_SHORT, KEY_ARRAY_CHAR, KEY_ARRAY_INT, KEY_ARRAY_LONG, KEY_ARRAY_BOOLEAN, KEY_ARRAY_FLOAT, KEY_ARRAY_DOUBLE) - } - - class BinaryPickleFormat extends PickleFormat with Constants { - type PickleType = BinaryPickle - type OutputType = ArrayOutput[Byte] - //def createBuilder() = new BinaryPickleBuilder(this, null) - def createBuilder(): PBuilder = createBuilder(new ByteArrayOutput) - //def createBuilder(out: ArrayOutput[Byte]): PBuilder = new BinaryPickleBuilder(this, out) - def createBuilder(out: ArrayOutput[Byte]): PBuilder = createBuilder(new PickleArrayOutput(out)) - def createBuilder(out: BinaryOutput): PBuilder = new BinaryPickleBuilder2(this, out) - def createBuilder(out: java.nio.ByteBuffer): PBuilder = createBuilder(new ByteBufferOutput(out)) - def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) - } +trait BinaryPBuilder extends PBuilder { + def result(): BinaryPickle } +class BinaryPickleFormat extends PickleFormat with Constants { + type PickleType = BinaryPickle + type OutputType = ArrayOutput[Byte] + //def createBuilder(): BinaryPBuilder = new BinaryPickleBuilder(this, null) + def createBuilder(): BinaryPBuilder = createBuilder(new ByteArrayOutput) + //def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = new BinaryPickleBuilder(this, out) + def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = createBuilder(new PickleArrayOutput(out)) + def createBuilder(out: BinaryOutput): BinaryPBuilder = new BinaryPickleBuilder2(this, out) + def createBuilder(out: java.nio.ByteBuffer): BinaryPBuilder = createBuilder(new ByteBufferOutput(out)) + def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) +} +trait Constants { + val NULL_TAG : Byte = -2 + val REF_TAG : Byte = -3 + val UNIT_TAG : Byte = -4 + val ELIDED_TAG: Byte = -5 + + val KEY_NULL = FastTypeTag.Null.key + val KEY_BYTE = FastTypeTag.Byte.key + val KEY_SHORT = FastTypeTag.Short.key + val KEY_CHAR = FastTypeTag.Char.key + val KEY_INT = FastTypeTag.Int.key + val KEY_LONG = FastTypeTag.Long.key + val KEY_BOOLEAN = FastTypeTag.Boolean.key + val KEY_FLOAT = FastTypeTag.Float.key + val KEY_DOUBLE = FastTypeTag.Double.key + val KEY_UNIT = FastTypeTag.Unit.key + + val KEY_STRING = FastTypeTag.String.key + + val KEY_ARRAY_BYTE = FastTypeTag.ArrayByte.key + val KEY_ARRAY_SHORT = FastTypeTag.ArrayShort.key + val KEY_ARRAY_CHAR = FastTypeTag.ArrayChar.key + val KEY_ARRAY_INT = FastTypeTag.ArrayInt.key + val KEY_ARRAY_LONG = FastTypeTag.ArrayLong.key + val KEY_ARRAY_BOOLEAN = FastTypeTag.ArrayBoolean.key + val KEY_ARRAY_FLOAT = FastTypeTag.ArrayFloat.key + val KEY_ARRAY_DOUBLE = FastTypeTag.ArrayDouble.key + + val KEY_REF = FastTypeTag.Ref.key + + val primitives = Set(KEY_NULL, KEY_REF, KEY_BYTE, KEY_SHORT, KEY_CHAR, KEY_INT, KEY_LONG, KEY_BOOLEAN, KEY_FLOAT, KEY_DOUBLE, KEY_UNIT, KEY_STRING, KEY_ARRAY_BYTE, KEY_ARRAY_SHORT, KEY_ARRAY_CHAR, KEY_ARRAY_INT, KEY_ARRAY_LONG, KEY_ARRAY_BOOLEAN, KEY_ARRAY_FLOAT, KEY_ARRAY_DOUBLE) + val nullablePrimitives = Set(KEY_NULL, KEY_STRING, KEY_ARRAY_BYTE, KEY_ARRAY_SHORT, KEY_ARRAY_CHAR, KEY_ARRAY_INT, KEY_ARRAY_LONG, KEY_ARRAY_BOOLEAN, KEY_ARRAY_FLOAT, KEY_ARRAY_DOUBLE) +} diff --git a/core/src/main/scala/pickling/binary/package.scala b/core/src/main/scala/pickling/binary/package.scala new file mode 100644 index 0000000000..b99679b762 --- /dev/null +++ b/core/src/main/scala/pickling/binary/package.scala @@ -0,0 +1,8 @@ +package scala.pickling + +import scala.language.implicitConversions + +package object binary { + implicit val pickleFormat = new BinaryPickleFormat + implicit def toBinaryPickle(value: Array[Byte]): BinaryPickle = BinaryPickle(value) +} diff --git a/core/src/test/scala/pickling/run/binary-list-t-new.scala b/core/src/test/scala/pickling/run/binary-list-t-new.scala index 6c5ac781d3..2fd31cb522 100644 --- a/core/src/test/scala/pickling/run/binary-list-t-new.scala +++ b/core/src/test/scala/pickling/run/binary-list-t-new.scala @@ -7,7 +7,12 @@ import binary._ class BinaryListTNewTest extends FunSuite { test("main") { val pickle = List(1, 2, 3).pickle - assert(pickle.toString === "BinaryPickle([0,0,0,50,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,36,99,111,108,111,110,36,99,111,108,111,110,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])") + val expected0 = "BinaryPickle([0,0,0,50,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,36,99,111,108,111,110,36,99,111,108,111,110,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])" + val expected = "BinaryPickle([0,0,0,50,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,36,99,111,108,111,110,36,99,111,108,111,110,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3])" + val result = pickle.toString + //Console.err.println("expected " + expected) + //Console.err.println("result " + result) + assert(result === expected) assert(pickle.unpickle[List[Int]] === List(1, 2, 3)) } } diff --git a/core/src/test/scala/pickling/run/share-binary.scala b/core/src/test/scala/pickling/run/share-binary.scala index 92a988fcc4..20ced6ee01 100644 --- a/core/src/test/scala/pickling/run/share-binary.scala +++ b/core/src/test/scala/pickling/run/share-binary.scala @@ -23,7 +23,9 @@ class ShareBinaryTest extends FunSuite { test("loop-share-nonprimitives") { c1.c = c3 val pickle = c1.pickle - assert(pickle.toString === "BinaryPickle([0,0,0,29,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,115,104,97,114,101,46,98,105,110,97,114,121,46,67,0,0,0,2,99,49,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-5,0,0,0,2,99,51,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-5,0,0,0,2,99,50,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-3,0,0,0,0])") + val expected = "BinaryPickle([0,0,0,29,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,115,104,97,114,101,46,98,105,110,97,114,121,46,67,0,0,0,2,99,49,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-5,0,0,0,2,99,51,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-5,0,0,0,2,99,50,0,0,0,4,100,101,115,99,0,0,0,1,1,0,0,0,-3,0,0,0,0])" + val found = pickle.toString + assert(found === expected) val c11 = pickle.unpickle[C] val c13 = c11.c diff --git a/core/src/test/scala/pickling/run/vector-int.scala b/core/src/test/scala/pickling/run/vector-int.scala index e7a1ab2fc2..162eb7ef1c 100644 --- a/core/src/test/scala/pickling/run/vector-int.scala +++ b/core/src/test/scala/pickling/run/vector-int.scala @@ -8,7 +8,9 @@ class VectorIntTest extends FunSuite { test("main") { val v = Vector(1, 2, 3) val pickle = v.pickle - assert(pickle.toString === "BinaryPickle([0,0,0,44,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,86,101,99,116,111,114,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])") + val expected0 = "BinaryPickle([0,0,0,44,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,86,101,99,116,111,114,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])" + val expected = "BinaryPickle([0,0,0,44,115,99,97,108,97,46,99,111,108,108,101,99,116,105,111,110,46,105,109,109,117,116,97,98,108,101,46,86,101,99,116,111,114,91,115,99,97,108,97,46,73,110,116,93,0,0,0,3,0,0,0,1,0,0,0,2,0,0,0,3])" + assert(pickle.toString === expected) assert(pickle.unpickle[Vector[Int]] === v) } } From 6c9bc2a25a8b30c8fb2bd5f6981decd05a2db805 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Tue, 7 Oct 2014 23:49:41 -0400 Subject: [PATCH 05/11] fixing some bugs, removing the old code --- .../scala/pickling/binary/BinaryInput.scala | 99 ++-- .../scala/pickling/binary/BinaryOutput.scala | 54 +- .../scala/pickling/binary/BinaryPickle.scala | 488 +----------------- .../pickling/binary/BinaryPickleFormat.scala | 4 +- .../pickling/run/binary-inputstream.scala | 2 + 5 files changed, 73 insertions(+), 574 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala index f24de0cb69..155e3ed744 100644 --- a/core/src/main/scala/pickling/binary/BinaryInput.scala +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -4,7 +4,9 @@ import scala.reflect.ClassTag abstract class BinaryInput { - def getBoolean(): Boolean + def getBoolean(): Boolean = { + getByte() != 0 + } def getByte(): Byte @@ -21,9 +23,7 @@ abstract class BinaryInput { def getDouble(): Double def getString(): String = { - val size = getIntWithLookahead - val array = Array.ofDim[Byte](size) - getBytes(array) + val array = getByteArray new String(array, "UTF-8") } @@ -56,9 +56,9 @@ abstract class BinaryInput { lookahead match { case Some(b) => var i = b << 24 - i |= getByte << 16 - i |= getByte << 8 - i |= getByte + i |= (getByte.toInt << 16) & 0xFF0000 + i |= (getByte.toInt << 8) & 0xFF00 + i |= (getByte.toInt) & 0xFF lookahead = None i case None => @@ -66,8 +66,6 @@ abstract class BinaryInput { } } - def getBytes(array: Array[Byte]) - def getStringWithLookahead(la: Byte): String = { val oldLa = lookahead setLookahead(la) @@ -80,7 +78,8 @@ abstract class BinaryInput { class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { - def getBoolean() = buffer.get.asInstanceOf[Boolean] + import java.nio.ByteOrder + assert(buffer.order == ByteOrder.BIG_ENDIAN) def getByte() = buffer.get @@ -96,15 +95,11 @@ class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { def getDouble() = buffer.getDouble -//def getString() = { -// val size = getInt -// val bytes = Array.ofDim[Byte](size) -// buffer.get(bytes) -// new String(bytes, "UTF-8") -//} - - def getBytes(array: Array[Byte]) { + override def getByteArray(): Array[Byte] = { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) buffer.get(array) + array } } @@ -113,10 +108,6 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { private var idx = 0 - def getBoolean() = { - getByte() != 0 - } - def getByte() = { val res = data(idx) idx += 1 @@ -126,7 +117,7 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { def getChar() = { var res = 0 res |= data(idx ) << 8 - res |= data(idx+1) + res |= data(idx+1).toInt & 0xFF idx += 2 res.asInstanceOf[Char] } @@ -134,17 +125,17 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { def getShort() = { var res = 0 res |= data(idx ) << 8 - res |= data(idx+1) + res |= data(idx+1).toInt & 0xFF idx += 2 res.asInstanceOf[Short] } def getInt() = { var res = (0: Int) - res |= data(idx ) << 24 - res |= data(idx+1) << 16 - res |= data(idx+2) << 8 - res |= data(idx+3) + res |= (data(idx ) << 24) + res |= (data(idx+1) << 16) & 0xFF0000 + res |= (data(idx+2) << 8 ) & 0xFF00 + res |= (data(idx+3) ) & 0xFF idx += 4 res } @@ -173,41 +164,35 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { java.lang.Double.longBitsToDouble(r) } -//def getString() = { -// val size = getInt -// val bytes = data.slice(idx, idx + size) -// idx += size -// new String(bytes, "UTF-8") -//} - - def getBytes(array: Array[Byte]) { - val size = array.size + override def getByteArray(): Array[Byte] = { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) data.view(idx, idx+size).copyToArray(array, 0, size) idx += size + array } //TODO override array for faster copy } -// class DataStreamInput(stream: java.io.DataInputStream) extends BinaryInput { - -// def getBoolean() = stream.readBoolean() - -// def getByte() = stream.readByte() - -// def getChar() = stream.readChar() - -// def getShort() = stream.readShort() - -// def getInt() = stream.readInt() - -// def getLong() = stream.readLong() - -// def getFloat() = stream.readFloat() - -// def getDouble() = stream.readDouble() - -// def getString() = stream.readUTF() +class StreamInput(stream: java.io.InputStream) extends BinaryInput { + val ds = new java.io.DataInputStream(stream) + def getByte() = ds.readByte() + def getChar() = ds.readChar() + def getShort() = ds.readShort() + def getInt() = ds.readInt() + def getLong() = ds.readLong() + def getFloat() = ds.readFloat() + def getDouble() = ds.readDouble() + + override def getByteArray(): Array[Byte] = { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) + ds.readFully(array) + array + } -// } + //TODO check endianness + +} diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index e0faf4504e..b9e5115fc0 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -6,7 +6,10 @@ abstract class BinaryOutput { def ensureCapacity(capacity: Int): Unit - def putBoolean(value: Boolean): Unit + def putBoolean(value: Boolean): Unit = { + if (value) putByte(1) + else putByte(0) + } def putByte(value: Byte): Unit @@ -95,8 +98,6 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { } } - def putBoolean(value: Boolean) = withReallocate[Byte](buffer.put, value.asInstanceOf[Byte]) - def putByte(value: Byte) = withReallocate[Byte](buffer.put, value) def putChar(value: Char) = withReallocate(buffer.putChar, value) @@ -131,11 +132,6 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { //TODO ignore for the moment, not sure if it is worth creating a new buffer and copying the old one } - def putBoolean(value: Boolean) { - if (value) putByte(1) - else putByte(0) - } - def putByte(value: Byte) { buffer.write(value) } @@ -203,11 +199,6 @@ class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends Binary buffer += i.asInstanceOf[Byte] } - def putBoolean(value: Boolean) { - if (value) write(1) - else write(0) - } - def putByte(value: Byte) { write(value) } @@ -257,24 +248,21 @@ class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends Binary } -// class DataStreamOutput(stream: java.io.DataOutputStream) extends BinaryOutput { -// -// def putBoolean(value: Boolean) = stream.writeBoolean(value) - -// def putByte(value: Byte) = stream.writeByte(value) - -// def putChar(value: Char) = stream.writeChar(value) - -// def putShort(value: Short) = stream.writeShort(value) - -// def putInt(value: Int) = stream.writeInt(value) - -// def putLong(value: Long) = stream.writeLong(value) - -// def putFloat(value: Float) = stream.writeFloat(value) - -// def putDouble(value: Double) = stream.writeDouble(value) - -// override def putString(value: String) = stream.writeUTF(value) +class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { + val ds = new java.io.DataOutputStream(stream) + def result = None + def ensureCapacity(capacity: Int) { } + def putByte(value: Byte) = ds.writeByte(value) + def putChar(value: Char) = ds.writeChar(value) + def putShort(value: Short) = ds.writeShort(value) + def putInt(value: Int) = ds.writeInt(value) + def putLong(value: Long) = ds.writeLong(value) + def putFloat(value: Float) = ds.writeFloat(value) + def putDouble(value: Double) = ds.writeDouble(value) + + override def putByteArray(value: Array[Byte]): Unit = { + putInt(value.length) + ds.write(value) + } -// } +} diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index fad019b627..0252aaf1d1 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -20,7 +20,7 @@ case class BinaryPickleArray(data: Array[Byte]) extends BinaryPickle { val value: Array[Byte] = data def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryPickleReader2(new ByteArrayInput(data), mirror, format) + new BinaryPickleReader(new ByteArrayInput(data), mirror, format) //new BinaryPickleReader(data, mirror, format) override def toString = s"""BinaryPickle(${value.mkString("[", ",", "]")})""" @@ -30,7 +30,8 @@ case class BinaryPickleStream(input: InputStream) extends BinaryPickle { val value: Array[Byte] = Array.ofDim[Byte](0) def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryInputStreamReader(input, mirror, format) + new BinaryPickleReader(new StreamInput(input), mirror, format) + //new BinaryInputStreamReader(input, mirror, format) /* Do not override def toString to avoid traversing the input stream. */ } @@ -39,7 +40,7 @@ case class BinaryInputPickle(input: BinaryInput) extends BinaryPickle { val value: Array[Byte] = Array.ofDim[Byte](0) def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryPickleReader2(input, mirror, format) + new BinaryPickleReader(input, mirror, format) /* Do not override def toString to avoid traversing the input stream. */ } @@ -50,7 +51,7 @@ object BinaryPickle { new BinaryPickleArray(a) } -class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) extends BinaryPBuilder with PickleTools { +class BinaryPickleBuilder(format: BinaryPickleFormat, output: BinaryOutput) extends BinaryPBuilder with PickleTools { import format._ @@ -152,112 +153,6 @@ class BinaryPickleBuilder2(format: BinaryPickleFormat, output: BinaryOutput) ext } -class BinaryPickleBuilder(format: BinaryPickleFormat, out: ArrayOutput[Byte]) extends BinaryPBuilder with PickleTools { - import format._ - - private var output: ArrayOutput[Byte] = out - - @inline private[this] def mkOutput(knownSize: Int): Unit = - if (output == null) - output = if (knownSize != -1) new scala.pickling.ByteArrayOutput(knownSize) - else new ByteArrayBufferOutput - - @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => - mkOutput(hints.knownSize) - - if (picklee == null) { - Util.encodeByte(output, NULL_TAG) - } else if (hints.oid != -1) { - Util.encodeByte(output, REF_TAG) - Util.encodeInt(output, hints.oid) - } else { - if (!hints.isElidedType) { - // quickly decide whether we should use picklee.getClass instead - val ts = - if (hints.tag.key.contains("anonfun$")) picklee.getClass.getName - else hints.tag.key - Util.encodeString(output, ts) - } - - // NOTE: it looks like we don't have to write object ids at all - // traversals employed by pickling and unpickling are exactly the same - // hence when unpickling it's enough to just increment the nextUnpicklee counter - // and everything will work out automatically! - - hints.tag.key match { // PERF: should store typestring once in hints. - case KEY_UNIT => - Util.encodeByte(output, UNIT_TAG) - case KEY_NULL => - Util.encodeByte(output, NULL_TAG) - case KEY_BYTE => - Util.encodeByte(output, picklee.asInstanceOf[Byte]) - case KEY_SHORT => - Util.encodeShort(output, picklee.asInstanceOf[Short]) - case KEY_CHAR => - Util.encodeChar(output, picklee.asInstanceOf[Char]) - case KEY_INT => - Util.encodeInt(output, picklee.asInstanceOf[Int]) - case KEY_LONG => - Util.encodeLong(output, picklee.asInstanceOf[Long]) - case KEY_BOOLEAN => - Util.encodeBoolean(output, picklee.asInstanceOf[Boolean]) - case KEY_FLOAT => - val intValue = java.lang.Float.floatToRawIntBits(picklee.asInstanceOf[Float]) - Util.encodeInt(output, intValue) - case KEY_DOUBLE => - val longValue = java.lang.Double.doubleToRawLongBits(picklee.asInstanceOf[Double]) - Util.encodeLong(output, longValue) - case KEY_STRING => - Util.encodeString(output, picklee.asInstanceOf[String]) - case KEY_ARRAY_BYTE => - Util.encodeByteArray(output, picklee.asInstanceOf[Array[Byte]]) - case KEY_ARRAY_CHAR => - Util.encodeCharArray(output, picklee.asInstanceOf[Array[Char]]) - case KEY_ARRAY_SHORT => - Util.encodeShortArray(output, picklee.asInstanceOf[Array[Short]]) - case KEY_ARRAY_INT => - Util.encodeIntArray(output, picklee.asInstanceOf[Array[Int]]) - case KEY_ARRAY_LONG => - Util.encodeLongArray(output, picklee.asInstanceOf[Array[Long]]) - case KEY_ARRAY_BOOLEAN => - Util.encodeBooleanArray(output, picklee.asInstanceOf[Array[Boolean]]) - case KEY_ARRAY_FLOAT => - Util.encodeFloatArray(output, picklee.asInstanceOf[Array[Float]]) - case KEY_ARRAY_DOUBLE => - Util.encodeDoubleArray(output, picklee.asInstanceOf[Array[Double]]) - case _ => - if (hints.isElidedType) Util.encodeByte(output, ELIDED_TAG) - } - } - this - } - - @inline def putField(name: String, pickler: PBuilder => Unit): PBuilder = { - // can skip writing name if we pickle/unpickle in the same order - pickler(this) - this - } - - @inline def endEntry(): Unit = { /* do nothing */ } - - @inline def beginCollection(length: Int): PBuilder = { - Util.encodeInt(output, length) - this - } - - @inline def putElement(pickler: PBuilder => Unit): PBuilder = { - pickler(this) - this - } - - @inline def endCollection(): Unit = { - } - - @inline def result() = { - BinaryPickle(output.result()) - } -} - abstract class AbstractBinaryReader(val mirror: Mirror) { protected var _lastTagRead: FastTypeTag[_] = null protected var _lastTypeStringRead: String = null @@ -272,7 +167,7 @@ abstract class AbstractBinaryReader(val mirror: Mirror) { } } -class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { +class BinaryPickleReader(in: BinaryInput, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { import format._ def beginEntryNoTag(): String = @@ -368,7 +263,7 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF def atObject: Boolean = !atPrimitive - def readField(name: String): BinaryPickleReader2 = + def readField(name: String): BinaryPickleReader = this def endEntry(): Unit = { /* do nothing */ } @@ -383,372 +278,3 @@ class BinaryPickleReader2(in: BinaryInput, mirror: Mirror, format: BinaryPickleF } -class BinaryInputStreamReader(in: InputStream, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - def nextByte(): Byte = { - val b = in.read() - if (b == -1) throw new EndOfStreamException - b.asInstanceOf[Byte] - } - - def decodeStringWithLookahead(la: Byte): String = { - // read 3 more bytes - val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) - val len = { - val len0 = Util.decodeIntFrom(buf, 0) - if (len0 > 1000) - throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") - else if (len0 < 0) - throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") - else - len0 - } - val bytes = Array.ofDim[Byte](len) - var num = in.read(bytes) - while (num < len) { - val readMore = in.read(bytes, num, len - num) - num += readMore - } - new String(bytes, "UTF-8") - } - - var gla: Option[Byte] = None - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - // if (debugOn) - // debug(s"hints: $hints") - - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = nextByte() - lookahead match { - case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null - case REF_TAG => FastTypeTag.Ref - case _ => gla = Some(lookahead); hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = nextByte() - // if (debugOn) - // debug(s"checking lookahead: $lookahead") - lookahead match { - case NULL_TAG => - FastTypeTag.Null - case ELIDED_TAG => - hints.tag - case REF_TAG => - FastTypeTag.Ref - case _ => - // do not consume lookahead byte - val res = try { - decodeStringWithLookahead(lookahead) - } catch { - case PicklingException(msg) => - val primInfo = if (hints.tag == null) "" - else s"\nnullable prim: ${nullablePrimitives.contains(hints.tag.key)}\nprim: ${primitives.contains(hints.tag.key)}" - throw PicklingException(s"error decoding type string. debug info: $hints$primInfo\ncause:$msg") - } - // if (debugOn) - // debug(s"decodeStringWithLookahead: $res") - res - } - } - } - if (res.isInstanceOf[String]) { - // if (debugOn) - // debug(s"replacing tag with last type string read: ${res.asInstanceOf[String]}") - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def decodeInt(): Int = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - } - - def decodeIntWithLookahead(): Int = gla match { - case Some(fstByte) => - gla = None // clear global lookahead - val buf = Array[Byte](fstByte, nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - case None => - decodeInt() - } - - def decodeShort(): Short = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toShort - val snd = (buf(1) & 0x00FF).toShort - (fst | snd).toShort - } - - def decodeChar(): Char = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toChar - val snd = (buf(1) & 0x00FF).toChar - (fst | snd).toChar - } - - def decodeLong(): Long = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte(), - nextByte(), nextByte(), nextByte(), nextByte()) - val elem1 = ((buf(0).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong - val elem2 = ((buf(1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong - val elem3 = ((buf(2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong - val elem4 = ((buf(3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong - val elem5 = ((buf(4).toLong << 24) & 0x00000000FFFFFFFFL).toLong - val elem6 = ((buf(5).toLong << 16) & 0x0000000000FFFFFFL).toLong - val elem7 = ((buf(6).toLong << 8) & 0x000000000000FFFFL).toLong - val elem8 = (buf(7).toLong & 0x00000000000000FFL).toLong - elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 - } - - def decodeBoolean(): Boolean = { - nextByte() != 0 - } - - def decodeString(): String = { - val len = decodeIntWithLookahead() - val bytes = Array.ofDim[Byte](len) - if (len > 0) { - val num = in.read(bytes) - if (num < len) throw new Exception("Could not read enough bytes from input stream") - } - new String(bytes, "UTF-8") - } - - def decodeByteArray(): Array[Byte] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeByteArray(arr, 0, len) - } - - def decodeShortArray(): Array[Short] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeShortArray(arr, 0, len) - } - - def decodeCharArray(): Array[Char] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeCharArray(arr, 0, len) - } - - def decodeIntArray(): Array[Int] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeIntArray(arr, 0, len) - } - - // Consider a macro such as Util.decodeArray[Long](in, len) - def decodeLongArray(): Array[Long] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeLongArray(arr, 0, len) - } - - def decodeBooleanArray(): Array[Boolean] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeBooleanArray(arr, 0, len) - } - - def decodeFloatArray(): Array[Float] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeFloatArray(arr, 0, len) - } - - def decodeDoubleArray(): Array[Double] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeDoubleArray(arr, 0, len) - } - - def readPrimitive(): Any = { - val res = lastTagRead.key match { - case KEY_NULL => null - case KEY_REF => lookupUnpicklee(decodeInt()) - case KEY_BYTE => nextByte() - case KEY_SHORT => decodeShort() - case KEY_CHAR => decodeChar() - case KEY_INT => decodeInt() - case KEY_LONG => decodeLong() - case KEY_BOOLEAN => decodeBoolean() - case KEY_FLOAT => - val r = decodeInt() - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = decodeLong() - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => decodeString() - - case KEY_ARRAY_BYTE => decodeByteArray() - case KEY_ARRAY_SHORT => decodeShortArray() - case KEY_ARRAY_CHAR => decodeCharArray() - case KEY_ARRAY_INT => decodeIntArray() - case KEY_ARRAY_LONG => decodeLongArray() - case KEY_ARRAY_BOOLEAN => decodeBooleanArray() - case KEY_ARRAY_FLOAT => decodeFloatArray() - case KEY_ARRAY_DOUBLE => decodeDoubleArray() - } - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryInputStreamReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - decodeInt() - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } -} - -class BinaryPickleReader(arr: Array[Byte], mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - private var pos = 0 - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = arr(pos) - lookahead match { - case UNIT_TAG => pos += 1; FastTypeTag.Unit - case NULL_TAG => pos += 1; FastTypeTag.Null - case REF_TAG => pos += 1; FastTypeTag.Ref - case _ => hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = arr(pos) - lookahead match { - case NULL_TAG => - pos += 1 - FastTypeTag.Null - case ELIDED_TAG => - pos += 1 - hints.tag - case REF_TAG => - pos += 1 - FastTypeTag.Ref - case _ => - val (typeString, newpos) = Util.decodeStringFrom(arr, pos) - pos = newpos - typeString - } - } - } - if (res.isInstanceOf[String]) { - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def readPrimitive(): Any = { - var newpos = pos - val res = lastTagRead.key match { - case KEY_UNIT => () - case KEY_NULL => null - case KEY_REF => newpos = pos+4 ; lookupUnpicklee(Util.decodeIntFrom(arr, pos)) - case KEY_BYTE => newpos = pos+1 ; arr(pos) - case KEY_SHORT => newpos = pos+2 ; Util.decodeShortFrom(arr, pos) - case KEY_CHAR => newpos = pos+2 ; Util.decodeCharFrom(arr, pos) - case KEY_INT => newpos = pos+4 ; Util.decodeIntFrom(arr, pos) - case KEY_LONG => newpos = pos+8 ; Util.decodeLongFrom(arr, pos) - case KEY_BOOLEAN => newpos = pos+1 ; Util.decodeBooleanFrom(arr, pos) - case KEY_FLOAT => - val r = Util.decodeIntFrom(arr, pos) - newpos = pos+4 - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = Util.decodeLongFrom(arr, pos) - newpos = pos+8 - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => val r = Util.decodeStringFrom(arr, pos); newpos = r._2 ; r._1 - - case KEY_ARRAY_BYTE => val r = Util.decodeByteArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_SHORT => val r = Util.decodeShortArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_CHAR => val r = Util.decodeCharArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_INT => val r = Util.decodeIntArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_LONG => val r = Util.decodeLongArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_BOOLEAN => val r = Util.decodeBooleanArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_FLOAT => val r = Util.decodeFloatArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_DOUBLE => val r = Util.decodeDoubleArrayFrom(arr, pos); newpos = r._2 ; r._1 - } - - pos = newpos - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryPickleReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - val length = Util.decodeIntFrom(arr, pos) - pos += 4 - length - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } -} diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index 23410e1a4c..45471c56be 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -13,11 +13,9 @@ trait BinaryPBuilder extends PBuilder { class BinaryPickleFormat extends PickleFormat with Constants { type PickleType = BinaryPickle type OutputType = ArrayOutput[Byte] - //def createBuilder(): BinaryPBuilder = new BinaryPickleBuilder(this, null) def createBuilder(): BinaryPBuilder = createBuilder(new ByteArrayOutput) - //def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = new BinaryPickleBuilder(this, out) def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = createBuilder(new PickleArrayOutput(out)) - def createBuilder(out: BinaryOutput): BinaryPBuilder = new BinaryPickleBuilder2(this, out) + def createBuilder(out: BinaryOutput): BinaryPBuilder = new BinaryPickleBuilder(this, out) def createBuilder(out: java.nio.ByteBuffer): BinaryPBuilder = createBuilder(new ByteBufferOutput(out)) def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) } diff --git a/core/src/test/scala/pickling/run/binary-inputstream.scala b/core/src/test/scala/pickling/run/binary-inputstream.scala index b24d4f9082..1f661b8c1c 100644 --- a/core/src/test/scala/pickling/run/binary-inputstream.scala +++ b/core/src/test/scala/pickling/run/binary-inputstream.scala @@ -80,6 +80,8 @@ class BinaryInputStreamReaderTest extends FunSuite { } catch { case _: EndOfStreamException => /* expected */ + case _: java.io.EOFException => + /* expected */ } finally { assert(obj1.toString == readObj1.toString) assert(obj2.toString == readObj2.toString) From 25964a8e1a285dffe273957a7c23f490553c1c64 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Thu, 9 Oct 2014 18:09:23 -0400 Subject: [PATCH 06/11] tests for ByteBuffer pickling --- .../scala/pickling/binary/BinaryOutput.scala | 40 +++++++---- .../test/scala/pickling/run/bytebuffers.scala | 70 +++++++++++++++++++ .../pickling/run/combinator-pickleinto.scala | 4 +- 3 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 core/src/test/scala/pickling/run/bytebuffers.scala diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index b9e5115fc0..45db5a4cff 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -55,11 +55,12 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { def result = None - private var buffer = _buffer + var buffer = _buffer assert(buffer.order == ByteOrder.BIG_ENDIAN) private def growTo(newSize: Int) { - assert(newSize > 0) + //println("growing to " + newSize) + assert(newSize > 0) //can we overflow before running out of memory ? val newBuffer = if (buffer.isDirect) ByteBuffer.allocateDirect(newSize) else ByteBuffer.allocate(newSize) @@ -69,6 +70,10 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { buffer.position(0) newBuffer.put(buffer) buffer = newBuffer + //assert(newBuffer.position == pos) + //assert((0 until pos).forall(i => buffer.get(i) == newBuffer.get(i))) + //println("capapcity = " + buffer.capacity) + //println(buffer.toString) } @@ -90,6 +95,7 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { try { op(value) return + assert(false, "after return ?") } catch { case _: java.nio.BufferOverflowException => buffer.reset @@ -98,26 +104,32 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { } } - def putByte(value: Byte) = withReallocate[Byte](buffer.put, value) - def putChar(value: Char) = withReallocate(buffer.putChar, value) + @inline private def bb(i: Byte) = { buffer.put(i) } + def putByte(value: Byte) = withReallocate[Byte](bb, value) - def putShort(value: Short) = withReallocate(buffer.putShort, value) + @inline private def cc(i: Char) = { buffer.putChar(i) } + def putChar(value: Char) = withReallocate(cc, value) - def putInt(value: Int) = withReallocate(buffer.putInt, value) + @inline private def ss(i: Short) = { buffer.putShort(i) } + def putShort(value: Short) = withReallocate(ss, value) - def putLong(value: Long) = withReallocate(buffer.putLong, value) + @inline private def ii(i: Int) = { buffer.putInt(i) } + def putInt(value: Int) = withReallocate(ii, value) - def putFloat(value: Float) = withReallocate(buffer.putFloat, value) + @inline private def ll(i: Long) = { buffer.putLong(i) } + def putLong(value: Long) = withReallocate(ll, value) - def putDouble(value: Double) = withReallocate(buffer.putDouble, value) + @inline private def ff(i: Float) = { buffer.putFloat(i) } + def putFloat(value: Float) = withReallocate(ff, value) + @inline private def dd(i: Double) = { buffer.putDouble(i) } + def putDouble(value: Double) = withReallocate(dd, value) + + @inline private def pba(value: Array[Byte]) = { buffer.put(value) } override def putByteArray(value: Array[Byte]): Unit = { - def process(value: Array[Byte]) = { - putInt(value.size) - buffer.put(value) - } - withReallocate(process, value) + putInt(value.size) + withReallocate(pba, value) } } diff --git a/core/src/test/scala/pickling/run/bytebuffers.scala b/core/src/test/scala/pickling/run/bytebuffers.scala new file mode 100644 index 0000000000..d0af5ef2c4 --- /dev/null +++ b/core/src/test/scala/pickling/run/bytebuffers.scala @@ -0,0 +1,70 @@ +package scala.pickling.bytebuffers + +import scala.pickling._ +import binary._ +import java.nio.ByteBuffer +import org.scalacheck.{Properties, Prop, Gen} +import org.scalatest.FunSuite + +object Primitives extends Properties("bytebuffer primitive tests") { + + def roundTrip[T: SPickler: Unpickler: FastTypeTag](obj: T): Boolean = { + try { + val out = new ByteBufferOutput(ByteBuffer.allocate(512)) + val builder = pickleFormat.createBuilder(out) + obj.pickleInto(builder) + val buf = out.buffer + buf.rewind + val p = new BinaryInputPickle(new ByteBufferInput(buf)) + val up = p.unpickle[T] + obj == up + } catch { + case e: Throwable => + Console.err.println(e) + e.printStackTrace + throw e + } + } + + property("Int") = Prop forAll { (i: Int) => roundTrip[Int](i) } + property("Double") = Prop forAll { (d: Double) => roundTrip[Double](d) } + property("Long") = Prop forAll { (l: Long) => roundTrip[Long](l) } + property("Char") = Prop forAll { (c: Char) => roundTrip[Char](c) } + property("Float") = Prop forAll { (f: Float) => roundTrip[Float](f) } + property("Boolean") = Prop forAll { (b: Boolean) => roundTrip[Boolean](b) } + property("Short") = Prop forAll { (s: Short) => roundTrip[Short](s) } + property("Byte") = Prop forAll { (b: Byte) => roundTrip[Byte](b) } + property("String") = Prop forAll { (s: String) => roundTrip[String](s) } + property("(Int, String)") = Prop forAll { (p: (Int, String)) => roundTrip[(Int,String)](p) } + +} + +class ByteBufferTest extends FunSuite { + + test("lookahead") { + val buf = ByteBuffer.allocate(32) + val out = new ByteBufferOutput(buf) + out.putInt(0x12345678) + buf.rewind + val in = new ByteBufferInput(buf) + val b = in.getByte + in.setLookahead(b) + val res = in.getIntWithLookahead + res == 0x12345678 + } + + test("reallocation") { + val obj = (0 until 500).toArray + val buf = ByteBuffer.allocate(12) + val out = new ByteBufferOutput(buf) + val builder = pickleFormat.createBuilder(out) + obj.pickleInto(builder) + val buf2 = out.buffer + buf2.rewind + val p = new BinaryInputPickle(new ByteBufferInput(buf2)) + val up = p.unpickle[Array[Int]] + obj == up + } + +} + diff --git a/core/src/test/scala/pickling/run/combinator-pickleinto.scala b/core/src/test/scala/pickling/run/combinator-pickleinto.scala index 504eade55a..6f928b48cb 100644 --- a/core/src/test/scala/pickling/run/combinator-pickleinto.scala +++ b/core/src/test/scala/pickling/run/combinator-pickleinto.scala @@ -69,7 +69,9 @@ class CombinatorPickleIntoTest extends FunSuite { val bart = new Person(2) val pickle = bart.pickle - assert(pickle.value.mkString("[", ",", "]") === "[0,0,0,43,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,99,111,109,98,105,110,97,116,111,114,46,112,105,99,107,108,101,105,110,116,111,46,80,101,114,115,111,110,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]") + val expected0 = "[0,0,0,43,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,99,111,109,98,105,110,97,116,111,114,46,112,105,99,107,108,101,105,110,116,111,46,80,101,114,115,111,110,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]" + val expected = "[0,0,0,43,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,99,111,109,98,105,110,97,116,111,114,46,112,105,99,107,108,101,105,110,116,111,46,80,101,114,115,111,110,0,0,0,2]" + assert(pickle.value.mkString("[", ",", "]") === expected) val p = pickle.unpickle[Person] assert(p.toString === "Person(Bart, 45)") From 6601dcf61fdd35ca9e4fa83a41e806ba72379568 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Fri, 10 Oct 2014 17:28:22 -0400 Subject: [PATCH 07/11] storing array in little-endian --- .../scala/pickling/binary/BinaryInput.scala | 78 +++++++++------- .../scala/pickling/binary/BinaryOutput.scala | 91 +++++++++++-------- 2 files changed, 99 insertions(+), 70 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala index 155e3ed744..ad59fcfced 100644 --- a/core/src/main/scala/pickling/binary/BinaryInput.scala +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -4,10 +4,6 @@ import scala.reflect.ClassTag abstract class BinaryInput { - def getBoolean(): Boolean = { - getByte() != 0 - } - def getByte(): Byte def getChar(): Char @@ -21,30 +17,54 @@ abstract class BinaryInput { def getFloat(): Float def getDouble(): Double + + def getBytes(target: Array[Byte], len: Int): Unit + + //////////////////////// + // Derived operations // + //////////////////////// + + def getBoolean(): Boolean = { + getByte() != 0 + } def getString(): String = { val array = getByteArray new String(array, "UTF-8") } - - //generic method when the performance is not an issue - @inline private def getArray[T: ClassTag](get: () => T): Array[T] = { + + private val chunkSize = 128 + private val chunk = Array.ofDim[Byte](chunkSize) + + def getArrayByChunk[T <: AnyVal: ClassTag](offset: Long, eltSize: Int): Array[T] = { val size = getIntWithLookahead val array = Array.ofDim[T](size) - for(i <- 0 until size) { - array(i) = get() + var toCopy = size * eltSize + var destOffset = offset + while (toCopy > 0) { + val byteLen = math.min(chunkSize, toCopy) + getBytes(chunk, byteLen) + UnsafeMemory.unsafe.copyMemory(chunk, UnsafeMemory.byteArrayOffset, array, destOffset, byteLen) + toCopy -= byteLen + destOffset += byteLen } array } + + def getByteArray(): Array[Byte] = { + val size = getIntWithLookahead + val array = Array.ofDim[Byte](size) + getBytes(array, size) + array + } - def getBooleanArray(): Array[Boolean] = getArray(getBoolean) - def getByteArray(): Array[Byte] = getArray(getByte) - def getCharArray(): Array[Char] = getArray(getChar) - def getDoubleArray(): Array[Double] = getArray(getDouble) - def getFloatArray(): Array[Float] = getArray(getFloat) - def getIntArray(): Array[Int] = getArray(getInt) - def getLongArray(): Array[Long] = getArray(getLong) - def getShortArray(): Array[Short] = getArray(getShort) + def getBooleanArray(): Array[Boolean] = getArrayByChunk[Boolean](UnsafeMemory.booleanArrayOffset, 1) + def getCharArray(): Array[Char] = getArrayByChunk[Char](UnsafeMemory.charArrayOffset, 2) + def getShortArray(): Array[Short] = getArrayByChunk[Short](UnsafeMemory.shortArrayOffset, 2) + def getIntArray(): Array[Int] = getArrayByChunk[Int](UnsafeMemory.intArrayOffset, 4) + def getFloatArray(): Array[Float] = getArrayByChunk[Float](UnsafeMemory.floatArrayOffset, 4) + def getLongArray(): Array[Long] = getArrayByChunk[Long](UnsafeMemory.longArrayOffset, 8) + def getDoubleArray(): Array[Double] = getArrayByChunk[Double](UnsafeMemory.doubleArrayOffset, 8) protected var lookahead: Option[Byte] = None @@ -95,11 +115,8 @@ class ByteBufferInput(buffer: java.nio.ByteBuffer) extends BinaryInput { def getDouble() = buffer.getDouble - override def getByteArray(): Array[Byte] = { - val size = getIntWithLookahead - val array = Array.ofDim[Byte](size) - buffer.get(array) - array + def getBytes(target: Array[Byte], len: Int): Unit = { + buffer.get(target, 0, len) } } @@ -164,12 +181,9 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { java.lang.Double.longBitsToDouble(r) } - override def getByteArray(): Array[Byte] = { - val size = getIntWithLookahead - val array = Array.ofDim[Byte](size) - data.view(idx, idx+size).copyToArray(array, 0, size) - idx += size - array + def getBytes(target: Array[Byte], len: Int): Unit = { + UnsafeMemory.unsafe.copyMemory(data, UnsafeMemory.byteArrayOffset + idx, target, UnsafeMemory.byteArrayOffset, len) + idx += len } //TODO override array for faster copy @@ -186,13 +200,9 @@ class StreamInput(stream: java.io.InputStream) extends BinaryInput { def getFloat() = ds.readFloat() def getDouble() = ds.readDouble() - override def getByteArray(): Array[Byte] = { - val size = getIntWithLookahead - val array = Array.ofDim[Byte](size) - ds.readFully(array) - array + def getBytes(target: Array[Byte], len: Int): Unit = { + ds.readFully(target, 0, len) } - //TODO check endianness } diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index 45db5a4cff..b74eab753c 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -2,15 +2,10 @@ package scala.pickling.binary abstract class BinaryOutput { - def result: Option[Array[Byte]] + def result: Option[Array[Byte]] //TODO allow multiple outputs type (e.g. bytebuffer) def ensureCapacity(capacity: Int): Unit - def putBoolean(value: Boolean): Unit = { - if (value) putByte(1) - else putByte(0) - } - def putByte(value: Byte): Unit def putChar(value: Char): Unit @@ -25,26 +20,52 @@ abstract class BinaryOutput { def putDouble(value: Double): Unit + def putBytes(bytes: Array[Byte], len: Int): Unit + + //////////////////////// + // Derived operations // + //////////////////////// + + def putBoolean(value: Boolean): Unit = { + if (value) putByte(1) + else putByte(0) + } + def putString(value: String) { val bytes = value.getBytes("UTF-8") putByteArray(bytes) } - - //generic method when the performance is not an issue - @inline private def putArray[T](array: Array[T], put: T => Unit): Unit = { - putInt(array.size) - for(elt <- array) put(elt) + private val chunkSize = 128 + private val chunk = Array.ofDim[Byte](chunkSize) + + protected def putArrayByChunk[T <: AnyVal](arr: Array[T], offset: Long, eltSize: Int) { + val nbrElt = arr.length + putInt(nbrElt) + var srcOffset = offset //UnsafeMemory.byteArrayOffset + var toCopy = nbrElt * eltSize + while (toCopy > 0) { + val byteLen = math.min(chunkSize, toCopy) + UnsafeMemory.unsafe.copyMemory(arr, srcOffset, chunk, UnsafeMemory.byteArrayOffset, byteLen) + toCopy -= byteLen + srcOffset += byteLen + putBytes(chunk, byteLen) + } } - def putBooleanArray(value: Array[Boolean]): Unit = putArray(value, putBoolean) - def putByteArray(value: Array[Byte]): Unit = putArray(value, putByte) - def putCharArray(value: Array[Char]): Unit = putArray(value, putChar) - def putDoubleArray(value: Array[Double]): Unit = putArray(value, putDouble) - def putFloatArray(value: Array[Float]): Unit = putArray(value, putFloat) - def putIntArray(value: Array[Int]): Unit = putArray(value, putInt) - def putLongArray(value: Array[Long]): Unit = putArray(value, putLong) - def putShortArray(value: Array[Short]): Unit = putArray(value, putShort) + def putByteArray(value: Array[Byte]): Unit = { + val size = value.size + putInt(size) + putBytes(value, size) + } + + def putBooleanArray(value: Array[Boolean]): Unit = putArrayByChunk(value, UnsafeMemory.booleanArrayOffset, 1) + def putCharArray(value: Array[Char]): Unit = putArrayByChunk(value, UnsafeMemory.charArrayOffset, 2) + def putShortArray(value: Array[Short]): Unit = putArrayByChunk(value, UnsafeMemory.shortArrayOffset, 2) + def putIntArray(value: Array[Int]): Unit = putArrayByChunk(value, UnsafeMemory.intArrayOffset, 4) + def putFloatArray(value: Array[Float]): Unit = putArrayByChunk(value, UnsafeMemory.floatArrayOffset, 4) + def putLongArray(value: Array[Long]): Unit = putArrayByChunk(value, UnsafeMemory.longArrayOffset, 8) + def putDoubleArray(value: Array[Double]): Unit = putArrayByChunk(value, UnsafeMemory.doubleArrayOffset, 8) } @@ -104,7 +125,6 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { } } - @inline private def bb(i: Byte) = { buffer.put(i) } def putByte(value: Byte) = withReallocate[Byte](bb, value) @@ -125,11 +145,10 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { @inline private def dd(i: Double) = { buffer.putDouble(i) } def putDouble(value: Double) = withReallocate(dd, value) - - @inline private def pba(value: Array[Byte]) = { buffer.put(value) } - override def putByteArray(value: Array[Byte]): Unit = { - putInt(value.size) - withReallocate(pba, value) + + @inline private def pbs(value: Array[Byte])(len: Int) = { buffer.put(value, 0, len) } + def putBytes(value: Array[Byte], len: Int): Unit = { + withReallocate(pbs(value), len) } } @@ -185,13 +204,12 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { val longValue = java.lang.Double.doubleToRawLongBits(value) putLong(longValue) } - - override def putByteArray(value: Array[Byte]): Unit = { - putInt(value.length) - buffer.write(value) - } - //TODO override array + def putBytes(value: Array[Byte], len: Int): Unit = { + buffer.write(value, 0, len) + } + + //TODO override array to avoid intermediate copy } @@ -257,6 +275,11 @@ class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends Binary putInt(value.length) buffer.put(value) } + + def putBytes(value: Array[Byte], len: Int): Unit = { + val slice = value.slice(0, len) + buffer.put(slice) + } } @@ -271,10 +294,6 @@ class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { def putLong(value: Long) = ds.writeLong(value) def putFloat(value: Float) = ds.writeFloat(value) def putDouble(value: Double) = ds.writeDouble(value) - - override def putByteArray(value: Array[Byte]): Unit = { - putInt(value.length) - ds.write(value) - } + def putBytes(value: Array[Byte], len: Int) = ds.write(value, 0, len) } From 690b7a81da71a7484568810dcef8f8f523487412 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Tue, 14 Oct 2014 19:22:24 -0400 Subject: [PATCH 08/11] changes in IO --- .../scala/pickling/binary/BinaryInput.scala | 18 +++++++++++---- .../scala/pickling/binary/BinaryOutput.scala | 23 +++++++++++++++---- .../scala/pickling/binary/BinaryPickle.scala | 15 +++++++++--- .../pickling/binary/BinaryPickleFormat.scala | 2 +- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryInput.scala b/core/src/main/scala/pickling/binary/BinaryInput.scala index ad59fcfced..68fc8f040a 100644 --- a/core/src/main/scala/pickling/binary/BinaryInput.scala +++ b/core/src/main/scala/pickling/binary/BinaryInput.scala @@ -33,10 +33,10 @@ abstract class BinaryInput { new String(array, "UTF-8") } - private val chunkSize = 128 + private val chunkSize = 1024 private val chunk = Array.ofDim[Byte](chunkSize) - def getArrayByChunk[T <: AnyVal: ClassTag](offset: Long, eltSize: Int): Array[T] = { + protected def getArrayByChunk[T <: AnyVal: ClassTag](offset: Long, eltSize: Int): Array[T] = { val size = getIntWithLookahead val array = Array.ofDim[T](size) var toCopy = size * eltSize @@ -182,12 +182,20 @@ class ByteArrayInput(data: Array[Byte]) extends BinaryInput { } def getBytes(target: Array[Byte], len: Int): Unit = { - UnsafeMemory.unsafe.copyMemory(data, UnsafeMemory.byteArrayOffset + idx, target, UnsafeMemory.byteArrayOffset, len) + UnsafeMemory.unsafe.copyMemory(data, UnsafeMemory.byteArrayOffset + idx, target, UnsafeMemory.byteArrayOffset, len) idx += len } - //TODO override array for faster copy - + //override array for faster copy (get rid of ckunk) + override protected def getArrayByChunk[T <: AnyVal: ClassTag](offset: Long, eltSize: Int): Array[T] = { + val size = getIntWithLookahead + val array = Array.ofDim[T](size) + var toCopy = size * eltSize + UnsafeMemory.unsafe.copyMemory(data, UnsafeMemory.byteArrayOffset + idx, array, offset, toCopy) + idx += toCopy + array + } + } class StreamInput(stream: java.io.InputStream) extends BinaryInput { diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index b74eab753c..8d730b51c3 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -36,8 +36,8 @@ abstract class BinaryOutput { putByteArray(bytes) } - private val chunkSize = 128 - private val chunk = Array.ofDim[Byte](chunkSize) + protected val chunkSize = 1024 + protected val chunk = Array.ofDim[Byte](chunkSize) protected def putArrayByChunk[T <: AnyVal](arr: Array[T], offset: Long, eltSize: Int) { val nbrElt = arr.length @@ -146,9 +146,24 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { @inline private def dd(i: Double) = { buffer.putDouble(i) } def putDouble(value: Double) = withReallocate(dd, value) - @inline private def pbs(value: Array[Byte])(len: Int) = { buffer.put(value, 0, len) } def putBytes(value: Array[Byte], len: Int): Unit = { - withReallocate(pbs(value), len) + ensureCapacity(len) + buffer.put(value, 0, len) + } + + override protected def putArrayByChunk[T <: AnyVal](arr: Array[T], offset: Long, eltSize: Int) { + val nbrElt = arr.length + putInt(nbrElt) + var srcOffset = offset //UnsafeMemory.byteArrayOffset + var toCopy = nbrElt * eltSize + ensureCapacity(toCopy) + while (toCopy > 0) { + val byteLen = math.min(chunkSize, toCopy) + UnsafeMemory.unsafe.copyMemory(arr, srcOffset, chunk, UnsafeMemory.byteArrayOffset, byteLen) + toCopy -= byteLen + srcOffset += byteLen + putBytes(chunk, byteLen) + } } } diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index 0252aaf1d1..7b68734733 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -51,12 +51,21 @@ object BinaryPickle { new BinaryPickleArray(a) } -class BinaryPickleBuilder(format: BinaryPickleFormat, output: BinaryOutput) extends BinaryPBuilder with PickleTools { +class BinaryPickleBuilder(format: BinaryPickleFormat, out: BinaryOutput) extends BinaryPBuilder with PickleTools { import format._ - + private var output: BinaryOutput = out + + @inline private[this] def mkOutput(knownSize: Int): Unit = { + if (output == null) + output = if (knownSize != -1) new ByteArrayOutput(knownSize) + else new ByteArrayOutput + else + output.ensureCapacity(knownSize) + } + @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => - output.ensureCapacity(hints.knownSize) + mkOutput(hints.knownSize) if (picklee == null) { output.putByte( NULL_TAG) diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index 45471c56be..d2147903eb 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -13,7 +13,7 @@ trait BinaryPBuilder extends PBuilder { class BinaryPickleFormat extends PickleFormat with Constants { type PickleType = BinaryPickle type OutputType = ArrayOutput[Byte] - def createBuilder(): BinaryPBuilder = createBuilder(new ByteArrayOutput) + def createBuilder(): BinaryPBuilder = new BinaryPickleBuilder(this, null) def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = createBuilder(new PickleArrayOutput(out)) def createBuilder(out: BinaryOutput): BinaryPBuilder = new BinaryPickleBuilder(this, out) def createBuilder(out: java.nio.ByteBuffer): BinaryPBuilder = createBuilder(new ByteBufferOutput(out)) From 8d8bef7784389c2aa64ec3c068be39482fb026fe Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Thu, 16 Oct 2014 11:55:04 -0400 Subject: [PATCH 09/11] experimenting with different outputs --- .../scala/pickling/binary/BinaryOutput.scala | 175 +++++++++++++++++- .../scala/pickling/binary/BinaryPickle.scala | 6 +- 2 files changed, 173 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index 8d730b51c3..d92a4007ea 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -168,15 +168,15 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { } -class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { +class ByteArrayOutputS(buffer: java.io.ByteArrayOutputStream) extends BinaryOutput { - val buffer = new java.io.ByteArrayOutputStream(initialCapacity) + def this(initialCapacity: Int) = this(new java.io.ByteArrayOutputStream(initialCapacity)) + + def this() = this(1024) def result = Some(buffer.toByteArray) - def ensureCapacity(capacity: Int) { - //TODO ignore for the moment, not sure if it is worth creating a new buffer and copying the old one - } + def ensureCapacity(capacity: Int) { } def putByte(value: Byte) { buffer.write(value) @@ -224,7 +224,170 @@ class ByteArrayOutput(initialCapacity: Int = 1024) extends BinaryOutput { buffer.write(value, 0, len) } - //TODO override array to avoid intermediate copy +} + +//TODO as a pool rather than a single array +//TODO that might be dangerous in terms of security (exfiltrate data through the preAlloc array) +object ByteArrayOutput { + + private val lock = new java.util.concurrent.locks.ReentrantLock() + private var preAlloc = Array.ofDim[Byte](64 * 1024 * 1024) // 64 MB + + def get = { + lock.lock + try { + val p = preAlloc + preAlloc = null + p + } finally { + lock.unlock + } + } + + def set(p: Array[Byte]) = { + lock.lock + try { + preAlloc = p + } finally { + lock.unlock + } + } + +} + +class ByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOutput { + + override protected val chunkSize = 1024 * 1024 + override protected val chunk = null + private val allowedWaste = 512 + + private var pos = 0 //current position in the head + private var head: Array[Byte] = null + private var preA: Array[Byte] = null + private var chunks = List[(Int, Array[Byte])]() + private var stored = 0 //current size + + def init() { + val h = ByteArrayOutput.get + if (h != null && h.size >= initialCapacity) { + head = h + preA = h + } else { + ByteArrayOutput.set(h) + head = new Array[Byte](initialCapacity) + } + } + + init() + + def result = { + val size = pos + stored + val toCopy = ((pos -> head) :: chunks).reverse + var idx = 0 + val target = Array.ofDim[Byte](size) + val off = UnsafeMemory.byteArrayOffset + for ( (size, arr) <- toCopy ) { + UnsafeMemory.unsafe.copyMemory(arr, off, target, off + idx, size) + idx += size + } + //release resources + if (preA != null) { + ByteArrayOutput.set(preA) + } + head = null + chunks = Nil + // + Some(target) + } + + def ensureCapacity(capacity: Int) { + val avail = head.size - pos + val need = capacity - avail + if (need > 0) { + if (pos == 0) { + head = Array.ofDim[Byte](capacity) + } else if (avail > allowedWaste && head.size != chunkSize) { + val newHead = Array.ofDim[Byte](math.max(chunkSize, need)) + val off = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(head, off, newHead, off, pos) + head = newHead + } else { + chunks = (pos -> head) :: chunks + head = Array.ofDim[Byte](math.max(chunkSize, need)) + stored += pos + pos = 0 + } + } + } + + def putByte(value: Byte) { + ensureCapacity(1) + head(pos) = value + pos += 1 + } + + def putChar(value: Char) { + ensureCapacity(2) + head(pos) = (value >>> 8 & 0xff).asInstanceOf[Byte] + head(pos+1) = (value & 0xff).asInstanceOf[Byte] + pos += 2 + } + + def putShort(value: Short) { + ensureCapacity(2) + head(pos) = (value >>> 8 & 0xff).asInstanceOf[Byte] + head(pos+1) = (value & 0xff).asInstanceOf[Byte] + pos += 2 + } + + def putInt(value: Int) { + ensureCapacity(4) + head(pos) = (value >>> 24 & 0xff).asInstanceOf[Byte] + head(pos+1) = (value >>> 16 & 0xff).asInstanceOf[Byte] + head(pos+2) = (value >>> 8 & 0xff).asInstanceOf[Byte] + head(pos+3) = (value & 0xff).asInstanceOf[Byte] + pos += 4 + } + + def putLong(value: Long) { + ensureCapacity(8) + head(pos) = (value >>> 56 & 0xff).asInstanceOf[Byte] + head(pos+1) = (value >>> 48 & 0xff).asInstanceOf[Byte] + head(pos+2) = (value >>> 40 & 0xff).asInstanceOf[Byte] + head(pos+3) = (value >>> 32 & 0xff).asInstanceOf[Byte] + head(pos+4) = (value >>> 24 & 0xff).asInstanceOf[Byte] + head(pos+5) = (value >>> 16 & 0xff).asInstanceOf[Byte] + head(pos+6) = (value >>> 8 & 0xff).asInstanceOf[Byte] + head(pos+7) = (value & 0xff).asInstanceOf[Byte] + pos += 8 + } + + def putFloat(value: Float) { + val intValue = java.lang.Float.floatToRawIntBits(value) + putInt(intValue) + } + + def putDouble(value: Double) { + val longValue = java.lang.Double.doubleToRawLongBits(value) + putLong(longValue) + } + + def putBytes(value: Array[Byte], len: Int): Unit = { + ensureCapacity(len) + val off = UnsafeMemory.byteArrayOffset + UnsafeMemory.unsafe.copyMemory(value, off, head, off + pos, len) + pos += len + } + + //a single chunk + override protected def putArrayByChunk[T <: AnyVal](arr: Array[T], offset: Long, eltSize: Int) { + val nbrElt = arr.length + var byteLen = nbrElt * eltSize + ensureCapacity(byteLen+4) + putInt(nbrElt) + UnsafeMemory.unsafe.copyMemory(arr, offset, head, UnsafeMemory.byteArrayOffset + pos, byteLen) + pos += byteLen + } } diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index 7b68734733..e96751b626 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -58,8 +58,10 @@ class BinaryPickleBuilder(format: BinaryPickleFormat, out: BinaryOutput) extends @inline private[this] def mkOutput(knownSize: Int): Unit = { if (output == null) - output = if (knownSize != -1) new ByteArrayOutput(knownSize) - else new ByteArrayOutput + output = if (knownSize != -1) new ByteArrayOutputS(knownSize) + else new ByteArrayOutputS + //output = if (knownSize != -1) new ByteArrayOutput(knownSize) + // else new ByteArrayOutput else output.ensureCapacity(knownSize) } From 77d2fd8df13003917978db9489b2b628966c7c10 Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Thu, 16 Oct 2014 12:29:20 -0400 Subject: [PATCH 10/11] refactoring code --- .../scala/pickling/binary/BinaryOutput.scala | 82 ++++--------------- 1 file changed, 15 insertions(+), 67 deletions(-) diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index d92a4007ea..5260209854 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -168,62 +168,24 @@ class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { } -class ByteArrayOutputS(buffer: java.io.ByteArrayOutputStream) extends BinaryOutput { +class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { + val ds = new java.io.DataOutputStream(stream) + def result: Option[Array[Byte]] = None + def ensureCapacity(capacity: Int) { } + def putByte(value: Byte) = ds.writeByte(value) + def putChar(value: Char) = ds.writeChar(value) + def putShort(value: Short) = ds.writeShort(value) + def putInt(value: Int) = ds.writeInt(value) + def putLong(value: Long) = ds.writeLong(value) + def putFloat(value: Float) = ds.writeFloat(value) + def putDouble(value: Double) = ds.writeDouble(value) + def putBytes(value: Array[Byte], len: Int) = ds.write(value, 0, len) +} +class ByteArrayOutputS(buffer: java.io.ByteArrayOutputStream) extends StreamOutput(buffer) { def this(initialCapacity: Int) = this(new java.io.ByteArrayOutputStream(initialCapacity)) - def this() = this(1024) - - def result = Some(buffer.toByteArray) - - def ensureCapacity(capacity: Int) { } - - def putByte(value: Byte) { - buffer.write(value) - } - - def putChar(value: Char) { - buffer.write(value >>> 8 & 0xff) - buffer.write(value & 0xff) - } - - def putShort(value: Short) { - buffer.write(value >>> 8 & 0xff) - buffer.write(value & 0xff) - } - - def putInt(value: Int) { - buffer.write(value >>> 24) - buffer.write(value >>> 16 & 0xff) - buffer.write(value >>> 8 & 0xff) - buffer.write(value & 0xff) - } - - def putLong(value: Long) { - buffer.write((value >>> 56 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 48 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 40 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 32 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 24 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 16 & 0xff).asInstanceOf[Int]) - buffer.write((value >>> 8 & 0xff).asInstanceOf[Int]) - buffer.write((value & 0xff).asInstanceOf[Int]) - } - - def putFloat(value: Float) { - val intValue = java.lang.Float.floatToRawIntBits(value) - putInt(intValue) - } - - def putDouble(value: Double) { - val longValue = java.lang.Double.doubleToRawLongBits(value) - putLong(longValue) - } - - def putBytes(value: Array[Byte], len: Int): Unit = { - buffer.write(value, 0, len) - } - + override def result = Some(buffer.toByteArray) } //TODO as a pool rather than a single array @@ -461,17 +423,3 @@ class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends Binary } -class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { - val ds = new java.io.DataOutputStream(stream) - def result = None - def ensureCapacity(capacity: Int) { } - def putByte(value: Byte) = ds.writeByte(value) - def putChar(value: Char) = ds.writeChar(value) - def putShort(value: Short) = ds.writeShort(value) - def putInt(value: Int) = ds.writeInt(value) - def putLong(value: Long) = ds.writeLong(value) - def putFloat(value: Float) = ds.writeFloat(value) - def putDouble(value: Double) = ds.writeDouble(value) - def putBytes(value: Array[Byte], len: Int) = ds.write(value, 0, len) - -} From 0464d32d4034eb8092767b981a03f309f44b83ed Mon Sep 17 00:00:00 2001 From: Damien Zufferey Date: Thu, 16 Oct 2014 15:32:16 -0400 Subject: [PATCH 11/11] clean up, rm bytebuffer realloc, pickleTo[S] using ByteArrayOutputStream as default removing fastbinary and related tests removing ArrayOutput[Byte] (superseded by BinaryOutput) removing most of Utils bytebuffer throw overflow exception instead of reallocate pickleTo has a "free" input type --- core/src/main/scala/pickling/Compat.scala | 2 +- core/src/main/scala/pickling/Output.scala | 70 --- .../scala/pickling/binary/BinaryOutput.scala | 195 +------ .../scala/pickling/binary/BinaryPickle.scala | 12 +- .../pickling/binary/BinaryPickleFormat.scala | 5 +- .../src/main/scala/pickling/binary/Util.scala | 368 +----------- .../fastbinary/BinaryPickleFormat.scala | 539 ------------------ .../fastbinary/ByteArrayBuilder.scala | 139 ----- .../scala/pickling/fastbinary/Output.scala | 81 --- .../scala/pickling/fastbinary/package.scala | 9 - core/src/main/scala/pickling/package.scala | 2 +- .../run/binary-inputstream-output.scala | 2 +- .../pickling/run/binary-inputstream.scala | 2 +- .../pickling/run/binary-outputstream.scala | 3 +- .../test/scala/pickling/run/binary-util.scala | 137 ----- .../pickling/run/byte-array-builder.scala | 123 ---- .../test/scala/pickling/run/bytebuffers.scala | 17 +- .../pickling/run/fast-array-output.scala | 23 - .../test/scala/pickling/run/fastbinary1.scala | 16 - .../scala/pickling/run/share-binary.scala | 2 +- 20 files changed, 43 insertions(+), 1704 deletions(-) delete mode 100644 core/src/main/scala/pickling/fastbinary/BinaryPickleFormat.scala delete mode 100644 core/src/main/scala/pickling/fastbinary/ByteArrayBuilder.scala delete mode 100644 core/src/main/scala/pickling/fastbinary/Output.scala delete mode 100644 core/src/main/scala/pickling/fastbinary/package.scala delete mode 100644 core/src/test/scala/pickling/run/binary-util.scala delete mode 100644 core/src/test/scala/pickling/run/byte-array-builder.scala delete mode 100644 core/src/test/scala/pickling/run/fast-array-output.scala delete mode 100644 core/src/test/scala/pickling/run/fastbinary1.scala diff --git a/core/src/main/scala/pickling/Compat.scala b/core/src/main/scala/pickling/Compat.scala index 779039ad75..3ac7f66a7d 100644 --- a/core/src/main/scala/pickling/Compat.scala +++ b/core/src/main/scala/pickling/Compat.scala @@ -37,7 +37,7 @@ object Compat { c.Expr[Unit](bundle.pickleInto[T](builder.tree)) } - def PickleMacros_pickleTo[T: c.WeakTypeTag](c: Context)(output: c.Expr[Output[_]])(format: c.Expr[PickleFormat]): c.Expr[Unit] = { + def PickleMacros_pickleTo[T: c.WeakTypeTag, S](c: Context)(output: c.Expr[S])(format: c.Expr[PickleFormat]): c.Expr[Unit] = { val c0: c.type = c val bundle = new { val c: c0.type = c0 } with PickleMacros c.Expr[Unit](bundle.pickleTo[T](output.tree)(format.tree)) diff --git a/core/src/main/scala/pickling/Output.scala b/core/src/main/scala/pickling/Output.scala index 08dc77aae0..657a7e5399 100644 --- a/core/src/main/scala/pickling/Output.scala +++ b/core/src/main/scala/pickling/Output.scala @@ -12,76 +12,6 @@ trait Output[T] { } -class OutputStreamOutput(out: OutputStream) extends ArrayOutput[Byte] { - def result(): Array[Byte] = - null - - def +=(obj: Byte) = - out.write(obj.asInstanceOf[Int]) - - def put(obj: Array[Byte]): this.type = { - out.write(obj) - this - } -} - -// Array output with a few more methods for performance -abstract class ArrayOutput[T: ClassTag] extends Output[Array[T]] { - // Put a single T - def +=(obj: T): Unit - // Allocate a new array. - def target(len: Int): (Array[T], Int) = - (Array.ofDim[T](len), 0) - // Flush the allocated array by target(). - def flush(arr: Array[T]): Unit = - this.put(arr) -} - - -class ByteArrayBufferOutput extends ArrayOutput[Byte] { - - private val buf = - ArrayBuffer[Byte]() - - def result(): Array[Byte] = - buf.toArray - - def +=(obj: Byte) = - buf += obj - - def put(obj: Array[Byte]): this.type = { - buf ++= obj - this - } -} - -class ByteArrayOutput(len: Int) extends ArrayOutput[Byte] { - - private var pos = 0 - private val arr = Array.ofDim[Byte](len) - - def result(): Array[Byte] = - arr - - def +=(obj: Byte) = { - arr(pos) = obj - pos = pos + 1 - } - - def put(obj: Array[Byte]): this.type = { - // target() should be used to avoid double copy - throw new java.lang.IllegalStateException - } - - override def target(len: Int) = { - val oldpos = pos - pos = pos + len - (arr, oldpos) - } - - override def flush(arr: Array[Byte]) = { /*noop*/ } -} - class StringOutput extends Output[String] { private val buf = diff --git a/core/src/main/scala/pickling/binary/BinaryOutput.scala b/core/src/main/scala/pickling/binary/BinaryOutput.scala index 5260209854..e2baa6716f 100644 --- a/core/src/main/scala/pickling/binary/BinaryOutput.scala +++ b/core/src/main/scala/pickling/binary/BinaryOutput.scala @@ -2,7 +2,7 @@ package scala.pickling.binary abstract class BinaryOutput { - def result: Option[Array[Byte]] //TODO allow multiple outputs type (e.g. bytebuffer) + def result: Array[Byte] //can be null def ensureCapacity(capacity: Int): Unit @@ -69,108 +69,33 @@ abstract class BinaryOutput { } -class ByteBufferOutput(_buffer: java.nio.ByteBuffer) extends BinaryOutput { +class ByteBufferOutput(buffer: java.nio.ByteBuffer) extends BinaryOutput { import java.nio.ByteOrder import java.nio.ByteBuffer - - def result = None - var buffer = _buffer assert(buffer.order == ByteOrder.BIG_ENDIAN) - - private def growTo(newSize: Int) { - //println("growing to " + newSize) - assert(newSize > 0) //can we overflow before running out of memory ? - val newBuffer = - if (buffer.isDirect) ByteBuffer.allocateDirect(newSize) - else ByteBuffer.allocate(newSize) - //copy the content - val pos = buffer.position - buffer.limit(pos) - buffer.position(0) - newBuffer.put(buffer) - buffer = newBuffer - //assert(newBuffer.position == pos) - //assert((0 until pos).forall(i => buffer.get(i) == newBuffer.get(i))) - //println("capapcity = " + buffer.capacity) - //println(buffer.toString) - } - - - private def grow { - val newSize = 2*buffer.capacity - growTo(newSize) - } - - def ensureCapacity(capacity: Int) { - val left = buffer.remaining - if (left < capacity) { - growTo(buffer.capacity + (capacity - left)) - } - } - - @inline private def withReallocate[A](op: A => ByteBuffer, value: A) { - while(true) { - buffer.mark - try { - op(value) - return - assert(false, "after return ?") - } catch { - case _: java.nio.BufferOverflowException => - buffer.reset - grow - } - } - } - - @inline private def bb(i: Byte) = { buffer.put(i) } - def putByte(value: Byte) = withReallocate[Byte](bb, value) - - @inline private def cc(i: Char) = { buffer.putChar(i) } - def putChar(value: Char) = withReallocate(cc, value) - - @inline private def ss(i: Short) = { buffer.putShort(i) } - def putShort(value: Short) = withReallocate(ss, value) - - @inline private def ii(i: Int) = { buffer.putInt(i) } - def putInt(value: Int) = withReallocate(ii, value) - - @inline private def ll(i: Long) = { buffer.putLong(i) } - def putLong(value: Long) = withReallocate(ll, value) - - @inline private def ff(i: Float) = { buffer.putFloat(i) } - def putFloat(value: Float) = withReallocate(ff, value) - - @inline private def dd(i: Double) = { buffer.putDouble(i) } - def putDouble(value: Double) = withReallocate(dd, value) - - def putBytes(value: Array[Byte], len: Int): Unit = { - ensureCapacity(len) - buffer.put(value, 0, len) - } - override protected def putArrayByChunk[T <: AnyVal](arr: Array[T], offset: Long, eltSize: Int) { - val nbrElt = arr.length - putInt(nbrElt) - var srcOffset = offset //UnsafeMemory.byteArrayOffset - var toCopy = nbrElt * eltSize - ensureCapacity(toCopy) - while (toCopy > 0) { - val byteLen = math.min(chunkSize, toCopy) - UnsafeMemory.unsafe.copyMemory(arr, srcOffset, chunk, UnsafeMemory.byteArrayOffset, byteLen) - toCopy -= byteLen - srcOffset += byteLen - putBytes(chunk, byteLen) - } - } + def result: Array[Byte] = null + def ensureCapacity(capacity: Int) { + if (buffer.remaining < capacity) + throw new java.nio.BufferOverflowException() + } + + def putByte(value: Byte) = buffer.put(value) + def putChar(value: Char) = buffer.putChar(value) + def putShort(value: Short) = buffer.putShort(value) + def putInt(value: Int) = buffer.putInt(value) + def putLong(value: Long) = buffer.putLong(value) + def putFloat(value: Float) = buffer.putFloat(value) + def putDouble(value: Double) = buffer.putDouble(value) + def putBytes(value: Array[Byte], len: Int) = buffer.put(value, 0, len) } class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { val ds = new java.io.DataOutputStream(stream) - def result: Option[Array[Byte]] = None + def result: Array[Byte] = null def ensureCapacity(capacity: Int) { } def putByte(value: Byte) = ds.writeByte(value) def putChar(value: Char) = ds.writeChar(value) @@ -182,15 +107,15 @@ class StreamOutput(stream: java.io.OutputStream) extends BinaryOutput { def putBytes(value: Array[Byte], len: Int) = ds.write(value, 0, len) } -class ByteArrayOutputS(buffer: java.io.ByteArrayOutputStream) extends StreamOutput(buffer) { +class ByteArrayOutput(buffer: java.io.ByteArrayOutputStream) extends StreamOutput(buffer) { def this(initialCapacity: Int) = this(new java.io.ByteArrayOutputStream(initialCapacity)) def this() = this(1024) - override def result = Some(buffer.toByteArray) + override def result = buffer.toByteArray } //TODO as a pool rather than a single array //TODO that might be dangerous in terms of security (exfiltrate data through the preAlloc array) -object ByteArrayOutput { +object FastByteArrayOutput { private val lock = new java.util.concurrent.locks.ReentrantLock() private var preAlloc = Array.ofDim[Byte](64 * 1024 * 1024) // 64 MB @@ -217,7 +142,7 @@ object ByteArrayOutput { } -class ByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOutput { +class FastByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOutput { override protected val chunkSize = 1024 * 1024 override protected val chunk = null @@ -230,12 +155,12 @@ class ByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOut private var stored = 0 //current size def init() { - val h = ByteArrayOutput.get + val h = FastByteArrayOutput.get if (h != null && h.size >= initialCapacity) { head = h preA = h } else { - ByteArrayOutput.set(h) + FastByteArrayOutput.set(h) head = new Array[Byte](initialCapacity) } } @@ -254,12 +179,12 @@ class ByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOut } //release resources if (preA != null) { - ByteArrayOutput.set(preA) + FastByteArrayOutput.set(preA) } head = null chunks = Nil // - Some(target) + target } def ensureCapacity(capacity: Int) { @@ -353,73 +278,3 @@ class ByteArrayOutput(initialCapacity: Int = 10 * 1024 * 1024) extends BinaryOut } -class PickleArrayOutput(buffer: scala.pickling.ArrayOutput[Byte]) extends BinaryOutput { - - def result = { - val res = buffer.result - if (res == null) None - else Some(res) - } - - def ensureCapacity(capacity: Int) { - //TODO ignore for the moment, not sure if it is worth creating a new buffer and copying the old one - } - - @inline private def write(i: Int) { - buffer += i.asInstanceOf[Byte] - } - - def putByte(value: Byte) { - write(value) - } - - def putChar(value: Char) { - write(value >>> 8 & 0xff) - write(value & 0xff) - } - - def putShort(value: Short) { - write(value >>> 8 & 0xff) - write(value & 0xff) - } - - def putInt(value: Int) { - write(value >>> 24) - write(value >>> 16 & 0xff) - write(value >>> 8 & 0xff) - write(value & 0xff) - } - - def putLong(value: Long) { - write((value >>> 56 & 0xff).asInstanceOf[Int]) - write((value >>> 48 & 0xff).asInstanceOf[Int]) - write((value >>> 40 & 0xff).asInstanceOf[Int]) - write((value >>> 32 & 0xff).asInstanceOf[Int]) - write((value >>> 24 & 0xff).asInstanceOf[Int]) - write((value >>> 16 & 0xff).asInstanceOf[Int]) - write((value >>> 8 & 0xff).asInstanceOf[Int]) - write((value & 0xff).asInstanceOf[Int]) - } - - def putFloat(value: Float) { - val intValue = java.lang.Float.floatToRawIntBits(value) - putInt(intValue) - } - - def putDouble(value: Double) { - val longValue = java.lang.Double.doubleToRawLongBits(value) - putLong(longValue) - } - - override def putByteArray(value: Array[Byte]): Unit = { - putInt(value.length) - buffer.put(value) - } - - def putBytes(value: Array[Byte], len: Int): Unit = { - val slice = value.slice(0, len) - buffer.put(slice) - } - -} - diff --git a/core/src/main/scala/pickling/binary/BinaryPickle.scala b/core/src/main/scala/pickling/binary/BinaryPickle.scala index e96751b626..e9466137ed 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickle.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickle.scala @@ -31,7 +31,6 @@ case class BinaryPickleStream(input: InputStream) extends BinaryPickle { def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = new BinaryPickleReader(new StreamInput(input), mirror, format) - //new BinaryInputStreamReader(input, mirror, format) /* Do not override def toString to avoid traversing the input stream. */ } @@ -58,10 +57,8 @@ class BinaryPickleBuilder(format: BinaryPickleFormat, out: BinaryOutput) extends @inline private[this] def mkOutput(knownSize: Int): Unit = { if (output == null) - output = if (knownSize != -1) new ByteArrayOutputS(knownSize) - else new ByteArrayOutputS - //output = if (knownSize != -1) new ByteArrayOutput(knownSize) - // else new ByteArrayOutput + output = if (knownSize != -1) new ByteArrayOutput(knownSize) + else new ByteArrayOutput else output.ensureCapacity(knownSize) } @@ -156,10 +153,7 @@ class BinaryPickleBuilder(format: BinaryPickleFormat, out: BinaryOutput) extends } @inline def result() = { - output.result match { - case Some(b) => BinaryPickle(b) - case None => BinaryPickle(null) //TODO is that safe ? - } + BinaryPickle(output.result) } } diff --git a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala index d2147903eb..39fcbad68d 100644 --- a/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala +++ b/core/src/main/scala/pickling/binary/BinaryPickleFormat.scala @@ -12,11 +12,12 @@ trait BinaryPBuilder extends PBuilder { class BinaryPickleFormat extends PickleFormat with Constants { type PickleType = BinaryPickle - type OutputType = ArrayOutput[Byte] + type OutputType = BinaryOutput def createBuilder(): BinaryPBuilder = new BinaryPickleBuilder(this, null) - def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = createBuilder(new PickleArrayOutput(out)) def createBuilder(out: BinaryOutput): BinaryPBuilder = new BinaryPickleBuilder(this, out) + //def createBuilder(out: ArrayOutput[Byte]): BinaryPBuilder = createBuilder(new PickleArrayOutput(out)) def createBuilder(out: java.nio.ByteBuffer): BinaryPBuilder = createBuilder(new ByteBufferOutput(out)) + def createBuilder(out: java.io.OutputStream): BinaryPBuilder = createBuilder(new StreamOutput(out)) def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) } diff --git a/core/src/main/scala/pickling/binary/Util.scala b/core/src/main/scala/pickling/binary/Util.scala index 10c00e70f6..b3970ae2a5 100644 --- a/core/src/main/scala/pickling/binary/Util.scala +++ b/core/src/main/scala/pickling/binary/Util.scala @@ -1,381 +1,21 @@ package scala.pickling.binary -import scala.pickling._ -import collection.mutable.Buffer -import UnsafeMemory._ - -object Util { - - val SizeOfByte = 1 - val SizeOfShort = 2 - val SizeOfInt = 4 - val SizeOfLong = 8 - val SizeOfFloat = 4 - val SizeOfDouble = 8 - val SizeOfChar = 2 - val SizeOfBoolean = 1 - - // Decoding functions - - def decodeShortFrom(arr: Array[Byte], i: Int): Short = { - val fst = ((arr(i) << 8) & 0xFFFF).toShort - val snd = (arr(i+1) & 0x00FF).toShort - (fst | snd).toShort - } - - def decodeIntFrom(arr: Array[Byte], i: Int): Int = { - val fst = (arr(i) << 24).toInt - val snd = ((arr(i+1) << 16) & 0x00FFFFFF).toInt - val thrd = ((arr(i+2) << 8) & 0x0000FFFF).toInt - val frth = (arr(i+3) & 0x000000FF).toInt - fst | snd | thrd | frth - } - - def decodeLongFrom(arr: Array[Byte], i: Int): Long = { - val elem1 = ((arr(i).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong - val elem2 = ((arr(i+1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong - val elem3 = ((arr(i+2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong - val elem4 = ((arr(i+3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong - val elem5 = ((arr(i+4).toLong << 24) & 0x00000000FFFFFFFFL).toLong - val elem6 = ((arr(i+5).toLong << 16) & 0x0000000000FFFFFFL).toLong - val elem7 = ((arr(i+6).toLong << 8) & 0x000000000000FFFFL).toLong - val elem8 = (arr(i+7).toLong & 0x00000000000000FFL).toLong - elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 - } - - def decodeCharFrom(arr: Array[Byte], i: Int): Char = { - val fst = ((arr(i) << 8) & 0xFFFF).toChar - val snd = (arr(i+1) & 0x00FF).toChar - (fst | snd).toChar - } - - def decodeStringFrom(arr: Array[Byte], i: Int): (String, Int) = { - val len = decodeIntFrom(arr, i) - val bytes = Array.ofDim[Byte](len) - Array.copy(arr, i + SizeOfInt, bytes, 0, len) - (new String(bytes, "UTF-8"), i + SizeOfInt + len) - } - - def decodeBooleanFrom(arr: Array[Byte], i: Int): Boolean = { - arr(i) != 0 - } - - def decodeByteArray(arr: Array[Byte], offset: Int, len: Int): Array[Byte] = { - val newArr = Array.ofDim[Byte](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfByte) - newArr - } - - def decodeByteArrayFrom(arr: Array[Byte], i: Int): (Array[Byte], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeByteArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfByte) - } - - def decodeShortArray(arr: Array[Byte], offset: Int, len: Int): Array[Short] = { - val newArr = Array.ofDim[Short](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.shortArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfShort) - newArr - } - - def decodeShortArrayFrom(arr: Array[Byte], i: Int): (Array[Short], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeShortArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfShort) - } - - def decodeCharArray(arr: Array[Byte], offset: Int, len: Int): Array[Char] = { - val newArr = Array.ofDim[Char](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.charArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfChar) - newArr - } - - def decodeCharArrayFrom(arr: Array[Byte], i: Int): (Array[Char], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeCharArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfChar) - } - - def decodeIntArray(arr: Array[Byte], offset: Int, len: Int): Array[Int] = { - val newArr = Array.ofDim[Int](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.intArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfInt) - newArr - } - - def decodeIntArrayFrom(arr: Array[Byte], i: Int): (Array[Int], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeIntArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfInt) - } - - def decodeLongArray(arr: Array[Byte], offset: Int, len: Int): Array[Long] = { - val newArr = Array.ofDim[Long](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.longArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfLong) - newArr - } - - def decodeLongArrayFrom(arr: Array[Byte], i: Int): (Array[Long], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeLongArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfLong) - } - - def decodeFloatArray(arr: Array[Byte], offset: Int, len: Int): Array[Float] = { - val newArr = Array.ofDim[Float](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.floatArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfFloat) - newArr - } - - def decodeFloatArrayFrom(arr: Array[Byte], i: Int): (Array[Float], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeFloatArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfFloat) - } - - def decodeDoubleArray(arr: Array[Byte], offset: Int, len: Int): Array[Double] = { - val newArr = Array.ofDim[Double](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.doubleArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfDouble) - newArr - } - - def decodeDoubleArrayFrom(arr: Array[Byte], i: Int): (Array[Double], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeDoubleArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfDouble) - } - - def decodeBooleanArray(arr: Array[Byte], offset: Int, len: Int): Array[Boolean] = { - val newArr = Array.ofDim[Boolean](len) - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.booleanArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, srcOffset + offset, newArr, destOffset, len * SizeOfBoolean) - newArr - } - - def decodeBooleanArrayFrom(arr: Array[Byte], i: Int): (Array[Boolean], Int) = { - val len = decodeIntFrom(arr, i) - val nextPos = i + SizeOfInt - val ia = decodeBooleanArray(arr, nextPos, len) - (ia, nextPos + len * SizeOfBoolean) - } - - // Encoding functions - - def encodeByte(arr: ArrayOutput[Byte], value: Byte): Unit = { - arr += value - } - - def encodeShort(arr: ArrayOutput[Byte], value: Short): Unit = { - val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] - val snd = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - } - - def encodeInt(arr: ArrayOutput[Byte], value: Int): Unit = { - val fst = (value >>> 24).asInstanceOf[Byte] - val snd = (value >>> 16 & 0xff).asInstanceOf[Byte] - val thrd = (value >>> 8 & 0xff).asInstanceOf[Byte] - val frth = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - arr += thrd - arr += frth - } - - - def encodeLong(arr: ArrayOutput[Byte], value: Long): Unit = { - val elem1 = (value >>> 56 & 0xff).asInstanceOf[Byte] - val elem2 = (value >>> 48 & 0xff).asInstanceOf[Byte] - val elem3 = (value >>> 40 & 0xff).asInstanceOf[Byte] - val elem4 = (value >>> 32 & 0xff).asInstanceOf[Byte] - val elem5 = (value >>> 24 & 0xff).asInstanceOf[Byte] - val elem6 = (value >>> 16 & 0xff).asInstanceOf[Byte] - val elem7 = (value >>> 8 & 0xff).asInstanceOf[Byte] - val elem8 = (value & 0xff).asInstanceOf[Byte] - arr += elem1 - arr += elem2 - arr += elem3 - arr += elem4 - arr += elem5 - arr += elem6 - arr += elem7 - arr += elem8 - } - - def encodeChar(arr: ArrayOutput[Byte], value: Char): Unit = { - val fst = (value >>> 8 & 0xff).asInstanceOf[Byte] - val snd = (value & 0xff).asInstanceOf[Byte] - arr += fst - arr += snd - } - - def encodeString(arr: ArrayOutput[Byte], value: String): Unit = { - val bytes = value.getBytes("UTF-8") - encodeInt(arr, bytes.length) - val (buf, pos) = arr.target(bytes.length * SizeOfByte) - encodeByteArrayTo(buf, pos, bytes) - arr.flush(buf) - } - - def encodeBoolean(arr: ArrayOutput[Byte], value: Boolean): Unit = { - arr += (if (value) 1 else 0) - } - - def encodeByteArray(arr: ArrayOutput[Byte], value: Array[Byte]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfByte) - encodeByteArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeShortArray(arr: ArrayOutput[Byte], value: Array[Short]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfShort) - encodeShortArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeIntArray(arr: ArrayOutput[Byte], value: Array[Int]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfInt) - encodeIntArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeLongArray(arr: ArrayOutput[Byte], value: Array[Long]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfLong) - encodeLongArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeFloatArray(arr: ArrayOutput[Byte], value: Array[Float]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfFloat) - encodeFloatArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeDoubleArray(arr: ArrayOutput[Byte], value: Array[Double]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfDouble) - encodeDoubleArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeCharArray(arr: ArrayOutput[Byte], value: Array[Char]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfChar) - encodeCharArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeBooleanArray(arr: ArrayOutput[Byte], value: Array[Boolean]): Unit = { - encodeInt(arr, value.length) - val (buf, pos) = arr.target(value.length * SizeOfBoolean) - encodeBooleanArrayTo(buf, pos, value) - arr.flush(buf) - } - - def encodeByteArrayTo(arr: Array[Byte], i: Int, value: Array[Byte]): Int = { - val srcOffset = UnsafeMemory.byteArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfByte) - i + value.length * SizeOfByte - } - - def encodeShortArrayTo(arr: Array[Byte], i: Int, value: Array[Short]): Int = { - val srcOffset = UnsafeMemory.shortArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfShort) - i + value.length * SizeOfShort - } - - def encodeIntArrayTo(arr: Array[Byte], i: Int, value: Array[Int]): Int = { - val srcOffset = UnsafeMemory.intArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfInt) - i + value.length * SizeOfInt - } - - def encodeLongArrayTo(arr: Array[Byte], i: Int, value: Array[Long]): Int = { - val srcOffset = UnsafeMemory.longArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfLong) - i + value.length * SizeOfLong - } - - def encodeFloatArrayTo(arr: Array[Byte], i: Int, value: Array[Float]): Int = { - val srcOffset = UnsafeMemory.floatArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfFloat) - i + value.length * SizeOfFloat - } - - def encodeDoubleArrayTo(arr: Array[Byte], i: Int, value: Array[Double]): Int = { - val srcOffset = UnsafeMemory.doubleArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfDouble) - i + value.length * SizeOfDouble - } - - def encodeCharArrayTo(arr: Array[Byte], i: Int, value: Array[Char]): Int = { - val srcOffset = UnsafeMemory.charArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfChar) - i + value.length * SizeOfChar - } - - def encodeBooleanArrayTo(arr: Array[Byte], i: Int, value: Array[Boolean]): Int = { - val srcOffset = UnsafeMemory.booleanArrayOffset - val destOffset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(value, srcOffset, arr, destOffset + i, value.length * SizeOfBoolean) - i + value.length * SizeOfBoolean - } -} - object UnsafeMemory { import sun.misc.Unsafe - private[pickling] val unsafe: Unsafe = - scala.concurrent.util.Unsafe.instance - + scala.concurrent.util.Unsafe.instance private[pickling] val byteArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Byte]]) private[pickling] val shortArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Short]]) - private[pickling] val intArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Int]]) + private[pickling] val intArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Int]]) private[pickling] val longArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Long]]) private[pickling] val floatArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Float]]) private[pickling] val doubleArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Double]]) private[pickling] val charArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Char]]) private[pickling] val booleanArrayOffset: Long = unsafe.arrayBaseOffset(classOf[Array[Boolean]]) - def putInt(arr: Array[Byte], i: Int, value: Int): Unit = { - unsafe.putInt(arr, byteArrayOffset + i, value) + unsafe.putInt(arr, byteArrayOffset + i, value) } - def getInt(arr: Array[Byte], i: Int): Int = { - unsafe.getInt(arr, byteArrayOffset + i) + unsafe.getInt(arr, byteArrayOffset + i) } } diff --git a/core/src/main/scala/pickling/fastbinary/BinaryPickleFormat.scala b/core/src/main/scala/pickling/fastbinary/BinaryPickleFormat.scala deleted file mode 100644 index e4b5dd7747..0000000000 --- a/core/src/main/scala/pickling/fastbinary/BinaryPickleFormat.scala +++ /dev/null @@ -1,539 +0,0 @@ -package scala.pickling.fastbinary - -import java.io.InputStream - -import scala.reflect.runtime.universe.Mirror - -import scala.pickling.binary.{Constants, Util} -import scala.pickling.internal.lookupUnpicklee -import scala.pickling.{Pickle, PickleFormat, PBuilder, PReader, PicklingException, EndOfStreamException, FastTypeTag, - PickleTools, ArrayOutput, ByteArrayOutput, ByteArrayBufferOutput} - - -class BinaryPickleFormat extends PickleFormat with Constants { - type PickleType = BinaryPickle - type OutputType = FastArrayOutput - - def createBuilder() = new BinaryPickleBuilder(this, null) - def createBuilder(out: FastArrayOutput): PBuilder = new BinaryPickleBuilder(this, out) - - def createReader(pickle: PickleType, mirror: Mirror) = pickle.createReader(mirror, this) -} - -abstract class BinaryPickle extends Pickle { - type PickleFormatType = BinaryPickleFormat - type ValueType = Array[Byte] - - val value: Array[Byte] - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader -} - -case class BinaryPickleArray(data: Array[Byte]) extends BinaryPickle { - val value: Array[Byte] = data - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryPickleReader(data, mirror, format) - - override def toString = s"""BinaryPickle(${value.mkString("[", ",", "]")})""" -} - -case class BinaryPickleStream(input: InputStream) extends BinaryPickle { - val value: Array[Byte] = Array.ofDim[Byte](0) - - def createReader(mirror: Mirror, format: BinaryPickleFormat): PReader = - new BinaryInputStreamReader(input, mirror, format) - - /* Do not override def toString to avoid traversing the input stream. */ -} - -object BinaryPickle { - def apply(a: Array[Byte]): BinaryPickle = - new BinaryPickleArray(a) -} - -final class BinaryPickleBuilder(format: BinaryPickleFormat, out: FastArrayOutput) extends PBuilder with PickleTools { - import format._ - - // if no output provided: create output caching large pre-allocated array - private var output: FastArrayOutput = - if (out == null) new FastArrayOutput - else out - - @inline def beginEntry(picklee: Any): PBuilder = withHints { hints => - if (picklee == null) { - Util.encodeByte(output, NULL_TAG) - } else if (hints.oid != -1) { - Util.encodeByte(output, REF_TAG) - Util.encodeInt(output, hints.oid) - } else { - if (!hints.isElidedType) { - // quickly decide whether we should use picklee.getClass instead - val ts = - if (hints.tag.key.contains("anonfun$")) picklee.getClass.getName - else hints.tag.key - Util.encodeString(output, ts) - } - - // NOTE: it looks like we don't have to write object ids at all - // traversals employed by pickling and unpickling are exactly the same - // hence when unpickling it's enough to just increment the nextUnpicklee counter - // and everything will work out automatically! - - hints.tag.key match { // PERF: should store typestring once in hints. - case KEY_UNIT => - Util.encodeByte(output, UNIT_TAG) - case KEY_NULL => - Util.encodeByte(output, NULL_TAG) - case KEY_BYTE => - Util.encodeByte(output, picklee.asInstanceOf[Byte]) - case KEY_SHORT => - Util.encodeShort(output, picklee.asInstanceOf[Short]) - case KEY_CHAR => - Util.encodeChar(output, picklee.asInstanceOf[Char]) - case KEY_INT => - Util.encodeInt(output, picklee.asInstanceOf[Int]) - case KEY_LONG => - Util.encodeLong(output, picklee.asInstanceOf[Long]) - case KEY_BOOLEAN => - Util.encodeBoolean(output, picklee.asInstanceOf[Boolean]) - case KEY_FLOAT => - val intValue = java.lang.Float.floatToRawIntBits(picklee.asInstanceOf[Float]) - Util.encodeInt(output, intValue) - case KEY_DOUBLE => - val longValue = java.lang.Double.doubleToRawLongBits(picklee.asInstanceOf[Double]) - Util.encodeLong(output, longValue) - case KEY_STRING => - Util.encodeString(output, picklee.asInstanceOf[String]) - case KEY_ARRAY_BYTE => - Util.encodeByteArray(output, picklee.asInstanceOf[Array[Byte]]) - case KEY_ARRAY_CHAR => - Util.encodeCharArray(output, picklee.asInstanceOf[Array[Char]]) - case KEY_ARRAY_SHORT => - Util.encodeShortArray(output, picklee.asInstanceOf[Array[Short]]) - case KEY_ARRAY_INT => - Util.encodeIntArray(output, picklee.asInstanceOf[Array[Int]]) - case KEY_ARRAY_LONG => - Util.encodeLongArray(output, picklee.asInstanceOf[Array[Long]]) - case KEY_ARRAY_BOOLEAN => - Util.encodeBooleanArray(output, picklee.asInstanceOf[Array[Boolean]]) - case KEY_ARRAY_FLOAT => - Util.encodeFloatArray(output, picklee.asInstanceOf[Array[Float]]) - case KEY_ARRAY_DOUBLE => - Util.encodeDoubleArray(output, picklee.asInstanceOf[Array[Double]]) - case _ => - if (hints.isElidedType) Util.encodeByte(output, ELIDED_TAG) - } - } - this - } - - @inline def putField(name: String, pickler: PBuilder => Unit): PBuilder = { - // can skip writing name if we pickle/unpickle in the same order - pickler(this) - this - } - - @inline def endEntry(): Unit = { /* do nothing */ } - - @inline def beginCollection(length: Int): PBuilder = { - Util.encodeInt(output, length) - this - } - - @inline def putElement(pickler: PBuilder => Unit): PBuilder = { - pickler(this) - this - } - - @inline def endCollection(): Unit = { - } - - @inline def result() = { - BinaryPickle(output.result()) - } -} - - abstract class AbstractBinaryReader(val mirror: Mirror) { - protected var _lastTagRead: FastTypeTag[_] = null - protected var _lastTypeStringRead: String = null - - protected def lastTagRead: FastTypeTag[_] = - if (_lastTagRead != null) - _lastTagRead - else { - // assume _lastTypeStringRead != null - _lastTagRead = FastTypeTag(mirror, _lastTypeStringRead) - _lastTagRead - } - } - - class BinaryInputStreamReader(in: InputStream, mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - def nextByte(): Byte = { - val b = in.read() - if (b == -1) throw new EndOfStreamException - b.asInstanceOf[Byte] - } - - def decodeStringWithLookahead(la: Byte): String = { - // read 3 more bytes - val buf = Array[Byte](la, nextByte(), nextByte(), nextByte()) - val len = { - val len0 = Util.decodeIntFrom(buf, 0) - if (len0 > 1000) - throw PicklingException(s"decodeStringWithLookahead: corrupted length of type string: $len0") - else if (len0 < 0) - throw PicklingException(s"decodeStringWithLookahead: negative length of type string: $len0\nbuf: [${buf.mkString(",")}]") - else - len0 - } - val bytes = Array.ofDim[Byte](len) - var num = in.read(bytes) - while (num < len) { - val readMore = in.read(bytes, num, len - num) - num += readMore - } - new String(bytes, "UTF-8") - } - - var gla: Option[Byte] = None - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - // if (debugOn) - // debug(s"hints: $hints") - - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = nextByte() - lookahead match { - case NULL_TAG => gla = Some(lookahead); FastTypeTag.Null - case REF_TAG => FastTypeTag.Ref - case _ => gla = Some(lookahead); hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = nextByte() - // if (debugOn) - // debug(s"checking lookahead: $lookahead") - lookahead match { - case NULL_TAG => - FastTypeTag.Null - case ELIDED_TAG => - hints.tag - case REF_TAG => - FastTypeTag.Ref - case _ => - // do not consume lookahead byte - val res = try { - decodeStringWithLookahead(lookahead) - } catch { - case PicklingException(msg) => - val primInfo = if (hints.tag == null) "" - else s"\nnullable prim: ${nullablePrimitives.contains(hints.tag.key)}\nprim: ${primitives.contains(hints.tag.key)}" - throw PicklingException(s"error decoding type string. debug info: $hints$primInfo\ncause:$msg") - } - // if (debugOn) - // debug(s"decodeStringWithLookahead: $res") - res - } - } - } - if (res.isInstanceOf[String]) { - // if (debugOn) - // debug(s"replacing tag with last type string read: ${res.asInstanceOf[String]}") - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def decodeInt(): Int = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - } - - def decodeIntWithLookahead(): Int = gla match { - case Some(fstByte) => - gla = None // clear global lookahead - val buf = Array[Byte](fstByte, nextByte(), nextByte(), nextByte()) - Util.decodeIntFrom(buf, 0) - case None => - decodeInt() - } - - def decodeShort(): Short = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toShort - val snd = (buf(1) & 0x00FF).toShort - (fst | snd).toShort - } - - def decodeChar(): Char = { - val buf = Array[Byte](nextByte(), nextByte()) - val fst = ((buf(0) << 8) & 0xFFFF).toChar - val snd = (buf(1) & 0x00FF).toChar - (fst | snd).toChar - } - - def decodeLong(): Long = { - val buf = Array[Byte](nextByte(), nextByte(), nextByte(), nextByte(), - nextByte(), nextByte(), nextByte(), nextByte()) - val elem1 = ((buf(0).toLong << 56) & 0xFFFFFFFFFFFFFFFFL).toLong - val elem2 = ((buf(1).toLong << 48) & 0x00FFFFFFFFFFFFFFL).toLong - val elem3 = ((buf(2).toLong << 40) & 0x0000FFFFFFFFFFFFL).toLong - val elem4 = ((buf(3).toLong << 32) & 0x000000FFFFFFFFFFL).toLong - val elem5 = ((buf(4).toLong << 24) & 0x00000000FFFFFFFFL).toLong - val elem6 = ((buf(5).toLong << 16) & 0x0000000000FFFFFFL).toLong - val elem7 = ((buf(6).toLong << 8) & 0x000000000000FFFFL).toLong - val elem8 = (buf(7).toLong & 0x00000000000000FFL).toLong - elem1 | elem2 | elem3 | elem4 | elem5 | elem6 | elem7 | elem8 - } - - def decodeBoolean(): Boolean = { - nextByte() != 0 - } - - def decodeString(): String = { - val len = decodeIntWithLookahead() - val bytes = Array.ofDim[Byte](len) - if (len > 0) { - val num = in.read(bytes) - if (num < len) throw new Exception("Could not read enough bytes from input stream") - } - new String(bytes, "UTF-8") - } - - def decodeByteArray(): Array[Byte] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeByteArray(arr, 0, len) - } - - def decodeShortArray(): Array[Short] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeShortArray(arr, 0, len) - } - - def decodeCharArray(): Array[Char] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 2) - in.read(arr) - Util.decodeCharArray(arr, 0, len) - } - - def decodeIntArray(): Array[Int] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeIntArray(arr, 0, len) - } - - // Consider a macro such as Util.decodeArray[Long](in, len) - def decodeLongArray(): Array[Long] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeLongArray(arr, 0, len) - } - - def decodeBooleanArray(): Array[Boolean] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len) - in.read(arr) - Util.decodeBooleanArray(arr, 0, len) - } - - def decodeFloatArray(): Array[Float] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 4) - in.read(arr) - Util.decodeFloatArray(arr, 0, len) - } - - def decodeDoubleArray(): Array[Double] = { - val len = decodeIntWithLookahead() - val arr = Array.ofDim[Byte](len * 8) - in.read(arr) - Util.decodeDoubleArray(arr, 0, len) - } - - def readPrimitive(): Any = { - val res = lastTagRead.key match { - case KEY_NULL => null - case KEY_REF => lookupUnpicklee(decodeInt()) - case KEY_BYTE => nextByte() - case KEY_SHORT => decodeShort() - case KEY_CHAR => decodeChar() - case KEY_INT => decodeInt() - case KEY_LONG => decodeLong() - case KEY_BOOLEAN => decodeBoolean() - case KEY_FLOAT => - val r = decodeInt() - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = decodeLong() - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => decodeString() - - case KEY_ARRAY_BYTE => decodeByteArray() - case KEY_ARRAY_SHORT => decodeShortArray() - case KEY_ARRAY_CHAR => decodeCharArray() - case KEY_ARRAY_INT => decodeIntArray() - case KEY_ARRAY_LONG => decodeLongArray() - case KEY_ARRAY_BOOLEAN => decodeBooleanArray() - case KEY_ARRAY_FLOAT => decodeFloatArray() - case KEY_ARRAY_DOUBLE => decodeDoubleArray() - } - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryInputStreamReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - decodeInt() - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } - } - - class BinaryPickleReader(arr: Array[Byte], mirror: Mirror, format: BinaryPickleFormat) extends AbstractBinaryReader(mirror) with PReader with PickleTools { - import format._ - - private var pos = 0 - - def beginEntryNoTag(): String = - beginEntryNoTagDebug(false) - - def beginEntryNoTagDebug(debugOn: Boolean): String = { - val res: Any = withHints { hints => - if (hints.isElidedType && nullablePrimitives.contains(hints.tag.key)) { - val lookahead = arr(pos) - lookahead match { - case UNIT_TAG => pos += 1; FastTypeTag.Unit - case NULL_TAG => pos += 1; FastTypeTag.Null - case REF_TAG => pos += 1; FastTypeTag.Ref - case _ => hints.tag - } - } else if (hints.isElidedType && primitives.contains(hints.tag.key)) { - hints.tag - } else { - val lookahead = arr(pos) - lookahead match { - case NULL_TAG => - pos += 1 - FastTypeTag.Null - case ELIDED_TAG => - pos += 1 - hints.tag - case REF_TAG => - pos += 1 - FastTypeTag.Ref - case _ => - val (typeString, newpos) = Util.decodeStringFrom(arr, pos) - pos = newpos - typeString - } - } - } - if (res.isInstanceOf[String]) { - _lastTagRead = null - _lastTypeStringRead = res.asInstanceOf[String] - _lastTypeStringRead - } else { - _lastTagRead = res.asInstanceOf[FastTypeTag[_]] - _lastTagRead.key - } - } - - def beginEntry(): FastTypeTag[_] = { - beginEntryNoTag() - lastTagRead - } - - def atPrimitive: Boolean = primitives.contains(lastTagRead.key) - - def readPrimitive(): Any = { - var newpos = pos - val res = lastTagRead.key match { - case KEY_UNIT => () - case KEY_NULL => null - case KEY_REF => newpos = pos+4 ; lookupUnpicklee(Util.decodeIntFrom(arr, pos)) - case KEY_BYTE => newpos = pos+1 ; arr(pos) - case KEY_SHORT => newpos = pos+2 ; Util.decodeShortFrom(arr, pos) - case KEY_CHAR => newpos = pos+2 ; Util.decodeCharFrom(arr, pos) - case KEY_INT => newpos = pos+4 ; Util.decodeIntFrom(arr, pos) - case KEY_LONG => newpos = pos+8 ; Util.decodeLongFrom(arr, pos) - case KEY_BOOLEAN => newpos = pos+1 ; Util.decodeBooleanFrom(arr, pos) - case KEY_FLOAT => - val r = Util.decodeIntFrom(arr, pos) - newpos = pos+4 - java.lang.Float.intBitsToFloat(r) - case KEY_DOUBLE => - val r = Util.decodeLongFrom(arr, pos) - newpos = pos+8 - java.lang.Double.longBitsToDouble(r) - - case KEY_STRING => val r = Util.decodeStringFrom(arr, pos); newpos = r._2 ; r._1 - - case KEY_ARRAY_BYTE => val r = Util.decodeByteArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_SHORT => val r = Util.decodeShortArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_CHAR => val r = Util.decodeCharArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_INT => val r = Util.decodeIntArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_LONG => val r = Util.decodeLongArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_BOOLEAN => val r = Util.decodeBooleanArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_FLOAT => val r = Util.decodeFloatArrayFrom(arr, pos); newpos = r._2 ; r._1 - case KEY_ARRAY_DOUBLE => val r = Util.decodeDoubleArrayFrom(arr, pos); newpos = r._2 ; r._1 - } - - pos = newpos - res - } - - def atObject: Boolean = !atPrimitive - - def readField(name: String): BinaryPickleReader = - this - - def endEntry(): Unit = { /* do nothing */ } - - def beginCollection(): PReader = this - - def readLength(): Int = { - val length = Util.decodeIntFrom(arr, pos) - pos += 4 - length - } - - def readElement(): PReader = this - - def endCollection(): Unit = { /* do nothing */ } - } diff --git a/core/src/main/scala/pickling/fastbinary/ByteArrayBuilder.scala b/core/src/main/scala/pickling/fastbinary/ByteArrayBuilder.scala deleted file mode 100644 index a1952dc80e..0000000000 --- a/core/src/main/scala/pickling/fastbinary/ByteArrayBuilder.scala +++ /dev/null @@ -1,139 +0,0 @@ -package scala.pickling.fastbinary - -import scala.pickling.binary.UnsafeMemory - - -private[fastbinary] sealed abstract class ByteArrayBuilder { - - /** Tests whether this instance is full and must be replaced - * by a new (typically growable) instance. - */ - def isFull: Boolean - - /** Contract: must have called `isFull` - */ - def +=(b: Byte): Unit - - def toArray: Array[Byte] - - def target(l: Int): (Array[Byte], Int) - - def flush(a: Array[Byte]): Unit - - /** Checkpoints the current state of the builder, so that - * its resources can be reused. - * - * @return `true` if `this` builder is ready to be reused - */ - def checkpoint(): Boolean -} - -private[fastbinary] final class ByteArray(len: Int) extends ByteArrayBuilder { - // TODO: use Unsafe for allocation? - private val arr = Array.ofDim[Byte](len) - private var pos = 0 - private var startPos = 0 // invariant: startPos <= pos - - def isFull: Boolean = - pos == len - - def +=(b: Byte): Unit = { - UnsafeMemory.putInt(arr, pos, b) - pos += 1 - } - - def toArray: Array[Byte] = { - val numBytes = pos - startPos - val newArray = Array.ofDim[Byte](numBytes) - // fast array copy - val offset = UnsafeMemory.byteArrayOffset - UnsafeMemory.unsafe.copyMemory(arr, offset + startPos, newArray, offset, numBytes) - newArray - } - - def target(l: Int): (Array[Byte], Int) = { - if (pos + l > len) - throw new IllegalArgumentException - val oldpos = pos - pos = pos + l - (arr, oldpos) - } - - def flush(a: Array[Byte]): Unit = { /* no-op */ } - - def checkpoint(): Boolean = { - startPos = pos - true - } -} - -private[fastbinary] final class ByteArrayBuffer(headArray: Array[Byte]) extends ByteArrayBuilder { - - private val headLength = headArray.length - - // size of newly created array chunks - private val chunkSize = 64 - - private def mkNode(): Node = new Node(Array.ofDim[Byte](chunkSize), null) - - private var lastNode = mkNode() - - private val headNode = new Node(headArray, lastNode) - - // position in array that is currently being filled (*not* array of `headNode`) - private var pos = 0 - - // cached size of entire builder; is never reset (unlike `pos`) - private var size0 = headLength - - def isFull: Boolean = false // growable - - def +=(b: Byte): Unit = if (pos < chunkSize) { - // array in `lastNode` has space - lastNode.arr(pos) = b - size0 += 1 - pos += 1 - } else { // pos == chunkSize - val newNode = mkNode() - lastNode.next = newNode - lastNode = newNode - pos = 0 - // try again - this += b - } - - def toArray: Array[Byte] = { - val newArray = Array.ofDim[Byte](size0) - - // copy `headArray` to `newArray` - System.arraycopy(headArray, 0, newArray, 0, headLength) - - // copy subsequent arrays (if any) - var destPos = headLength - var n = headNode.next - while (n != null) { - // decide how many bytes to copy (`pos` many or `chunkSize` many) - val numToCopy = if (n.next != null) chunkSize else pos - System.arraycopy(n.arr, 0, newArray, destPos, numToCopy) - destPos += numToCopy - n = n.next - } - - newArray - } - - def target(l: Int): (Array[Byte], Int) = - (Array.ofDim[Byte](l), 0) - - def flush(a: Array[Byte]): Unit = { - var i = 0 - while (i < a.length) { - this += a(i) - i += 1 - } - } - - def checkpoint(): Boolean = false -} - -private[fastbinary] final class Node(val arr: Array[Byte], var next: Node) diff --git a/core/src/main/scala/pickling/fastbinary/Output.scala b/core/src/main/scala/pickling/fastbinary/Output.scala deleted file mode 100644 index 297ed581d6..0000000000 --- a/core/src/main/scala/pickling/fastbinary/Output.scala +++ /dev/null @@ -1,81 +0,0 @@ -package scala.pickling.fastbinary - -import java.io.OutputStream -import scala.pickling.ArrayOutput - - -object FastArrayOutput { - - private[pickling] val arrayBuilder = new ThreadLocal[ByteArrayBuilder] { - // initially a builder with pre-allocated array - override def initialValue(): ByteArrayBuilder = new ByteArray(64 * 1024 * 1024) // 64 MB - } - -} - -class FastArrayOutput extends ArrayOutput[Byte] { - - private var builder: ByteArrayBuilder = { - val b = FastArrayOutput.arrayBuilder.get() - val reuse = b.checkpoint() - if (!reuse) { - // allocate new array builder - val newb = new ByteArray(64 * 1024 * 1024) // 64 MB - // put into thread-local storage - FastArrayOutput.arrayBuilder.set(newb) - newb.checkpoint() // guaranteed to return `true` - newb - } else b - } - - // replace `builder` with a growable one - private def replaceBuilder(): Unit = { - builder = new ByteArrayBuffer(builder.toArray) - FastArrayOutput.arrayBuilder.set(builder) - } - - def result(): Array[Byte] = builder.toArray - - def +=(obj: Byte) = { - // if builder is full replace `builder` with a growable one - if (builder.isFull) replaceBuilder() - builder += obj - } - - def put(obj: Array[Byte]): this.type = { - // should use `target` - throw new java.lang.IllegalStateException - } - - override def target(l: Int) = try { - builder.target(l) - } catch { - case e: IllegalArgumentException => - // `builder` does not have enough space - replaceBuilder() - builder.target(l) // guaranteed to succeed - } - - override def flush(a: Array[Byte]) = - builder.flush(a) -} - - -final class FastOutputStreamOutput(out: OutputStream) extends FastArrayOutput { - override def result(): Array[Byte] = - null - - override def +=(obj: Byte) = - out.write(obj.asInstanceOf[Int]) - - override def put(obj: Array[Byte]): this.type = { - out.write(obj) - this - } - - override def target(len: Int): (Array[Byte], Int) = - (Array.ofDim[Byte](len), 0) - - override def flush(arr: Array[Byte]): Unit = - this.put(arr) -} diff --git a/core/src/main/scala/pickling/fastbinary/package.scala b/core/src/main/scala/pickling/fastbinary/package.scala deleted file mode 100644 index b435ad869e..0000000000 --- a/core/src/main/scala/pickling/fastbinary/package.scala +++ /dev/null @@ -1,9 +0,0 @@ -package scala.pickling - -import scala.language.implicitConversions - - -package object fastbinary { - implicit val pickleFormat = new BinaryPickleFormat - implicit def toBinaryPickle(value: Array[Byte]): BinaryPickle = BinaryPickle(value) -} diff --git a/core/src/main/scala/pickling/package.scala b/core/src/main/scala/pickling/package.scala index cceae22b37..f37adfe06a 100644 --- a/core/src/main/scala/pickling/package.scala +++ b/core/src/main/scala/pickling/package.scala @@ -8,7 +8,7 @@ package object pickling { implicit class PickleOps[T](picklee: T) { def pickle(implicit format: PickleFormat): format.PickleType = macro Compat.PickleMacros_pickle[T] def pickleInto(builder: PBuilder): Unit = macro Compat.PickleMacros_pickleInto[T] - def pickleTo(output: Output[_])(implicit format: PickleFormat): Unit = macro Compat.PickleMacros_pickleTo[T] + def pickleTo[S](output: S)(implicit format: PickleFormat): Unit = macro Compat.PickleMacros_pickleTo[T,S] } implicit class UnpickleOps(val thePickle: Pickle) { diff --git a/core/src/test/scala/pickling/run/binary-inputstream-output.scala b/core/src/test/scala/pickling/run/binary-inputstream-output.scala index 8fc259f834..9c4339b2b3 100644 --- a/core/src/test/scala/pickling/run/binary-inputstream-output.scala +++ b/core/src/test/scala/pickling/run/binary-inputstream-output.scala @@ -14,7 +14,7 @@ class BinaryInputStreamReaderOutputTest extends FunSuite { val obj1 = Employee("James", 30) val obj2 = Employee("Jim", 40) - val output = new ByteArrayBufferOutput + val output = new ByteArrayOutput obj1.pickleTo(output) obj2.pickleTo(output) diff --git a/core/src/test/scala/pickling/run/binary-inputstream.scala b/core/src/test/scala/pickling/run/binary-inputstream.scala index 1f661b8c1c..ea2db5649d 100644 --- a/core/src/test/scala/pickling/run/binary-inputstream.scala +++ b/core/src/test/scala/pickling/run/binary-inputstream.scala @@ -67,7 +67,7 @@ class BinaryInputStreamReaderTest extends FunSuite { val obj1 = Employee("James", 30) val obj2 = Employee("Jim", 40) - val output = new ByteArrayBufferOutput + val output = new ByteArrayOutput obj1.pickleTo(output) obj2.pickleTo(output) diff --git a/core/src/test/scala/pickling/run/binary-outputstream.scala b/core/src/test/scala/pickling/run/binary-outputstream.scala index 83663ee043..6f8bb03bc5 100644 --- a/core/src/test/scala/pickling/run/binary-outputstream.scala +++ b/core/src/test/scala/pickling/run/binary-outputstream.scala @@ -3,7 +3,6 @@ package scala.pickling.test.binary import org.scalatest.FunSuite import java.io.{OutputStream, ByteArrayOutputStream, ByteArrayInputStream} -import scala.pickling.OutputStreamOutput import scala.pickling._ import binary._ @@ -14,7 +13,7 @@ class BinaryOutputStreamTest extends FunSuite { val obj2 = Employee("Jim", 40) val stream = new ByteArrayOutputStream - val output = new OutputStreamOutput(stream) + val output = new StreamOutput(stream) obj1.pickleTo(output) obj2.pickleTo(output) diff --git a/core/src/test/scala/pickling/run/binary-util.scala b/core/src/test/scala/pickling/run/binary-util.scala deleted file mode 100644 index 2f9837ebd8..0000000000 --- a/core/src/test/scala/pickling/run/binary-util.scala +++ /dev/null @@ -1,137 +0,0 @@ -package scala.pickling.binary.util - -import org.scalatest.FunSuite - -import scala.pickling._ -import binary.Util - -class BinaryUtilTest extends FunSuite { - - val len = 128 - val rand = new scala.util.Random() - - test("sizeOf") { - assert(Util.SizeOfByte === 1) - assert(Util.SizeOfShort === 2) - assert(Util.SizeOfInt === 4) - assert(Util.SizeOfLong === 8) - assert(Util.SizeOfFloat === 4) - assert(Util.SizeOfDouble === 8) - assert(Util.SizeOfChar === 2) - assert(Util.SizeOfBoolean === 1) - } - - test("byte") { - val arr = new ByteArrayOutput(Util.SizeOfByte) - val data = rand.nextInt().toByte - Util.encodeByte(arr, data) - assert(arr.result()(0) === data) - } - - test("short") { - val arr = new ByteArrayOutput(Util.SizeOfShort) - val data = rand.nextInt().toShort - Util.encodeShort(arr, data) - assert(Util.decodeShortFrom(arr.result(), 0) === data) - } - - test("int") { - val arr = new ByteArrayOutput(Util.SizeOfInt) - val data = rand.nextInt() - Util.encodeInt(arr, data) - assert(Util.decodeIntFrom(arr.result(), 0) === data) - } - - test("long") { - val arr = new ByteArrayOutput(Util.SizeOfLong) - val data = rand.nextLong() - Util.encodeLong(arr, data) - assert(Util.decodeLongFrom(arr.result(), 0) === data) - } - - /* TODO: complete Float and Double - test("float") { - val arr = new ByteArrayOutput(Util.SizeOfFloat) - val data = rand.nextFloat() - Util.encodeFloat(arr, data) - assert(Util.decodeFloatFrom(arr.result(), 0) === data) - } - - test("double") { - val arr = new ByteArrayOutput(Util.SizeOfDouble) - val data = rand.nextDouble() - Util.encodeDouble(arr, data) - assert(Util.decodeDoubleFrom(arr.result(), 0) === data) - } - */ - - test("char") { - val arr = new ByteArrayOutput(Util.SizeOfChar) - val data = rand.nextInt().toChar - Util.encodeChar(arr, data) - assert(Util.decodeCharFrom(arr.result(), 0) === data) - } - - test("string") { - val data = rand.nextString(len) - val arr = new ByteArrayOutput(data.getBytes("UTF-8").length+Util.SizeOfInt) - Util.encodeString(arr, data) - assert(Util.decodeStringFrom(arr.result(), 0)._1 === data) - } - - test("byte-array") { - val arr = new ByteArrayOutput(Util.SizeOfByte*len+Util.SizeOfInt) - val data = Array.ofDim[Byte](len).map(_ => rand.nextInt().toByte) - Util.encodeByteArray(arr, data) - assert(Util.decodeByteArrayFrom(arr.result(), 0)._1 === data) - } - - test("short-array") { - val arr = new ByteArrayOutput(Util.SizeOfShort*len+Util.SizeOfInt) - val data = Array.ofDim[Short](len).map(_ => rand.nextInt().toShort) - Util.encodeShortArray(arr, data) - assert(Util.decodeShortArrayFrom(arr.result(), 0)._1 === data) - } - - test("int-array") { - val arr = new ByteArrayOutput(Util.SizeOfInt*len+Util.SizeOfInt) - val data = Array.ofDim[Int](len).map(_ => rand.nextInt()) - Util.encodeIntArray(arr, data) - assert(Util.decodeIntArrayFrom(arr.result(), 0)._1 === data) - } - - test("long-array") { - val arr = new ByteArrayOutput(Util.SizeOfLong*len+Util.SizeOfInt) - val data = Array.ofDim[Long](len).map(_ => rand.nextLong()) - Util.encodeLongArray(arr, data) - assert(Util.decodeLongArrayFrom(arr.result(), 0)._1 === data) - } - - test("float-array") { - val arr = new ByteArrayOutput(Util.SizeOfFloat*len+Util.SizeOfInt) - val data = Array.ofDim[Float](len).map(_ => rand.nextFloat()) - Util.encodeFloatArray(arr, data) - assert(Util.decodeFloatArrayFrom(arr.result(), 0)._1 === data) - } - - test("double-array") { - val arr = new ByteArrayOutput(Util.SizeOfDouble*len+Util.SizeOfInt) - val data = Array.ofDim[Double](len).map(_ => rand.nextDouble()) - Util.encodeDoubleArray(arr, data) - assert(Util.decodeDoubleArrayFrom(arr.result(), 0)._1 === data) - } - - test("char-array") { - val arr = new ByteArrayOutput(Util.SizeOfChar*len+Util.SizeOfInt) - val data = Array.ofDim[Char](len).map(_ => rand.nextInt().toChar) - Util.encodeCharArray(arr, data) - assert(Util.decodeCharArrayFrom(arr.result(), 0)._1 === data) - } - - test("boolean-array") { - val arr = new ByteArrayOutput(Util.SizeOfBoolean*len+Util.SizeOfInt) - val data = Array.ofDim[Boolean](len).map(_ => rand.nextBoolean()) - Util.encodeBooleanArray(arr, data) - assert(Util.decodeBooleanArrayFrom(arr.result(), 0)._1 === data) - } -} diff --git a/core/src/test/scala/pickling/run/byte-array-builder.scala b/core/src/test/scala/pickling/run/byte-array-builder.scala deleted file mode 100644 index 14d23b488e..0000000000 --- a/core/src/test/scala/pickling/run/byte-array-builder.scala +++ /dev/null @@ -1,123 +0,0 @@ -package scala.pickling.fastbinary -package test - -import org.scalatest.FunSuite - -class ByteArrayBuilderTest extends FunSuite { - - def bufferWithEmptyArray(): ByteArrayBuilder = { - val empty = Array.ofDim[Byte](0) - new ByteArrayBuffer(empty) - } - - def bufferOfDim(size: Int): ByteArrayBuilder = - new ByteArrayBuffer(Array.ofDim[Byte](size)) - - test("buffer with empty array: isFull") { - val builder: ByteArrayBuilder = bufferWithEmptyArray() - assert(builder.isFull == false) - } - - test("buffer with empty array: +=") { - val builder: ByteArrayBuilder = bufferWithEmptyArray() - builder += 5 - val a = builder.toArray - assert(a.length == 1) - assert(a(0) == 5) - } - - test("buffer with empty array: toArray") { - val builder: ByteArrayBuilder = bufferWithEmptyArray() - assert(builder.toArray.length == 0) - } - - test("buffer with non-empty array") { - val builder: ByteArrayBuilder = bufferOfDim(1) - builder += 5 - val a = builder.toArray - assert(a.length == 2) - assert(a(0) == 0) - assert(a(1) == 5) - } - - test("byte array, empty: toArray") { - val builder: ByteArrayBuilder = new ByteArray(4) - val a = builder.toArray - assert(a.length == 0) - } - - test("byte array, non-empty: toArray") { - val builder: ByteArrayBuilder = new ByteArray(4) - builder += 1 - builder += 2 - val a = builder.toArray - assert(a.length == 2) - assert(a(0) == 1) - assert(a(1) == 2) - } - - test("byte array, non-empty: checkpoint 1") { - val builder: ByteArrayBuilder = new ByteArray(4) - builder += 1 - builder += 2 - val ok = builder.checkpoint() - assert(ok == true) - val a = builder.toArray - assert(a.length == 0) - } - - test("byte array, non-empty: checkpoint 2") { - val builder: ByteArrayBuilder = new ByteArray(4) - val ok = builder.checkpoint() - assert(ok == true) - builder += 1 - builder += 2 - val a = builder.toArray - assert(a.length == 2) - } - - test("byte array, non-empty: checkpoint 3") { - val builder: ByteArrayBuilder = new ByteArray(4) - val ok1 = builder.checkpoint() - assert(ok1 == true) - builder += 1 - builder += 2 - val a1 = builder.toArray - assert(a1.length == 2) - assert(a1(0) == 1) - val ok2 = builder.checkpoint() - assert(ok2 == true) - builder += 3 - builder += 4 - val a2 = builder.toArray - assert(a2.length == 2) - assert(a2(0) == 3) - } - - test("byte array: target with overflow") { - val builder: ByteArrayBuilder = new ByteArray(1) - val ok1 = builder.checkpoint() - assert(ok1 == true) - try { - builder.target(2) - assert(false) - } catch { - case e: IllegalArgumentException => - assert(true) - } - } - - test("byte array: target without overflow") { - val builder: ByteArrayBuilder = new ByteArray(2) - val ok1 = builder.checkpoint() - assert(ok1 == true) - builder += 1 - try { - builder.target(1) - assert(true) - } catch { - case e: IllegalArgumentException => - assert(false) - } - } -} diff --git a/core/src/test/scala/pickling/run/bytebuffers.scala b/core/src/test/scala/pickling/run/bytebuffers.scala index d0af5ef2c4..6e8320a253 100644 --- a/core/src/test/scala/pickling/run/bytebuffers.scala +++ b/core/src/test/scala/pickling/run/bytebuffers.scala @@ -10,10 +10,10 @@ object Primitives extends Properties("bytebuffer primitive tests") { def roundTrip[T: SPickler: Unpickler: FastTypeTag](obj: T): Boolean = { try { - val out = new ByteBufferOutput(ByteBuffer.allocate(512)) + val buf = ByteBuffer.allocate(1024) + val out = new ByteBufferOutput(buf) val builder = pickleFormat.createBuilder(out) obj.pickleInto(builder) - val buf = out.buffer buf.rewind val p = new BinaryInputPickle(new ByteBufferInput(buf)) val up = p.unpickle[T] @@ -53,18 +53,5 @@ class ByteBufferTest extends FunSuite { res == 0x12345678 } - test("reallocation") { - val obj = (0 until 500).toArray - val buf = ByteBuffer.allocate(12) - val out = new ByteBufferOutput(buf) - val builder = pickleFormat.createBuilder(out) - obj.pickleInto(builder) - val buf2 = out.buffer - buf2.rewind - val p = new BinaryInputPickle(new ByteBufferInput(buf2)) - val up = p.unpickle[Array[Int]] - obj == up - } - } diff --git a/core/src/test/scala/pickling/run/fast-array-output.scala b/core/src/test/scala/pickling/run/fast-array-output.scala deleted file mode 100644 index 8293be3be5..0000000000 --- a/core/src/test/scala/pickling/run/fast-array-output.scala +++ /dev/null @@ -1,23 +0,0 @@ -package scala.pickling.fastbinary -package test - -import org.scalatest.FunSuite - -class FastArrayOutputTest extends FunSuite { - - test("target() overflow") { - val out = new FastArrayOutput - // fill up cached array - for (_ <- 1 to 64 * 1024 * 1024 - 1) { - out += 5 - } - val out2 = new FastArrayOutput // does checkpoint() - val (a, pos) = out2.target(2) // (Array[Byte], Int) - a(pos) = 6 - out2.flush(a) - val res = out2.result() - assert(res.length == 2) - assert(res(0) == 6) - } - -} diff --git a/core/src/test/scala/pickling/run/fastbinary1.scala b/core/src/test/scala/pickling/run/fastbinary1.scala deleted file mode 100644 index 849433cc50..0000000000 --- a/core/src/test/scala/pickling/run/fastbinary1.scala +++ /dev/null @@ -1,16 +0,0 @@ -package scala.pickling.binary.`case`.`class`.int.string - -import org.scalatest.FunSuite - -import scala.pickling._ -import fastbinary._ - -class FastBinaryCaseClassIntStringTest extends FunSuite { - test("main") { - val p = Person("Jim", 43) - val pickle = p.pickle - assert(pickle.value.mkString("[", ",", "]") === "[0,0,0,50,115,99,97,108,97,46,112,105,99,107,108,105,110,103,46,98,105,110,97,114,121,46,99,97,115,101,46,99,108,97,115,115,46,105,110,116,46,115,116,114,105,110,103,46,80,101,114,115,111,110,0,0,0,3,74,105,109,0,0,0,43]") - val up = pickle.unpickle[Person] - assert(p === up) - } -} diff --git a/core/src/test/scala/pickling/run/share-binary.scala b/core/src/test/scala/pickling/run/share-binary.scala index 20ced6ee01..4b3089cca8 100644 --- a/core/src/test/scala/pickling/run/share-binary.scala +++ b/core/src/test/scala/pickling/run/share-binary.scala @@ -92,7 +92,7 @@ class ShareBinaryTest extends FunSuite { } test("register many unpicklees") { - val output = new ByteArrayBufferOutput + val output = new ByteArrayOutput val arr = Array.ofDim[Simple](66000) for (i <- 0 until 66000) {