diff --git a/project.clj b/project.clj index e5adb57..a3a1275 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject funcool/octet "1.0.1" +(defproject funcool/octet "1.0.2" :description "A clojure(script) library for work with binary data." :url "https://github.com/funcool/octet" :license {:name "Public Domain" diff --git a/src/octet/core.cljc b/src/octet/core.cljc index cf1c953..1b5992b 100644 --- a/src/octet/core.cljc +++ b/src/octet/core.cljc @@ -32,6 +32,7 @@ [octet.spec.basic :as basic-spec] [octet.spec.string :as string-spec] [octet.spec.collections :as coll-spec] + [octet.spec.reference :as ref-spec] [octet.buffer :as buffer])) (util/defalias compose spec/compose) @@ -41,6 +42,8 @@ (util/defalias string string-spec/string) (util/defalias string* string-spec/string*) (util/defalias vector* coll-spec/vector*) +(util/defalias ref-string* ref-spec/ref-string*) +(util/defalias ref-bytes* ref-spec/ref-bytes*) (util/defalias int16 basic-spec/int16) (util/defalias uint16 basic-spec/uint16) (util/defalias int32 basic-spec/int32) diff --git a/src/octet/spec.cljc b/src/octet/spec.cljc index bf452fc..fd9c3d0 100644 --- a/src/octet/spec.cljc +++ b/src/octet/spec.cljc @@ -31,7 +31,8 @@ For more examples see the `spec` function docstring." (:refer-clojure :exclude [type read float double long short byte bytes repeat]) - (:require [octet.buffer :as buffer])) + (:require [octet.buffer :as buffer] + [octet.util :refer [assoc-ordered]])) ;; --- Protocols @@ -48,6 +49,12 @@ "Abstraction for calculate size for dynamic specs." (size* [_ data] "Calculate the size in bytes of the object having a data.")) +(defprotocol ISpecWithRef + "Abstraction to support specs having references to other + specs within an AssociativeSpec or an IndexedSpec" + (read* [_ buff start data] "Read data from buffer, use data to calculate length etc") + (write* [_ buff start value types data] "Write data from buffer, use data to store length etc")) + ;; --- Composed Spec Types (deftype AssociativeSpec [data dict types] @@ -74,20 +81,26 @@ ISpec (read [_ buff pos] - (loop [index pos result {} pairs data] + (loop [index pos result (array-map) pairs data] (if-let [[fieldname type] (first pairs)] - (let [[readedbytes readeddata] (read type buff index)] + (let [[readedbytes readeddata] + (if (satisfies? ISpecWithRef type) + (read* type buff index result) + (read type buff index))] (recur (+ index readedbytes) - (assoc result fieldname readeddata) + (assoc-ordered result fieldname readeddata) (rest pairs))) - [(- index pos) result]))) + [(- index pos) result]))) (write [_ buff pos data'] - (let [written (reduce (fn [index [fieldname type]] - (let [value (get data' fieldname nil) - written (write type buff index value)] - (+ index written))) - pos data)] + (let [written + (reduce (fn [index [fieldname type]] + (let [value (get data' fieldname nil) + written (if (satisfies? ISpecWithRef type) + (write* type buff index value dict data') + (write type buff index value))] + (+ index written))) + pos data)] (- written pos)))) (deftype IndexedSpec [types] @@ -116,7 +129,10 @@ (read [_ buff pos] (loop [index pos result [] types types] (if-let [type (first types)] - (let [[readedbytes readeddata] (read type buff index)] + (let [[readedbytes readeddata] + (if (satisfies? ISpecWithRef type) + (read* type buff index result) + (read type buff index))] (recur (+ index readedbytes) (conj result readeddata) (rest types))) @@ -126,7 +142,9 @@ (let [indexedtypes (map-indexed vector types) written (reduce (fn [pos [index type]] (let [value (nth data' index nil) - written (write type buff pos value)] + written (if (satisfies? ISpecWithRef type) + (write* type buff pos value types data') + (write type buff pos value))] (+ pos written))) pos indexedtypes)] (- written pos)))) @@ -138,10 +156,14 @@ ;; --- Spec Constructors +(defn- spec? [s] + (or (satisfies? ISpecWithRef s) + (satisfies? ISpec s))) + (defn- associative-spec [& params] (let [data (mapv vec (partition 2 params)) - dict (into {} data) + dict (apply array-map params) types (map second data)] (AssociativeSpec. data dict types))) @@ -179,12 +201,12 @@ [& params] (let [numparams (count params)] (cond - (every? #(satisfies? ISpec %) params) + (every? spec? params) (apply indexed-spec params) (and (even? numparams) (keyword? (first params)) - (satisfies? ISpec (second params))) + (spec? (second params))) (apply associative-spec params) :else diff --git a/src/octet/spec/reference.cljc b/src/octet/spec/reference.cljc new file mode 100644 index 0000000..c021d9a --- /dev/null +++ b/src/octet/spec/reference.cljc @@ -0,0 +1,143 @@ +;; Copyright (c) 2015-2016 Andrey Antukh +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, this +;; list of conditions and the following disclaimer. +;; +;; * Redistributions in binary form must reproduce the above copyright notice, +;; this list of conditions and the following disclaimer in the documentation +;; and/or other materials provided with the distribution. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +(ns octet.spec.reference + (:require [octet.buffer :as buffer] + [octet.spec :as spec] + [octet.spec.string :as string-spec])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Spec types for arbitrary length byte arrays/strings with a length reference +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- ref-size [type data] + (if (satisfies? spec/ISpecSize type) + (.size type) + (.size* type data))) + +(defn- coerce-types [types] + "to make it terser to handle both maps (assoc spec) and seq/vector + (indexed spec), we coerce the seq/vector types to maps where the + keys are indexes" + (cond (map? types) types + (or (seq? types) + (vector? types)) (apply array-map (interleave (range) types)) + :else (throw (ex-info "invalid type structure, not map, seq or vector" + {:type-structure types})))) + +(defn- ref-len-offset [ref-kw-or-index types data] + "for ref-xxxx* specs, calculate the byte offset of the + spec containing the length, i.e. (spec :a (int16) :b (int16) c: (ref-string* :b)) + would cause this method to be called with :b (since the ref-string has a :b + reference as its length) and should then return 2 as the second int16 is at byte offset 2" + (reduce-kv + (fn [acc kw-or-index type] + (if (= ref-kw-or-index kw-or-index) + (reduced acc) + (+ acc (ref-size type (get data kw-or-index))))) + 0 + types)) + +(defn- ref-write* [ref-kw-or-index buff pos value types data] + "write a ref spec, will also write the length to the length spec" + (let [input (if (string? value) (string-spec/string->bytes value) value) + length (count input) + types (coerce-types types) + len-offset (ref-len-offset ref-kw-or-index types data) + len-type (get types ref-kw-or-index)] + (.write len-type buff len-offset length) + (buffer/write-bytes buff pos length input) + (+ length))) + +(defn- ref-read* [ref-kw-or-index buff pos parent] + "read ref spec, will read the length from the length spec" + (let [datasize (cond (map? parent) (ref-kw-or-index parent) + (or (seq? parent) (vector? parent)) (get parent ref-kw-or-index) + :else (throw (ex-info + (str "bad ref-string*/ref-bytes* length reference - " ref-kw-or-index) + {:length-kw ref-kw-or-index + :data-read parent}))) + data (buffer/read-bytes buff pos datasize)] + [datasize data])) + +(defn ref-bytes* + [ref-kw-or-index] + "create a dynamic length byte array where the length of the byte array is stored in another + spec within the containing indexed spec or associative spec. Example usages: + (spec (int16) (int16) (ref-bytes* 1)) + (spec :a (int16) :b (int32) (ref-bytes* :b)) + where the first example would store the length of the byte array in the second int16 and + the second example would store the length of the byte array in the int32 at key :b." + (reify + #?@(:clj + [clojure.lang.IFn + (invoke [s] s)] + :cljs + [cljs.core/IFn + (-invoke [s] s)]) + + spec/ISpecDynamicSize + (size* [_ data] + (count data)) + + spec/ISpecWithRef + (read* [_ buff pos parent] + (ref-read* ref-kw-or-index buff pos parent)) + + (write* [_ buff pos value types data] + (ref-write* ref-kw-or-index buff pos value types data)))) + +(defn ref-string* + "create a dynamic length string where the length of the string is stored in another + spec within the containing indexed spec or associative spec. Example usages: + (spec (int16) (int16) (ref-string* 1)) + (spec :a (int16) :b (int32) (ref-string* :b)) + where the first example would store the length of the string in the second int16 and + the second example would store the length of the string in the int32 at key :b." + [ref-kw-or-index] + (reify + #?@(:clj + [clojure.lang.IFn + (invoke [s] s)] + :cljs + [cljs.core/IFn + (-invoke [s] s)]) + + spec/ISpecDynamicSize + (size* [_ data] + (let [data (string-spec/string->bytes data)] + (count data))) + + spec/ISpecWithRef + (read* [_ buff pos parent] + (let [[datasize bytes] (ref-read* ref-kw-or-index buff pos parent)] + [datasize (string-spec/bytes->string bytes datasize)])) + + (write* [_ buff pos value types data] + (ref-write* ref-kw-or-index buff pos value types data)))) + +(defn ref-vector* + [ref-kw-or-index] + ;; TODO: implement this + ) diff --git a/src/octet/util.cljc b/src/octet/util.cljc index 4cbfee2..f52419e 100644 --- a/src/octet/util.cljc +++ b/src/octet/util.cljc @@ -22,7 +22,9 @@ ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -(ns octet.util) +(ns octet.util + (:require [clojure.string :as str :refer [join ]]) + (:import [java.util.Arrays])) (defmacro defalias [sym sym2] @@ -30,4 +32,124 @@ (def ~sym ~sym2) (alter-meta! (var ~sym) merge (dissoc (meta (var ~sym2)) :name)))) +(defn assoc-ordered [a-map key val & rest] + "assoc into an array-map, keeping insertion order. The normal clojure + assoc function switches to hash maps on maps > size 10 and loses insertion order" + (let [kvs (interleave (concat (keys a-map) (list key)) + (concat (vals a-map) (list val))) + ret (apply array-map kvs)] + (if rest + (if (next rest) + (recur ret (first rest) (second rest) (nnext rest)) + (throw (IllegalArgumentException. + "assoc-ordered expects even number of arguments after map/vector, found odd number"))) + ret))) +;; +;; Hexdumps +;; + + +(defn bytes->hex [^bytes bytes] + "converts a byte array to a hex string" + (let [[f & r] bytes + fh (fn [_ b] + (let [h (Integer/toHexString (bit-and b 0xFF))] + (if (<= 0 b 15) (str "0" h) h)))] + (join (reductions fh (fh 0 f) r)))) + +(defn byte->ascii [byte] + "convert a byte to 'printable' ascii where possible, otherwise ." + (if (<= 32 (bit-and byte 0xFF) 127) (char byte) \.)) + +(defn- bytes->ascii [^bytes bytes] + "returns a 16-per-line printable ascii view of the bytes" + (->> bytes + (map byte->ascii) + (partition 16 16 " ") + (map join))) + +(defn- format-hex-line [^String hex-line] + "formats a 'line' (32 hex chars) of hex output" + (->> hex-line + (partition-all 4) + (map join) + (split-at 4) + (map #(join " " %)) + (join " "))) + +(defn- bytes->hexdump [^bytes bytes] + "formats a byte array to a sequence of formatted hex lines" + (->> bytes + bytes->hex + (partition 32 32 (join (repeat 32 " "))) + (map format-hex-line))) + +(defn- copy-bytes [bytes offset size] + "utility function - copy bytes, return new byte array" + (let [size (if (nil? size) (alength bytes) size)] + (if (and (= 0 offset) (= (alength bytes) size)) + bytes ; short circuit + (java.util.Arrays/copyOfRange bytes + offset + (+ offset size))))) + +(defn get-dump-bytes [x offset size] + "utility function - return byte array from offset offset and with + size size for nio ByteBuffer, netty ByteBuf, byte array, and String" + (cond (and (satisfies? octet.buffer/IBufferBytes x) + (satisfies? octet.buffer/IBufferLimit x)) + (let [size (if (nil? size) (octet.buffer/get-capacity x) size)] + (octet.buffer/read-bytes x offset size)) + + (instance? (type (byte-array 0)) x) + (copy-bytes x offset size) + + (instance? String x) + (copy-bytes (.getBytes x) offset size))) + + +; Example usage of hex-dump +; +;(hex-dump (byte-array (range 200))) +; -------------------------------------------------------------------- +;|00000000: 0001 0203 0405 0607 0809 0a0b 0c0d 0e0f ................| +;|00000010: 1011 1213 1415 1617 1819 1a1b 1c1d 1e1f ................| +;|00000020: 2021 2223 2425 2627 2829 2a2b 2c2d 2e2f !"#$%&'()*+,-./| +;|00000030: 3031 3233 3435 3637 3839 3a3b 3c3d 3e3f 0123456789:;<=>?| +;|00000040: 4041 4243 4445 4647 4849 4a4b 4c4d 4e4f @ABCDEFGHIJKLMNO| +;|00000050: 5051 5253 5455 5657 5859 5a5b 5c5d 5e5f PQRSTUVWXYZ[\]^_| +;|00000060: 6061 6263 6465 6667 6869 6a6b 6c6d 6e6f `abcdefghijklmno| +;|00000070: 7071 7273 7475 7677 7879 7a7b 7c7d 7e7f pqrstuvwxyz{|}~| +;|00000080: 8081 8283 8485 8687 8889 8a8b 8c8d 8e8f ................| +;|00000090: 9091 9293 9495 9697 9899 9a9b 9c9d 9e9f ................| +;|000000a0: a0a1 a2a3 a4a5 a6a7 a8a9 aaab acad aeaf ................| +;|000000b0: b0b1 b2b3 b4b5 b6b7 b8b9 babb bcbd bebf ................| +;|000000c0: c0c1 c2c3 c4c5 c6c7 ........ | +; -------------------------------------------------------------------- + +(defn hex-dump + "Create hex dump. Accepts byte array, java.nio.ByteBuffer, + io.netty.buffer.ByteBuf, or String as first argument. Offset will + start the dump from an offset in the byte array, size will limit + the number of bytes dumped, and frames will print a frame around + the dump if true. Set print to true to print the dump on stdout + (default) or false to return it as a string. Example call: + (hex-dump (byte-array (range 200)) :print false)" + [x & {:keys [offset size print frame] + :or {offset 0 + print true + frame true}}] + {:pre [(not (nil? x))]} + (let [bytes (get-dump-bytes x offset size) + size (if (nil? size) (alength bytes) size) + dump (bytes->hexdump bytes) + ascii (bytes->ascii bytes) + offs (map #(format "%08x" %) + (range offset (+ offset size 16) 16)) + header (str " " (join (repeat 68 "-"))) + border (if frame "|" "") + lines (map #(str border %1 ": " %2 " " %3 border) offs dump ascii) + lines (if frame (concat [header] lines [header]) lines) + result (join \newline lines)] + (if print (println result) result))) diff --git a/test/octet/tests/core.cljc b/test/octet/tests/core.cljc index 7f0fa53..d810f96 100644 --- a/test/octet/tests/core.cljc +++ b/test/octet/tests/core.cljc @@ -32,7 +32,21 @@ (defn equals? [^bytes a ^bytes b] - (java.util.Arrays/equals a b))) + (java.util.Arrays/equals a b)) + + (def bytes-type (Class/forName "[B")) + + (defn bytes->vecs [m] + "test helper - byte arrays do not compare to + (= a b) true, so we convert maps with byte + array values to maps with vectors of bytes for + comparison" + (into {} + (map (fn [[k v]] + (if (= (type v) bytes-type) + (vector k (apply (partial vector-of :byte) v)) + (vector k v))) + m)))) :cljs (do @@ -250,6 +264,145 @@ (t/is (= readed 18)) (t/is (= data ["1234567890" 1000]))))) +(t/deftest spec-data-with-indexed-ref-string-single + (let [spec (buf/spec (buf/int32) + (buf/int32) + (buf/int32) + (buf/ref-string* 1)) + buffer (buf/allocate 15)] + (buf/write! buffer [1 3 1 "123"] spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed 15)) + (t/is (= data [1 3 1 "123"]))))) + +(t/deftest spec-data-with-indexed-ref-string-lengths + (let [lens [0 1 10 100 1000] + spec (buf/spec (buf/int32) + (buf/int32) + (buf/int32) + (buf/ref-string* 1))] + (doseq [len lens] + (let [str (clojure.string/join (repeat len \x)) + total (+ 12 len) + buffer (buf/allocate total)] + (buf/write! buffer [1 len 1 str] spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed total)) + (t/is (= data [1 len 1 str]))))))) + +(t/deftest spec-data-with-indexed-ref-string-interleaved + ; 0 1 2 3 4 5 6 + (let [datas [[22 [0 1 0 3 "a",, 0 "xyz"]] + [24 [9 3 7 3 "abc" 5 "xyz"]] + [18 [0 0 0 0 "",,, 0 ""]] + [20 [1 1 1 1 "a",, 1 "x"]] + [21 [9 0 7 3 "",,, 9 "xyz"]] + [21 [9 3 7 0 "abc" 5 ""]]] + spec (buf/spec (buf/int32) ;0 + (buf/int32) ;1 + (buf/int32) ;2 + (buf/int32) ;3 + (buf/ref-string* 1) ;4 + (buf/int16) ;5 + (buf/ref-string* 3))] ;6 + (doseq [[count data] datas] + (let [buffer (buf/allocate count)] + (buf/write! buffer data spec) + (let [[c d] (buf/read* buffer spec)] + (t/is (= d data)) + (t/is (= c count))))))) + +(t/deftest spec-data-with-assoc-ref-string-single + (let [spec (buf/spec :bogus1 (buf/int32) + :length (buf/int32) + :bogus2 (buf/int32) + :varchar (buf/ref-string* :length)) + buffer (buf/allocate 15)] + (buf/write! buffer {:bogus1 1 + :length 3 + :bogus2 1 + :varchar "123"} spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed 15)) + (t/is (= data {:bogus1 1 + :length 3 + :bogus2 1 + :varchar "123"}))))) + +(t/deftest spec-data-with-assoc-ref-strings-interleaved + (let [spec (buf/spec :bogus1 (buf/int32) + :length1 (buf/int32) + :bogus2 (buf/int32) + :length2 (buf/int32) + :varchar1 (buf/ref-string* :length1) + :bogus3 (buf/int16) + :varchar2 (buf/ref-string* :length2)) + buffer (buf/allocate 24)] + (buf/write! buffer {:bogus1 12 + :length1 3 + :bogus2 23 + :length2 3 + :varchar1 "123" + :bogus3 34 + :varchar2 "abc"} spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed 24)) + (t/is (= data {:bogus1 12 + :length1 3 + :bogus2 23 + :length2 3 + :varchar1 "123" + :bogus3 34 + :varchar2 "abc"}))))) + +(t/deftest spec-data-with-assoc-ref-bytes-single + (let [barr (byte-array [7 13 17]) + spec (buf/spec :bogus1 (buf/int32) + :length (buf/int32) + :bogus2 (buf/int32) + :varbytes (buf/ref-bytes* :length)) + buffer (buf/allocate 15)] + (buf/write! buffer {:bogus1 1 + :length 3 + :bogus2 1 + :varbytes barr} spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed 15)) + (t/is (= (bytes->vecs data) + (bytes->vecs {:bogus1 1 + :length 3 + :bogus2 1 + :varbytes barr})))))) + +(t/deftest spec-data-with-assoc-ref-bytes-interleaved + (let [barr1 (byte-array [7 13 17]) + barr2 (byte-array [19 23 29]) + spec (buf/spec :bogus1 (buf/int32) + :length1 (buf/int32) + :bogus2 (buf/int32) + :length2 (buf/int32) + :varbytes1 (buf/ref-bytes* :length1) + :bogus3 (buf/int16) + :varbytes2 (buf/ref-bytes* :length2)) + buffer (buf/allocate 24)] + (buf/write! buffer {:bogus1 12 + :length1 3 + :bogus2 23 + :length2 3 + :varbytes1 barr1 + :bogus3 34 + :varbytes2 barr2} spec) + (let [[readed data] (buf/read* buffer spec)] + (t/is (= readed 24)) + (t/is (= (bytes->vecs data) + (bytes->vecs {:bogus1 12 + :length1 3 + :bogus2 23 + :length2 3 + :varbytes1 barr1 + :bogus3 34 + :varbytes2 barr2})))))) + (t/deftest spec-composition (let [spec (buf/spec (buf/spec (buf/int32) (buf/int32)) (buf/spec (buf/string 10) (buf/string 5)))