Change document serialization format for hashing and signatures

README.md

@@ -398,36 +398,22 @@ let unsub = storage.onChange.subscribe(() => console.log('something changed'));
 // Later, you can turn off your subscription.
 unsub();
 ```
+
 ----
+
 ## Details and notes
 
+**NOTE** The rest of this README is somewhat out of date!
+
 ### Signatures
 
-Document hashes are used within signatures and to link to specific versions of a doc.
+Document hashes are used within signatures and could be used to link to specific versions of a doc.
 
-There's a simple canonical way to hash a doc: mostly you just concat all the fields in a predefined order:
-```ts
-    // How to hash a document (from es4.ts)
-
-    // Fields in alphabetical order.
-    // Convert numbers to strings.
-    // Replace optional properties with '' if they're missing.
-    // Use the contentHash instead of the content.
-    return sha256([
-        doc.author,
-        doc.contentHash,
-        doc.deleteAfter === undefined ? '' : '' + doc.deleteAfter,
-        doc.format,
-        doc.path,
-        '' + doc.timestamp,
-        doc.workspace,
-    ].join('\n'));
-```
-None of those fields are allowed to contain newlines (except content, which is hashed for that reason) so newlines are safe to use as a delimiter.
+There's a simple canonical way to hash a document.  See [docs/serialization-and-hashing.md](Serialization and Hashing) for details.
 
 To sign a doc, you sign its hash with the author's secret key.
 
-Note that docs only ever have to get transformed INTO this representation just before hashing -- we never need to convert from this representation BACK into a real doc.
+Note that docs only ever have to get transformed INTO this representation just before hashing -- we never need to convert from this representation BACK into a real doc -- so it can be nice and simple.
 
 There is no canonical encoding for storage or networking - only the canonical hash encoding, above.  Databases and network code can represent the doc in any way they want.
 
@@ -436,6 +422,7 @@ The hash and signature specification may change as the schema evolves beyond `es
 ### Sync over duplex streams:
 
 Here's a very simple but inefficient algorithm to start with:
+
 ```
     sort paths by (path, timestamp DESC, signature ASC)
     filter by my interest query
@@ -448,6 +435,7 @@ Here's a very simple but inefficient algorithm to start with:
 ### Sync over HTTP when only one peer is publicly available:
 
 Here's a very simple but inefficient algorithm to start with:
+
 ```
     the client side is in charge and does these actions:
     sort paths by (path, timestamp DESC, signature ASC)

docs/serialization-and-hashing.md

@@ -1,4 +1,4 @@
-# Serialization and Hashing
+# Serialization and Hashing In "es.4" Format
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->

src/test/validator.es4.test.ts

@@ -72,7 +72,19 @@ t.test('hashDocument', (t: any) => {
         author: '@suzy.xxxxxxxxxxx',
         signature: 'xxxxxxxxxxxxx',
     };
-    t.equal(Val.hashDocument(doc1), 'bcvqc4nz2a5fuxpjoxum2fwollrdp7w3mjf4crpbftyltuojshwlq', 'expected document hash');
+    t.equal(Val.hashDocument(doc1), 'bz6ye6gvzo7w6igkht3qqn4jvrp5qehvcmo5kyp3gldnbbmdy7vdq', 'expected document hash, no deleteAfter');
+    let doc2: Document = {
+        format: 'es.4',
+        workspace: '+gardenclub.xxxxxxxxxxxxxxxxxxxx',
+        path: '/path1',
+        contentHash: sha256base32('content1'),
+        content: 'content1',
+        timestamp: 1,
+        deleteAfter: 2,  // with deleteAfter
+        author: '@suzy.xxxxxxxxxxx',
+        signature: 'xxxxxxxxxxxxx',
+    };
+    t.equal(Val.hashDocument(doc2), 'bl3yoc4h4iubuev5izxr4trnxrfhnmdoqy2uciajq73quvu22vyna', 'expected document hash, with deleteAfter');
     t.done();
 });
 

src/validator/es4.ts

@@ -48,26 +48,28 @@ export const ValidatorEs4 : IValidatorES4 = class {
         // The hash of the document is used for signatures and references to specific docs.
         // We use the hash of the content in case we want to drop the actual content
         // and only keep the hash around for verifying signatures.
-        // None of these fields are allowed to contain newlines
-        // except for content, but content is hashed, so it's safe to
-        // use newlines as a field separator.
+        // None of these fields are allowed to contain tabs or newlines
+        // (except content, but we use contentHash instead).
 
         let err = this._checkBasicDocumentValidity(doc);
         if (isErr(err)) { return err; }
 
         // Fields in alphabetical order.
         // Convert numbers to strings.
-        // Replace optional properties with '' if they're missing.
+        // Omit optional properties if they're missing.
         // Use the contentHash instead of the content.
-        return sha256base32([
-            doc.author,
-            doc.contentHash,
-            doc.deleteAfter === undefined ? '' : '' + doc.deleteAfter,
-            doc.format,
-            doc.path,
-            '' + doc.timestamp,
-            doc.workspace,
-        ].join('\n'));
+        // Omit the signature.
+        return sha256base32(
+            `author\t${doc.author}\n` +
+            // (omit content itself)
+            `contentHash\t${doc.contentHash}\n` +
+            (doc.deleteAfter === undefined ? '' : `deleteAfter\t${doc.deleteAfter}\n`) +
+            `format\t${doc.format}\n` +
+            `path\t${doc.path}\n` +
+            // (omit signature)
+            `timestamp\t${doc.timestamp}\n` +
+            `workspace\t${doc.workspace}\n`  // \n at the end also, not just between
+        );
     }
     static signDocument(keypair: AuthorKeypair, doc: Document): Document | ValidationError {
         // Add an author signature to the document.