Data below a threshold should not be stored in DataStore.

README.md

@@ -3,7 +3,7 @@
 # Decentralized Web Node (DWN) SDK <!-- omit in toc -->
 
 Code Coverage
-![Statements](https://img.shields.io/badge/statements-97.51%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-94.35%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.86%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-97.51%25-brightgreen.svg?style=flat)
+![Statements](https://img.shields.io/badge/statements-97.53%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-94.42%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.88%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-97.53%25-brightgreen.svg?style=flat)
 
 
 - [Introduction](#introduction)

src/core/dwn-constant.ts

@@ -1,6 +1,7 @@
 export class DwnConstant {
   /**
    * The maximum size in bytes of raw data that will be returned as `encodedData`.
+   * this is also the maximum size that we will store within MessageStore.
    */
-  public static readonly maxDataSizeAllowedToBeEncoded = 10_000;
+  public static readonly maxDataSizeAllowedToBeEncoded = 50_000;
 }

src/core/dwn-error.ts

@@ -48,7 +48,8 @@ export enum DwnErrorCode {
   RecordsWriteDataCidMismatch = 'RecordsWriteDataCidMismatch',
   RecordsWriteDataSizeMismatch = 'RecordsWriteDataSizeMismatch',
   RecordsWriteMissingAuthorizationSignatureInput = 'RecordsWriteMissingAuthorizationSignatureInput',
-  RecordsWriteMissingData = 'RecordsWriterMissingData',
+  RecordsWriteMissingDataInPrevious = 'RecordsWriteMissingDataInPrevious',
+  RecordsWriteMissingDataAssociation = 'RecordsWriteMissingDataAssociation',
   RecordsWriteMissingDataStream = 'RecordsWriteMissingDataStream',
   RecordsWriteMissingProtocol = 'RecordsWriteMissingProtocol',
   RecordsWriteMissingSchema = 'RecordsWriteMissingSchema',

src/core/message.ts

@@ -86,7 +86,14 @@ export abstract class Message<M extends GenericMessage> {
     // the message will contain properties that should not be part of the CID computation
     // and we need to strip them out (like `encodedData` that we historically had for a long time),
     // but we can remove this method entirely if the code becomes stable and it is apparent that the wrapper is not needed
-    const cid = await Cid.computeCid(message);
+
+    // ^--- seems like we might need to keep this around for now.
+    const rawMessage = { ...message } as any;
+    if (rawMessage.encodedData) {
+      delete rawMessage.encodedData;
+    }
+
+    const cid = await Cid.computeCid(rawMessage as GenericMessage);
     return cid;
   }
 

src/handlers/messages-get.ts

@@ -2,16 +2,13 @@ import type { DataStore } from '../types/data-store.js';
 import type { DidResolver } from '../did/did-resolver.js';
 import type { MessageStore } from '../types/message-store.js';
 import type { MethodHandler } from '../types/method-handler.js';
-import type { RecordsWriteMessage } from '../types/records-types.js';
+import type { RecordsWriteMessageWithOptionalEncodedData } from '../store/storage-controller.js';
 import type { MessagesGetMessage, MessagesGetReply, MessagesGetReplyEntry } from '../types/messages-types.js';
 
-import { DataStream } from '../utils/data-stream.js';
-import { DwnConstant } from '../core/dwn-constant.js';
-import { Encoder } from '../utils/encoder.js';
 import { messageReplyFromError } from '../core/message-reply.js';
 import { MessagesGet } from '../interfaces/messages-get.js';
 import { authenticate, authorize } from '../core/auth.js';
-import { DwnInterfaceName, DwnMethodName, Message } from '../core/message.js';
+import { DwnInterfaceName, DwnMethodName } from '../core/message.js';
 
 type HandleArgs = { tenant: string, message: MessagesGetMessage };
 
@@ -54,7 +51,6 @@ export class MessagesGetHandler implements MethodHandler {
     // for every message, include associated data as `encodedData` IF:
     //  * its a RecordsWrite
     //  * the data size is equal or smaller than the size threshold
-    //! NOTE: this is somewhat duplicate code that also exists in `StorageController.query`.
     for (const entry of messages) {
       const { message } = entry;
 
@@ -67,19 +63,12 @@ export class MessagesGetHandler implements MethodHandler {
         continue;
       }
 
-      // RecordsWrite specific handling
-      const recordsWrite = message as RecordsWriteMessage;
-      const dataCid = recordsWrite.descriptor.dataCid;
-      const dataSize = recordsWrite.descriptor.dataSize;
-
-      if (dataCid !== undefined && dataSize! <= DwnConstant.maxDataSizeAllowedToBeEncoded) {
-        const messageCid = await Message.getCid(message);
-        const result = await this.dataStore.get(tenant, messageCid, dataCid);
-
-        if (result) {
-          const dataBytes = await DataStream.toBytes(result.dataStream);
-          entry.encodedData = Encoder.bytesToBase64Url(dataBytes);
-        }
+      // RecordsWrite specific handling, if MessageStore has embedded `encodedData` return it with the entry.
+      // we store `encodedData` along with the message if the data is below a certain threshold.
+      const recordsWrite = message as RecordsWriteMessageWithOptionalEncodedData;
+      if (recordsWrite.encodedData !== undefined) {
+        entry.encodedData = recordsWrite.encodedData;
+        delete recordsWrite.encodedData;
       }
     }
 

src/handlers/records-query.ts

@@ -6,7 +6,6 @@ import type { RecordsQueryMessage, RecordsQueryReply, RecordsQueryReplyEntry, Re
 import { authenticate } from '../core/auth.js';
 import { lexicographicalCompare } from '../utils/string.js';
 import { messageReplyFromError } from '../core/message-reply.js';
-import { StorageController } from '../store/storage-controller.js';
 
 import { DateSort, RecordsQuery } from '../interfaces/records-query.js';
 import { DwnInterfaceName, DwnMethodName } from '../core/message.js';
@@ -86,7 +85,8 @@ export class RecordsQueryHandler implements MethodHandler {
       method            : DwnMethodName.Write,
       isLatestBaseState : true
     };
-    const records = await StorageController.query(this.messageStore, this.dataStore, tenant, filter);
+
+    const records = (await this.messageStore.query(tenant, filter)) as RecordsWriteMessageWithOptionalEncodedData[];
     return records;
   }
 
@@ -133,7 +133,7 @@ export class RecordsQueryHandler implements MethodHandler {
       published         : true,
       isLatestBaseState : true
     };
-    const publishedRecords = await StorageController.query(this.messageStore, this.dataStore, tenant, filter);
+    const publishedRecords = (await this.messageStore.query(tenant, filter)) as RecordsWriteMessageWithOptionalEncodedData[];
     return publishedRecords;
   }
 
@@ -152,8 +152,7 @@ export class RecordsQueryHandler implements MethodHandler {
       isLatestBaseState : true,
       published         : false
     };
-    const unpublishedRecordsForQueryAuthor = await StorageController.query(this.messageStore, this.dataStore, tenant, filter);
-
+    const unpublishedRecordsForQueryAuthor = (await this.messageStore.query(tenant, filter)) as RecordsWriteMessageWithOptionalEncodedData[];
     return unpublishedRecordsForQueryAuthor;
   }
 
@@ -172,7 +171,7 @@ export class RecordsQueryHandler implements MethodHandler {
       isLatestBaseState : true,
       published         : false
     };
-    const unpublishedRecordsForQueryAuthor = await StorageController.query(this.messageStore, this.dataStore, tenant, filter);
+    const unpublishedRecordsForQueryAuthor = (await this.messageStore.query(tenant, filter)) as RecordsWriteMessageWithOptionalEncodedData[];
     return unpublishedRecordsForQueryAuthor;
   }
 }

src/handlers/records-read.ts

@@ -1,13 +1,15 @@
 import type { MethodHandler } from '../types/method-handler.js';
+import type { RecordsWriteMessageWithOptionalEncodedData } from '../store/storage-controller.js';
 import type { TimestampedMessage } from '../types/message-types.js';
 import type { DataStore, DidResolver, MessageStore } from '../index.js';
-import type { RecordsReadMessage, RecordsReadReply, RecordsWriteMessage } from '../types/records-types.js';
+import type { RecordsReadMessage, RecordsReadReply } from '../types/records-types.js';
 
 import { authenticate } from '../core/auth.js';
 import { Message } from '../core/message.js';
 import { messageReplyFromError } from '../core/message-reply.js';
 import { RecordsRead } from '../interfaces/records-read.js';
 import { RecordsWrite } from '../interfaces/records-write.js';
+import { DataStream, Encoder } from '../index.js';
 import { DwnInterfaceName, DwnMethodName } from '../core/message.js';
 
 export class RecordsReadHandler implements MethodHandler {
@@ -51,28 +53,35 @@ export class RecordsReadHandler implements MethodHandler {
       };
     }
 
-    const newestRecordsWrite = newestExistingMessage as RecordsWriteMessage;
+    const newestRecordsWrite = newestExistingMessage as RecordsWriteMessageWithOptionalEncodedData;
     try {
       await recordsRead.authorize(tenant, await RecordsWrite.parse(newestRecordsWrite), this.messageStore);
     } catch (error) {
       return messageReplyFromError(error, 401);
     }
 
-    const messageCid = await Message.getCid(newestRecordsWrite);
-    const result = await this.dataStore.get(tenant, messageCid, newestRecordsWrite.descriptor.dataCid);
-
-    if (result?.dataStream === undefined) {
-      return {
-        status: { code: 404, detail: 'Not Found' }
-      };
+    let data;
+    if (newestRecordsWrite.encodedData !== undefined) {
+      const dataBytes = Encoder.base64UrlToBytes(newestRecordsWrite.encodedData);
+      data = DataStream.fromBytes(dataBytes);
+      delete newestRecordsWrite.encodedData;
+    } else {
+      const messageCid = await Message.getCid(newestRecordsWrite);
+      const result = await this.dataStore.get(tenant, messageCid, newestRecordsWrite.descriptor.dataCid);
+      if (result?.dataStream === undefined) {
+        return {
+          status: { code: 404, detail: 'Not Found' }
+        };
+      }
+      data = result.dataStream;
     }
 
     const { authorization: _, ...recordsWriteWithoutAuthorization } = newestRecordsWrite; // a trick to stripping away `authorization`
     const messageReply: RecordsReadReply ={
       status : { code: 200, detail: 'OK' },
       record : {
         ...recordsWriteWithoutAuthorization,
-        data: result.dataStream
+        data,
       }
     };
     return messageReply;