Baseline redundancy branch

README.md

@@ -199,12 +199,12 @@ Then select the options to upload/download/audit files while connecting to node1
   - `batchain -l`
 4. You can always run `batchain -h` to review available command and options
 
-## Note:
+#### Note:
 
-For `npm`: 
-1. Run `npm install -g` before running any `batchain` option or command, make sure to 
+For `npm`:
+1. Run `npm install -g` before running any `batchain` option or command, make sure to
 2. Need to run `npm install -g` when making bin changes
-3. If "chalk" is not working for you, run `npm insatll chalk --save` to make the command line more colorful
+3. If "chalk" is not working for you, run `npm install chalk --save` to make the command line more colorful
 
 For `yarn`:
 1. Run `yarn link` to create a symbolic link between project directory and executable command
@@ -223,3 +223,51 @@ For `yarn`:
     -h, --help  output usage information
     -l, --list  view your list of uploaded files in BatChain network
   ```
+
+#### Local CLI demo 2 - upload and audit a file
+
+First step is to make some temporary changes to allow the code to run locally
+
+Uncomment the seed node information and comment out the remote seed node info. The file should end up looking like this:
+
+```
+// For network testing:
+// exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: '167.99.2.1', port: 80 }];
+// exports.CLI_SERVER = {host: 'localhost', port: 1800};
+// exports.BATNODE_SERVER_PORT = 1900;
+// exports.KADNODE_PORT = 80;
+
+// For local testing
+exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }]
+exports.BASELINE_REDUNDANCY = 3;
+```
+
+Next, change this line of code in the `while` loop
+
+```
+getClosestBatNodeToShard(shardId, callback){
+  this.kadenceNode.iterativeFindNode(shardId, (err, res) => {
+    let i = 0
+    let targetKadNode = res[0]; // res is an array of these tuples: [id, {hostname, port}]
+    while (targetKadNode[1].hostname === this.kadenceNode.contact.hostname &&
+          (targetKadNode[1].port === this.kadenceNode.contact.port) {
+```
+
+to this.
+
+```
+    // while (targetKadNode[1].hostname === this.kadenceNode.contact.hostname &&
+    while (targetKadNode[1].port === this.kadenceNode.contact.port) {
+```
+
+Now we can proceed with the demo.
+
+1. `cd` into `/audit` directory
+2. If you haven't already, run `yarn link`  to create a symbolic link between project directory and executable command. This only needs to be done once.
+3. Open 3 additional terminal windows or tabs that are also in the `/audit` directory
+4. In the first terminal, `cd` into `server` directory. Run `rm/db` first and then run `node node.js`
+5. In the second terminal, `cd` into `server2` directory. Run `rm/db` first and then run `node node.js`
+6. In the third terminal, `cd` into `client` directory. Run `rm/db` first and then run `node node.js`. This boots up the CLI server which will listen for CLI commands. Wait for a message to log out saying the CLI is ready before issuing any commands.
+7. In the fourth terminal, `cd` into `client` as well. Here we can issue `batchain` CLI commands.
+8. There should be a example file in the `personal` directory, so run `batchain -u ./personal/example.txt`. Wait a few seconds for the 24 or so shard files to be written to `server` and `server2` `/host` directories.
+9. Kill the process manually (Control-C) and run `batchain -a ./manifest/$MANIFESTNAME.batchain`. Replace `$MANIFESTNAME` with the manifest file name generated on `client/manifest` directory.

audit/client/node.js

@@ -1,14 +1,14 @@
-const bunyan = require('bunyan');
 const levelup = require('levelup');
 const leveldown = require('leveldown');
 const encoding = require('encoding-down');
 const kad = require('@kadenceproject/kadence');
 const BatNode = require('../../batnode').BatNode;
 const kad_bat = require('../../kadence_plugin').kad_bat;
 const seed = require('../../constants').SEED_NODE
+const backoff = require('backoff');
 
 // Create a third batnode kadnode pair
-kadnode3 = new kad.KademliaNode({
+const kadnode3 = new kad.KademliaNode({
   transport: new kad.HTTPTransport(),
   storage: levelup(encoding(leveldown('./dbbb'))),
   contact: { hostname: 'localhost', port: 1252 }
@@ -19,12 +19,67 @@ kadnode3.plugin(kad_bat)
 kadnode3.listen(1252)
 const batnode3 = new BatNode(kadnode3)
 kadnode3.batNode = batnode3
-batnode3.createServer(1985, '127.0.0.1')
 
 // Join
 kadnode3.join(seed, () => {
   console.log('you have joined the network! Ready to accept commands from the CLI!')
-  batnode3.uploadFile('./personal/example.txt');
+  // batnode3.uploadFile('./personal/example.txt');
   // batnode3.retrieveFile('./manifest/85a2ea0f0d11634d334886d9fb073b0d64506199.batchain')
   // batnode3.auditFile('./manifest/85a2ea0f0d11634d334886d9fb073b0d64506199.batchain')
 })
+
+const nodeCLIConnectionCallback = (serverConnection) => {
+
+  serverConnection.on('data', (data) => {
+    let receivedData = JSON.parse(data);
+
+    if (receivedData.messageType === "CLI_UPLOAD_FILE") {
+      let filePath = receivedData.filePath;
+
+      batnode3.uploadFile(filePath);
+    } else if (receivedData.messageType === "CLI_DOWNLOAD_FILE") {
+      let filePath = receivedData.filePath;
+
+      batnode3.retrieveFile(filePath);
+    } else if (receivedData.messageType === "CLI_AUDIT_FILE") {
+      let filePath = receivedData.filePath;
+      let fibonacciBackoff = backoff.exponential({
+          randomisationFactor: 0,
+          initialDelay: 20,
+          maxDelay: 2000
+      });
+      console.log("received path: ", filePath);
+      batnode3.auditFile(filePath);
+
+      // post audit cleanup
+      serverConnection.on('close', () => {
+        batnode3._audit.ready = false;
+        batnode3._audit.data = null;
+        batnode3._audit.passed = false;
+      });
+
+      fibonacciBackoff.failAfter(10);
+
+      fibonacciBackoff.on('backoff', function(number, delay) {
+          console.log(number + ' ' + delay + 'ms');
+      });
+
+      fibonacciBackoff.on('ready', function() {
+        if (!batnode3._audit.ready) {
+          fibonacciBackoff.backoff();
+        } else {
+          serverConnection.write(JSON.stringify(batnode3._audit.passed));
+          return;
+        }
+      });
+
+      fibonacciBackoff.on('fail', function() {
+        console.log('Timeout: failed to complete audit');
+      });
+
+      fibonacciBackoff.backoff();
+    }
+  });
+}
+
+batnode3.createCLIServer(1800, 'localhost', nodeCLIConnectionCallback);

audit/server/node.js

@@ -29,25 +29,27 @@ const nodeConnectionCallback = (serverConnection) => {
   });
 
   serverConnection.on('data', (receivedData, error) => {
+    if (error) { throw error; }
     receivedData = JSON.parse(receivedData)
     if (receivedData.messageType === "RETRIEVE_FILE") {
-      batnode1.readFile(`./hosted/${receivedData.fileName}`, (error, data) => {
+      batnode1.readFile(`./hosted/${receivedData.fileName}`, (err, data) => {
+        if (err) { throw err; }
         serverConnection.write(data)
       });
     } else if (receivedData.messageType === "STORE_FILE") {
       let fileName = receivedData.fileName;
-      batnode1.kadenceNode.iterativeStore(fileName, [batnode1.kadenceNode.identity.toString(), batnode1.kadenceNode.contact], (err, stored) => {
+      batnode1.kadenceNode.iterativeStore(fileName, [batnode1.kadenceNode.identity.toString(), batnode1.kadenceNode.contact], (innerError, stored) => {
+        if (innerError) { throw innerError; }
         console.log('nodes who stored this value: ', stored)
         let fileContent = new Buffer(receivedData.fileContent)
         batnode1.writeFile(`./hosted/${fileName}`, fileContent, (err) => {
-          if (err) {
-            throw err;
-          }
+          if (err) { throw err; }
           serverConnection.write(JSON.stringify({messageType: "SUCCESS"}))
         });
       });
     } else if (receivedData.messageType === "AUDIT_FILE") {
-      fs.readFile(`./hosted/${receivedData.fileName}`, (error, data) => {
+      fs.readFile(`./hosted/${receivedData.fileName}`, (innerErr, data) => {
+        if (innerErr) { throw innerErr; }
         const shardSha1 = fileUtils.sha1HashData(data);
         serverConnection.write(shardSha1);
       });

audit/server2/node.js

@@ -1,4 +1,3 @@
-const bunyan = require('bunyan');
 const levelup = require('levelup');
 const leveldown = require('leveldown');
 const encoding = require('encoding-down');
@@ -27,27 +26,27 @@ const nodeConnectionCallback = (serverConnection) => {
     console.log('end')
   })
   serverConnection.on('data', (receivedData, error) => {
+    if (error) { throw error; }
     receivedData = JSON.parse(receivedData)
 
     if (receivedData.messageType === "RETRIEVE_FILE") {
-      batnode2.readFile(`./hosted/${receivedData.fileName}`, (error, data) => {
-        if (error) { throw error; }
+      batnode2.readFile(`./hosted/${receivedData.fileName}`, (err, data) => {
+        if (err) { throw err; }
         serverConnection.write(data)
       })
     } else if (receivedData.messageType === "STORE_FILE") {
       let fileName = receivedData.fileName
       batnode2.kadenceNode.iterativeStore(fileName, [batnode2.kadenceNode.identity.toString(), batnode2.kadenceNode.contact], (err, stored) => {
         console.log('nodes who stored this value: ', stored)
         let fileContent = new Buffer(receivedData.fileContent)
-        batnode2.writeFile(`./hosted/${fileName}`, fileContent, (err) => {
-          if (err) {
-            throw err;
-          }
+        batnode2.writeFile(`./hosted/${fileName}`, fileContent, (innerError) => {
+          if (innerError) { throw innerError; }
           serverConnection.write(JSON.stringify({messageType: "SUCCESS"}))
         })
       });
     } else if (receivedData.messageType === "AUDIT_FILE") {
-      fs.readFile(`./hosted/${receivedData.fileName}`, (error, data) => {
+      fs.readFile(`./hosted/${receivedData.fileName}`, (innerErr, data) => {
+        if (innerErr) { throw innerErr; }
         console.log('AUDIT_FILE - data: ', data);
         const shardSha1 = fileUtils.sha1HashData(data);
         console.log('shardSha1: ', shardSha1);

batnode.js

@@ -4,10 +4,12 @@ const path = require('path');
 const PERSONAL_DIR = require('./utils/file').PERSONAL_DIR;
 const HOSTED_DIR = require('./utils/file').HOSTED_DIR;
 const fs = require('fs');
+const constants = require('./constants');
 
 class BatNode {
   constructor(kadenceNode = {}) {
     this._kadenceNode = kadenceNode;
+    this._audit = { ready: false, data: null, passed: false };
     fileUtils.generateEnvFile()
   }
 
@@ -206,10 +208,17 @@ class BatNode {
     const manifest = fileUtils.loadManifest(manifestFilePath);
     const shards = manifest.chunks;
     const shaIds = Object.keys(shards);
-    const fileName = manifest.fileName;
+    const shardAuditData = this.prepareAuditData(shards, shaIds);
     let shaIdx = 0;
 
-    const shardAuditData = shaIds.reduce((acc, shaId) => {
+    while (shaIds.length > shaIdx) {
+      this.auditShardsGroup(shards, shaIds, shaIdx, shardAuditData);
+      shaIdx += 1;
+    }
+  }
+
+  prepareAuditData(shards, shaIds) {
+    return shaIds.reduce((acc, shaId) => {
       acc[shaId] = {};
 
       shards[shaId].forEach((shardId) => {
@@ -218,11 +227,6 @@ class BatNode {
 
       return acc;
     }, {});
-
-    while (shaIds.length > shaIdx) {
-      this.auditShardsGroup(shards, shaIds, shaIdx, shardAuditData);
-      shaIdx += 1;
-    }
   }
   /**
    * Tests the redudant copies of the original shard for data integrity.
@@ -232,23 +236,24 @@ class BatNode {
    * @param {shardAuditData} Object - same as shards param except instead of an
    * array of shard ids it's an object of shard ids and their audit status
   */
-  auditShardsGroup(shards, shaIds, shaIdx, shardAuditData) {
+  auditShardsGroup(shards, shaIds, shaIdx, shardAuditData, done) {
     let shardDupIdx = 0;
-    let duplicatesAudited = 0;
     const shaId = shaIds[shaIdx];
 
     while (shards[shaId].length > shardDupIdx) {
-      this.auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData);
+      this.auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData, done);
       shardDupIdx += 1;
     }
   }
 
-  auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData) {
+  auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData, done) {
     const shardId = shards[shaId][shardDupIdx];
 
-    this.kadenceNode.iterativeFindValue(shardId, (err, value, responder) => {
+    this.kadenceNode.iterativeFindValue(shardId, (error, value, responder) => {
+      if (error) { throw error; }
       let kadNodeTarget = value.value;
-      this.kadenceNode.getOtherBatNodeContact(kadNodeTarget, (error, batNode) => {
+      this.kadenceNode.getOtherBatNodeContact(kadNodeTarget, (err, batNode) => {
+        if (err) { throw err; }
         this.auditShardData(batNode, shards, shaIdx, shardDupIdx, shardAuditData)
       })
     })
@@ -281,24 +286,33 @@ class BatNode {
       }
 
       if (finalShaGroup && finalShard) {
-        this.auditResults(shardAuditData, shaKeys);
+        const hasBaselineRedundancy = this.auditResults(shardAuditData, shaKeys);
+        this._audit.ready = true;
+        this._audit.data = shardAuditData;
+        this._audit.passed = hasBaselineRedundancy;
+
+        console.log(shardAuditData);
+        if (hasBaselineRedundancy) {
+          console.log('Passed audit!');
+        } else {
+          console.log('Failed Audit');
+        }
       }
     })
   }
 
-  auditResults(shardAuditData, shaKeys) {
-    const dataValid = shaKeys.every((shaId) => {
-      // For each key in the values object for the shaId key
-      return Object.keys(shardAuditData[shaId]).every((shardId) => {
-        return shardAuditData[shaId][shardId] === true;
-      })
-    });
-    console.log(shardAuditData);
-    if (dataValid) {
-      console.log('Passed audit!');
-    } else {
-      console.log('Failed Audit');
+  auditResults(auditData, shaKeys) {
+    const isRedundant = (shaId) => {
+      let validShards = 0;
+      // For each key (shardId) under the shard content's shaId key
+      Object.keys(auditData[shaId]).forEach((shardId) => {
+        if (auditData[shaId][shardId] === true) { validShards += 1; }
+      });
+
+      return validShards >= constants.BASELINE_REDUNDANCY ? true : false;
     }
+
+    return shaKeys.every(isRedundant);
   }
 
 }

bin/index.js

@@ -48,10 +48,14 @@ function sendAuditMessage() {
     messageType: "CLI_AUDIT_FILE",
     filePath: batchain.audit,
   };
-
-  console.log("message: ", message);
-
+
   client.write(JSON.stringify(message));
+
+  client.on('data', (data, error) => {
+    if (error) { throw error; }
+    const manifest = fileSystem.loadManifest(batchain.audit);
+    console.log(`File name: ${manifest.fileName} | Manifest: ${batchain.audit} | Data integrity: ${data.toString('utf8')}`);
+  })
 }
 
 function displayFileList() {

constants.js

@@ -5,4 +5,5 @@ exports.BATNODE_SERVER_PORT = 1900;
 exports.KADNODE_PORT = 80;
 
 // For local testing
-//exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }]
+// exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }]
+exports.BASELINE_REDUNDANCY = 3;

package.json

@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "@kadenceproject/kadence": "^3.1.2",
+    "backoff": "^2.5.0",
     "chalk": "^2.3.2",
     "dotenv": "^5.0.1",
     "public-ip": "^2.4.0"