Dump zone functionality

lib/node/dumpzone.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const bns = require('bns');
+const {types} = require('bns/lib/constants');
+const stream = require('stream');
+
+const NameState = require('../covenants/namestate');
+const {Resource} = require('../dns/resource');
+
+/**
+ * @typedef {import('../blockchain').Chain} Chain
+ */
+
+/**
+ * readableStream produces a newline-delimited list of all
+ * DNS records on a chain, except for RRSIG, as a utf8
+ * encoded string.
+ *
+ * @param {Chain} chain the chain to stream from
+ * @returns {stream.Readable} a readable stream of DNS records
+ */
+function readableStream(chain) {
+  const iter = chain.db.tree.iterator(true);
+
+  async function* gen() {
+    while (await iter.next()) {
+      /** @type {NameState} */
+      const ns = NameState.decode(iter.value);
+      if (ns.data.length <= 0)
+        continue;
+
+      /** @type {string} */
+      const fqdn = bns.util.fqdn(ns.name.toString('ascii'));
+
+      /** @type {Resource} */
+      const resource = Resource.decode(ns.data);
+      const zone = resource.toZone(fqdn);
+      for (const record of zone) {
+        if (record.type !== types.RRSIG && record.type !== types.TXT) {
+          yield Buffer.from(record.toString() + '\n', 'utf8');
+        }
+      }
+    }
+  }
+
+  return stream.Readable.from(gen(), {objectMode: false});
+}
+
+module.exports = {
+    readableStream: readableStream
+};

lib/node/fullnode.js

@@ -142,7 +142,11 @@ class FullNode extends Node {
       port: this.config.uint('http-port'),
       apiKey: this.config.str('api-key'),
       noAuth: this.config.bool('no-auth'),
-      cors: this.config.bool('cors')
+      cors: this.config.bool('cors'),
+      s3: {
+        bucket: this.config.str('s3-dump-bucket'),
+        key: this.config.str('s3-dump-key')
+      }
     });
 
     this.ns = new RootServer({

lib/node/http.js

@@ -21,6 +21,8 @@ const Claim = require('../primitives/claim');
 const Address = require('../primitives/address');
 const Network = require('../protocol/network');
 const pkg = require('../pkg');
+const AWS = require('aws-sdk');
+const dumpzone = require('./dumpzone');
 
 /**
  * HTTP
@@ -48,6 +50,11 @@ class HTTP extends Server {
     this.miner = this.node.miner;
     this.rpc = this.node.rpc;
 
+    this.s3 = null;
+    /** @type {AWS.S3.ManagedUpload | null} */
+    this.runningUpload = null;
+    this.uploadProgress = null;
+
     this.init();
   }
 
@@ -72,6 +79,22 @@ class HTTP extends Server {
 
     this.initRouter();
     this.initSockets();
+    this.initS3();
+  }
+
+  /**
+   * Initialize the S3 service, if possible.
+   * @private
+   */
+
+  initS3() {
+    AWS.config.getCredentials((err) => {
+      if (err)
+        this.logger.warning('couldn\'t load AWS credentials', err.stack);
+      // credentials not loaded
+      else
+        this.s3 = new AWS.S3();
+    });
   }
 
   /**
@@ -452,6 +475,79 @@ class HTTP extends Server {
 
       res.json(200, { success: true });
     });
+
+    // Initiate a zone dump to S3
+    this.post('/dump-zone-to-s3', async (req, res) => {
+      if(this.s3 === null) {
+        res.json(501,
+          {
+            success: false,
+            message: 'AWS is not properly configured'
+          }
+        );
+        return;
+      }
+
+      if(this.runningUpload !== null) {
+        res.json(202,
+          {
+            success: true,
+            message: 'upload already in progress'
+          }
+        );
+        return;
+      }
+
+      this.runningUpload = this.s3.upload({
+        Bucket: this.options.s3DumpConfig.bucket,
+        Key: this.options.s3DumpConfig.key,
+        Body: dumpzone.readableStream(this.chain)
+      }, (err, data) => {
+        // TODO - capture status, do a rename?
+        this.runningUpload = null;
+        this.uploadProgress = null;
+      });
+
+      this.runningUpload.on('httpUploadProgress', (progress) => {
+        this.uploadProgress = progress;
+      });
+
+      res.json(202, {
+        success: true,
+        message: 'upload started'
+      });
+      return;
+    });
+
+    this.get('/dump-zone-to-s3', async (req, res) => {
+      if(this.s3 === null) {
+        res.json(501,
+          {
+            success: false,
+            message: 'AWS is not properly configured'
+          }
+        );
+        return;
+      }
+
+      if(this.runningUpload === null) {
+        res.json(200,
+          {
+            success: true,
+            message: 'No upload is currently running'
+          }
+        );
+        return;
+      }
+
+      res.json(200,
+        {
+          success: true,
+          message: 'Upload in progress',
+          progress: this.uploadProgress
+        }
+      );
+    });
   }
 
   /**
@@ -894,6 +990,11 @@ class HTTPOptions {
         this.noAuth = true;
     }
 
+    if (options.s3 != null) {
+      assert(typeof options.s3 === 'object');
+      this.s3DumpConfig = options.s3;
+    }
+
     return this;
   }
 

lib/node/rpc.js

@@ -6,6 +6,7 @@
 
 'use strict';
 
+const fs = require('fs');
 const assert = require('bsert');
 const bweb = require('bweb');
 const {Lock} = require('bmutex');
@@ -42,6 +43,8 @@ const AirdropProof = require('../primitives/airdropproof');
 const {EXP} = consensus;
 const RPCBase = bweb.RPC;
 const RPCError = bweb.RPCError;
+const dumpzone = require('./dumpzone');
+const stream = require('stream');
 
 /*
  * Constants
@@ -247,6 +250,7 @@ class RPC extends RPCBase {
     this.add('validateresource', this.validateResource);
 
     this.add('resetrootcache', this.resetRootCache);
+    this.add('dumpzone', this.dumpzone);
 
     // Compat
     // this.add('getnameinfo', this.getNameInfo);
@@ -2577,6 +2581,35 @@ class RPC extends RPCBase {
     return null;
   }
 
+ async dumpzone(args, help) {
+    if (help || args.length !== 1)
+      throw new RPCError(errs.MISC_ERROR, 'dumpzone <filename>');
+
+    const valid = new Validator(args);
+    const filename = valid.str(0, null);
+    if (filename == null)
+      throw new RPCError(errs.MISC_ERROR, 'dumpzone <filename>');
+
+    const tmp = filename + '~';
+    this.logger.debug(`dumping root zone to file: ${filename}`);
+
+    return new Promise((resolve, reject) => {
+      stream.pipeline(
+        dumpzone.readableStream(this.chain),
+        fs.createWriteStream(tmp, { flags: 'a+' }),
+        (err) => {
+          if (err) {
+            reject(err);
+          } else {
+            fs.renameSync(tmp, filename);
+            this.logger.debug('root zone dump complete');
+            resolve(filename);
+          }
+        }
+      );
+    });
+  }
+
   /*
    * Helpers
    */