Handle website update #83

commands/deploy.js

@@ -5,7 +5,8 @@ import yesno from 'yesno';
 import { exit } from 'process';
 import * as cli from './cli.js'
 import AEWeb from '../lib/api.js';
-import path from 'path'
+import PathLib from 'path'
+import fetch from "cross-fetch"
 
 const { deriveAddress } = Crypto
 const { originPrivateKey, fromBigInt, uint8ArrayToHex } = Utils
@@ -23,15 +24,13 @@ const builder = {
     type: 'string',
     alias: 's',
   },
-
   endpoint: {
     describe:
       'Endpoint is the URL of a welcome node to receive the transaction',
     demandOption: true, // Required
     type: 'string',
     alias: 'e',
   },
-
   path: {
     describe: 'Path to the folder or the file to deploy',
     demandOption: true, // Required
@@ -60,6 +59,7 @@ const builder = {
 
 const handler = async function (argv) {
   try {
+    var isWebsiteUpdate = false, prevRefTxContent = undefined, transactions = [];
     // Get ssl configuration
     const {
       sslCertificate,
@@ -82,9 +82,6 @@ const handler = async function (argv) {
     const filesAddress = deriveAddress(filesSeed, 0)
 
     // Initialize endpoint connection
-    // when given endpoint ends with "/" http://192.168.1.8:4000/ results in irregular links
-    // bad link=> http://192.168.1.8:4000//api/web_hosting/address/
-    // bad link=> http://192.168.1.8:4000//explorer/transaction/000
     const endpoint = new URL(argv.endpoint).origin
 
     console.log(`Connecting to ${endpoint}`)
@@ -96,34 +93,46 @@ const handler = async function (argv) {
     const refIndex = await archethic.transaction.getTransactionIndex(refAddress)
     let filesIndex = await archethic.transaction.getTransactionIndex(filesAddress)
 
+    // Check if website is already deployed
+    if ((refIndex) !== 0) {
+      isWebsiteUpdate = true;
+      const lastRefTx = await fetchLastRefTx(refAddress, archethic.nearestEndpoints[0]);
+      prevRefTxContent = JSON.parse(lastRefTx.data.content);
+    }
+
     // Convert directory structure into array of file content
     console.log(chalk.blue('Creating file structure and compress content...'))
 
-    const aeweb = new AEWeb(archethic)
+    const aeweb = new AEWeb(archethic, prevRefTxContent)
     const files = cli.getFiles(folderPath, includeGitIgnoredFiles)
 
-    if (files.length === 0) throw 'folder "' + path.basename(folderPath) + '" is empty'
+    if (files.length === 0) throw 'folder "' + PathLib.basename(folderPath) + '" is empty'
 
     files.forEach(({ filePath, data }) => aeweb.addFile(filePath, data))
 
+    if (isWebsiteUpdate) await logUpdateInfo(aeweb)
+
     // Create transaction
     console.log(chalk.blue('Creating transactions ...'))
 
-    let transactions = aeweb.getFilesTransactions()
+    if (!isWebsiteUpdate || (aeweb.listModifiedFiles().length)) {
+      // when files changes does exist
 
-    // Sign files transactions
-    transactions = transactions.map(tx => {
-      const index = filesIndex
-      filesIndex++
-      return tx.build(filesSeed, index).originSign(originPrivateKey)
-    })
+      transactions = aeweb.getFilesTransactions()
+
+      // Sign files transactions
+      transactions = transactions.map(tx => {
+        const index = filesIndex
+        filesIndex++
+        return tx.build(filesSeed, index).originSign(originPrivateKey)
+      })
+    }
 
     aeweb.addSSLCertificate(sslCertificate, sslKey)
-    const refTx = await aeweb.getRefTransaction(transactions)
+    const refTx = await aeweb.getRefTransaction(transactions);
 
     // Sign ref transaction
     refTx.build(refSeed, refIndex).originSign(originPrivateKey)
-
     transactions.push(refTx)
 
     // Estimate fees
@@ -135,7 +144,9 @@ const handler = async function (argv) {
     const transferTx = archethic.transaction.new()
       .setType('transfer')
       .addUCOTransfer(refAddress, refTxFees)
-      .addUCOTransfer(filesAddress, filesTxFees)
+
+    // handle no new files tx, but update to ref tx
+    if (filesTxFees) transferTx.addUCOTransfer(filesAddress, filesTxFees)
 
     transferTx.build(baseSeed, baseIndex).originSign(originPrivateKey)
 
@@ -227,6 +238,89 @@ async function sendTransactions(transactions, index, endpoint) {
   })
 }
 
+
+async function fetchLastRefTx(txnAddress, endpoint) {
+  if (typeof txnAddress !== "string" && !(txnAddress instanceof Uint8Array)) {
+    throw "'address' must be a string or Uint8Array";
+  }
+
+  if (typeof txnAddress == "string") {
+    if (!isHex(txnAddress)) {
+      throw "'address' must be in hexadecimal form if it's string";
+    }
+  }
+
+  if (txnAddress instanceof Uint8Array) {
+    txnAddress = uint8ArrayToHex(txnAddress);
+  }
+  const url = new URL("/api", endpoint);
+  const query =
+    `query {
+                lastTransaction(
+                    address: "${txnAddress}"
+                    ){
+                        data{
+                                content
+                            }
+                    }
+            }`
+
+  return fetch(url,
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Accept": "application/json"
+      },
+      body:
+        JSON.stringify({
+          query
+        })
+    })
+    .then(handleResponse)
+    .then(({
+      data: { lastTransaction: data }
+    }) => {
+      return data;
+    })
+}
+
+function handleResponse(response) {
+  return new Promise(function (resolve, reject) {
+    if (response.status >= 200 && response.status <= 299) {
+      response.json().then(resolve);
+    } else {
+      reject(response.statusText);
+    }
+  });
+}
+
+async function logUpdateInfo(aeweb) {
+
+  let modifiedFiles = aeweb.listModifiedFiles();
+  let removedFiles = aeweb.listRemovedFiles();
+
+  if (!modifiedFiles.length && !removedFiles.length) { throw 'No files to update' }
+
+  console.log(
+    chalk.greenBright
+      (`
+            Found ${modifiedFiles.length} New/Modified files 
+            Found ${removedFiles.length} Removed files
+      `));
+
+  if (await yesno({
+    question: chalk.yellowBright('Do you want to List Changes. (yes/no)'
+    ),
+  })) {
+    console.log(chalk.blue('New/Modified files:'))
+    modifiedFiles.forEach((file_path) => { console.log(chalk.green(`     ${file_path}`)) })
+
+    console.log(chalk.blue('Removed files:'))
+    removedFiles.forEach((file_path) => { console.log(chalk.red(`     ${file_path}     `)) })
+  }
+}
+
 export default {
   command,
   describe,

lib/api.js

@@ -1,41 +1,50 @@
 import zlib from 'zlib'
-import isEqual from 'lodash/isEqual.js'
-import path from 'path'
-import Archethic, { Crypto, Utils } from 'archethic'
+import Archethic, { Crypto } from 'archethic'
+import {
+  hashContent, getFilePath, MAX_FILE_SIZE, handleBigFile, handleNormalFile,
+  getRefTxContent
+} from "./utils.js"
 
 const { aesEncrypt, ecEncrypt, randomSecretKey } = Crypto
-const { uint8ArrayToHex } = Utils
-import nativeCryptoLib from 'crypto'
-
-// 3_145_728 represent the maximum size for a transaction
-// 45_728 represent json tree size
-const MAX_FILE_SIZE = 3_145_728 - 45_728
-const AEWEB_VERSION = 1
-const HASH_FUNCTION = 'sha1'
-
 export default class AEWeb {
-  constructor(archethic) {
+
+  constructor(archethic, prevRefTxContent = undefined) {
     if (!(archethic instanceof Archethic)) {
       throw 'archethic is not an instance of Archethic'
     }
+
+    if (prevRefTxContent !== undefined) this.prevRefTxMetaData = prevRefTxContent.metaData
     this.archethic = archethic
     this.txsContent = []
     this.metaData = {}
+    this.modifiedFiles = []
   }
 
   addFile(naivePath, data) {
     const size = Buffer.byteLength(data)
     if (size === 0) return
 
-    const hash = nativeCryptoLib.createHash(HASH_FUNCTION).update(data).digest('hex')
-    const content = zlib.gzipSync(data).toString('base64url')
+    const hash = hashContent(data)
+    const filePath = getFilePath(naivePath)
+
+    if (this.prevRefTxMetaData !== undefined) {
+      // A website update operation
+      const prevFileMetaData = this.prevRefTxMetaData[filePath];
+      delete (this.prevRefTxMetaData[filePath]);
 
-    const tabPath = naivePath.split(path.sep)
-    if (tabPath[0] === '') tabPath.shift()
-    const filePath = tabPath.join('/')
+      if (prevFileMetaData != undefined && prevFileMetaData["hash"] == hash) {
+        // File exists and  not changed
+        // priority given to new files/metadata
+        this.metaData[filePath] = prevFileMetaData;
+        return;
+      }
+    }
+    // File has changed or is new
+    this.modifiedFiles.push(filePath);
 
     this.metaData[filePath] = { hash: hash, size: size, encoding: 'gzip', addresses: [] }
 
+    const content = zlib.gzipSync(data).toString('base64url')
     // Handle file over than Max size. The file is splitted in multiple transactions,
     // firsts parts take a full transaction, the last part follow the normal sized file construction
     if (content.length >= MAX_FILE_SIZE) {
@@ -62,11 +71,11 @@ export default class AEWeb {
 
       return tx
     })
+
   }
 
   async getRefTransaction(transactions) {
-    const { metaData, refContent } = getMetaData(this.txsContent, transactions, this.metaData, this.sslCertificate)
-    this.metaData = metaData
+    const refContent = getRefTxContent(this.txsContent, transactions, this.metaData, this.sslCertificate)
 
     const refTx = this.archethic.transaction.new()
       .setType('hosting')
@@ -80,7 +89,6 @@ export default class AEWeb {
 
       refTx.addOwnership(encryptedSslKey, [{ publicKey: storageNoncePublicKey, encryptedSecretKey: encryptedSecretKey }])
     }
-
     return refTx
   }
 
@@ -90,96 +98,13 @@ export default class AEWeb {
     this.sslKey = undefined
     this.metaData = {}
   }
-}
 
-function handleBigFile(txsContent, filePath, content) {
-  while (content.length > 0) {
-    // Split the file
-    const part = content.slice(0, MAX_FILE_SIZE)
-    content = content.replace(part, '')
-    // Set the value in transaction content
-    const txContent = {
-      content: {},
-      size: part.length,
-      refPath: [],
-    }
-    txContent.content[filePath] = part
-    txContent.refPath.push(filePath)
-    txsContent.push(txContent)
+  listModifiedFiles() {
+    return this.modifiedFiles;
   }
-}
 
-function handleNormalFile(txsContent, filePath, content) {
-  // 4 x "inverted commas + 1x :Colon + 1x ,Comma + 1x space = 7
-  const fileSize = content.length + filePath.length + 7
-  // Get first transaction content that can be filled with file content
-  const txContent = getContentToFill(txsContent, fileSize)
-  const index = txsContent.indexOf(txContent)
-
-  txContent.content[filePath] = content
-  txContent.refPath.push(filePath)
-  txContent.size += fileSize
-
-  if (index === -1) {
-    // Push new transaction
-    txsContent.push(txContent)
-  } else {
-    // Update existing transaction
-    txsContent.splice(index, 1, txContent)
+  listRemovedFiles() {
+    return Object.keys(this.prevRefTxMetaData);
   }
 }
 
-function getContentToFill(txsContent, contentSize) {
-  const content = txsContent.find(txContent => (txContent.size + contentSize) <= MAX_FILE_SIZE)
-  if (content) {
-    return content
-  } else {
-    return {
-      content: {},
-      size: 0,
-      refPath: []
-    }
-  }
-}
-
-function getMetaData(txsContent, transactions, metaData, sslCertificate) {
-  // For each transaction
-  transactions.forEach((tx) => {
-    if (!tx.address) throw 'Transaction is not built'
-
-    const txContent = txsContent.find(val => isEqual(val.content, tx.data.content))
-
-    if (!txContent) throw 'Transaction content not expected'
-    // For each filePath
-    const address = uint8ArrayToHex(tx.address)
-    // Update the metadata at filePath with address
-    return txContent.refPath.forEach((filePath) => {
-      const { addresses } = metaData[filePath]
-      addresses.push(address)
-      metaData[filePath]['addresses'] = addresses
-    })
-  })
-
-  metaData = sortObject(metaData)
-
-  const ref = {
-    aewebVersion: AEWEB_VERSION,
-    hashFunction: HASH_FUNCTION,
-    metaData: metaData
-  }
-
-  if (sslCertificate) {
-    ref.sslCertificate = sslCertificate
-  }
-
-  return { metadata: metaData, refContent: JSON.stringify((ref)) }
-}
-
-function sortObject(obj) {
-  return Object.keys(obj)
-    .sort()
-    .reduce(function (acc, key) {
-      acc[key] = obj[key]
-      return acc
-    }, {})
-}