Merge pull request #1 from ZeroX-DG/ehhc-scan-unused

Allow user to view attachments and clear unused attachments

browser/lib/utils.js

@@ -136,9 +136,24 @@ export function isMarkdownTitleURL (str) {
   return /(^#{1,6}\s)(?:\w+:|^)\/\/(?:[^\s\.]+\.\S{2}|localhost[\:?\d]*)/.test(str)
 }
 
+export function humanFileSize (bytes) {
+  const threshold = 1000
+  if (Math.abs(bytes) < threshold) {
+    return bytes + ' B'
+  }
+  var units = ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+  var u = -1
+  do {
+    bytes /= threshold
+    ++u
+  } while (Math.abs(bytes) >= threshold && u < units.length - 1)
+  return bytes.toFixed(1) + ' ' + units[u]
+}
+
 export default {
   lastFindInArray,
   escapeHtmlCharacters,
   isObjectEqual,
-  isMarkdownTitleURL
+  isMarkdownTitleURL,
+  humanFileSize
 }

browser/main/lib/dataApi/attachmentManagement.js

@@ -624,6 +624,76 @@ function deleteAttachmentsNotPresentInNote (markdownContent, storageKey, noteKey
   }
 }
 
+/**
+ * @description Get all existing attachments related to a specific note
+ including their status (in use or not) and their path. Return null if there're no attachment related to note or specified parametters are invalid
+ * @param markdownContent markdownContent of the current note
+ * @param storageKey StorageKey of the current note
+ * @param noteKey NoteKey of the currentNote
+ * @return {Promise<Array<{path: String, isInUse: bool}>>} Promise returning the
+ list of attachments with their properties */
+function getAttachmentsPathAndStatus (markdownContent, storageKey, noteKey) {
+  if (storageKey == null || noteKey == null || markdownContent == null) {
+    return
+  }
+  const targetStorage = findStorage.findStorage(storageKey)
+  const attachmentFolder = path.join(targetStorage.path, DESTINATION_FOLDER, noteKey)
+  const attachmentsInNote = getAttachmentsInMarkdownContent(markdownContent)
+  const attachmentsInNoteOnlyFileNames = []
+  if (attachmentsInNote) {
+    for (let i = 0; i < attachmentsInNote.length; i++) {
+      attachmentsInNoteOnlyFileNames.push(attachmentsInNote[i].replace(new RegExp(STORAGE_FOLDER_PLACEHOLDER + escapeStringRegexp(path.sep) + noteKey + escapeStringRegexp(path.sep), 'g'), ''))
+    }
+  }
+  if (fs.existsSync(attachmentFolder)) {
+    return new Promise((resolve, reject) => {
+      fs.readdir(attachmentFolder, (err, files) => {
+        if (err) {
+          console.error('Error reading directory "' + attachmentFolder + '". Error:')
+          console.error(err)
+          reject(err)
+          return
+        }
+        const attachments = []
+        for (const file of files) {
+          const absolutePathOfFile = path.join(targetStorage.path, DESTINATION_FOLDER, noteKey, file)
+          if (!attachmentsInNoteOnlyFileNames.includes(file)) {
+            attachments.push({ path: absolutePathOfFile, isInUse: false })
+          } else {
+            attachments.push({ path: absolutePathOfFile, isInUse: true })
+          }
+        }
+        resolve(attachments)
+      })
+    })
+  } else {
+    return null
+  }
+}
+
+/**
+ * @description Remove all specified attachment paths
+ * @param attachments attachment paths
+ * @return {Promise} Promise after all attachments are removed */
+function removeAttachmentsByPaths (attachments) {
+  const promises = []
+  for (const attachment of attachments) {
+    const promise = new Promise((resolve, reject) => {
+      fs.unlink(attachment, (err) => {
+        if (err) {
+          console.error('Could not delete "%s"', attachment)
+          console.error(err)
+          reject(err)
+          return
+        }
+        resolve()
+      })
+    })
+    promises.push(promise)
+  }
+  return Promise.all(promises)
+}
+
 /**
  * Clones the attachments of a given note.
  * Copies the attachments to their new destination and updates the content of the new note so that the attachment-links again point to the correct destination.
@@ -726,8 +796,10 @@ module.exports = {
   getAbsolutePathsOfAttachmentsInContent,
   importAttachments,
   removeStorageAndNoteReferences,
+  removeAttachmentsByPaths,
   deleteAttachmentFolder,
   deleteAttachmentsNotPresentInNote,
+  getAttachmentsPathAndStatus,
   moveAttachments,
   cloneAttachments,
   isAttachmentLink,

browser/main/modals/PreferencesModal/StoragesTab.js

@@ -3,8 +3,11 @@ import React from 'react'
 import CSSModules from 'browser/lib/CSSModules'
 import styles from './StoragesTab.styl'
 import dataApi from 'browser/main/lib/dataApi'
+import attachmentManagement from 'browser/main/lib/dataApi/attachmentManagement'
 import StorageItem from './StorageItem'
 import i18n from 'browser/lib/i18n'
+import { humanFileSize } from 'browser/lib/utils'
+import fs from 'fs'
 
 const electron = require('electron')
 const { shell, remote } = electron
@@ -35,8 +38,29 @@ class StoragesTab extends React.Component {
         name: 'Unnamed',
         type: 'FILESYSTEM',
         path: ''
-      }
+      },
+      attachments: []
     }
+    this.loadAttachmentStorage()
+  }
+
+  loadAttachmentStorage () {
+    const promises = []
+    this.props.data.noteMap.map(note => {
+      const promise = attachmentManagement.getAttachmentsPathAndStatus(
+        note.content,
+        note.storage,
+        note.key
+      )
+      if (promise) promises.push(promise)
+    })
+
+    Promise.all(promises)
+      .then(data => {
+        const result = data.reduce((acc, curr) => acc.concat(curr), [])
+        this.setState({attachments: result})
+      })
+      .catch(console.error)
   }
 
   handleAddStorageButton (e) {
@@ -57,8 +81,39 @@ class StoragesTab extends React.Component {
     e.preventDefault()
   }
 
+  handleRemoveUnusedAttachments (attachments) {
+    attachmentManagement.removeAttachmentsByPaths(attachments)
+      .then(() => this.loadAttachmentStorage())
+      .catch(console.error)
+  }
+
   renderList () {
     const { data, boundingBox } = this.props
+    const { attachments } = this.state
+
+    const unusedAttachments = attachments.filter(attachment => !attachment.isInUse)
+    const inUseAttachments = attachments.filter(attachment => attachment.isInUse)
+
+    const totalUnusedAttachments = unusedAttachments.length
+    const totalInuseAttachments = inUseAttachments.length
+    const totalAttachments = totalUnusedAttachments + totalInuseAttachments
+
+    const totalUnusedAttachmentsSize = unusedAttachments
+      .reduce((acc, curr) => {
+        const stats = fs.statSync(curr.path)
+        const fileSizeInBytes = stats.size
+        return acc + fileSizeInBytes
+      }, 0)
+    const totalInuseAttachmentsSize = inUseAttachments
+      .reduce((acc, curr) => {
+        const stats = fs.statSync(curr.path)
+        const fileSizeInBytes = stats.size
+        return acc + fileSizeInBytes
+      }, 0)
+    const totalAttachmentsSize = totalUnusedAttachmentsSize + totalInuseAttachmentsSize
+
+    const unusedAttachmentPaths = unusedAttachments
+      .reduce((acc, curr) => acc.concat(curr.path), [])
 
     if (!boundingBox) { return null }
     const storageList = data.storageMap.map((storage) => {
@@ -82,6 +137,20 @@ class StoragesTab extends React.Component {
             <i className='fa fa-plus' /> {i18n.__('Add Storage Location')}
           </button>
         </div>
+        <div styleName='header'>{i18n.__('Attachment storage')}</div>
+        <p styleName='list-attachment-label'>
+          Unused attachments size: {humanFileSize(totalUnusedAttachmentsSize)} ({totalUnusedAttachments} items)
+        </p>
+        <p styleName='list-attachment-label'>
+          In use attachments size: {humanFileSize(totalInuseAttachmentsSize)} ({totalInuseAttachments} items)
+        </p>
+        <p styleName='list-attachment-label'>
+          Total attachments size: {humanFileSize(totalAttachmentsSize)} ({totalAttachments} items)
+        </p>
+        <button styleName='list-attachement-clear-button'
+          onClick={() => this.handleRemoveUnusedAttachments(unusedAttachmentPaths)}>
+          {i18n.__('Clear unused attachments')}
+        </button>
       </div>
     )
   }

browser/main/modals/PreferencesModal/StoragesTab.styl

@@ -33,6 +33,17 @@
   colorDefaultButton()
   font-size $tab--button-font-size
   border-radius 2px
+.list-attachment-label
+  margin-bottom 10px
+  color $ui-text-color
+.list-attachement-clear-button
+  height 30px
+  border none
+  border-top-right-radius 2px
+  border-bottom-right-radius 2px
+  colorPrimaryButton()
+  vertical-align middle
+  padding 0 20px
 
 .addStorage
   margin-bottom 15px
@@ -154,8 +165,8 @@ body[data-theme="dark"]
   .addStorage-body-control-cancelButton
     colorDarkDefaultButton()
     border-color $ui-dark-borderColor
-
-
+  .list-attachement-clear-button
+    colorDarkPrimaryButton()
 
 body[data-theme="solarized-dark"]
   .root
@@ -194,6 +205,8 @@ body[data-theme="solarized-dark"]
   .addStorage-body-control-cancelButton
     colorDarkDefaultButton()
     border-color $ui-solarized-dark-borderColor
+  .list-attachement-clear-button
+    colorSolarizedDarkPrimaryButton()
 
 body[data-theme="monokai"]
   .root
@@ -232,6 +245,8 @@ body[data-theme="monokai"]
   .addStorage-body-control-cancelButton
     colorDarkDefaultButton()
     border-color $ui-monokai-borderColor
+  .list-attachement-clear-button
+    colorMonokaiPrimaryButton()
 
 body[data-theme="dracula"]
   .root
@@ -269,4 +284,6 @@ body[data-theme="dracula"]
     colorDarkPrimaryButton()
   .addStorage-body-control-cancelButton
     colorDarkDefaultButton()
-    border-color $ui-dracula-borderColor
+    border-color $ui-dracula-borderColor
+  .list-attachement-clear-button
+    colorDraculaPrimaryButton()

tests/dataApi/attachmentManagement.test.js

@@ -578,6 +578,72 @@ it('should test that deleteAttachmentsNotPresentInNote does nothing if noteKey,
   expect(fs.unlink).not.toHaveBeenCalled()
 })
 
+it('should test that getAttachmentsPathAndStatus return null if noteKey, storageKey or noteContent was undefined', function () {
+  const noteKey = undefined
+  const storageKey = undefined
+  const markdownContent = ''
+
+  const result = systemUnderTest.getAttachmentsPathAndStatus(markdownContent, storageKey, noteKey)
+  expect(result).toBeNull()
+})
+
+it('should test that getAttachmentsPathAndStatus return null if noteKey, storageKey or noteContent was null', function () {
+  const noteKey = null
+  const storageKey = null
+  const markdownContent = ''
+
+  const result = systemUnderTest.getAttachmentsPathAndStatus(markdownContent, storageKey, noteKey)
+  expect(result).toBeNull()
+})
+
+it('should test that getAttachmentsPathAndStatus return the correct path and status for attachments', async function () {
+  const dummyStorage = {path: 'dummyStoragePath'}
+  const noteKey = 'noteKey'
+  const storageKey = 'storageKey'
+  const markdownContent =
+    'Test input' +
+    '![' + systemUnderTest.STORAGE_FOLDER_PLACEHOLDER + path.win32.sep + noteKey + path.win32.sep + 'file2.pdf](file2.pdf) \n'
+  const dummyFilesInFolder = ['file1.txt', 'file2.pdf', 'file3.jpg']
+
+  findStorage.findStorage = jest.fn(() => dummyStorage)
+  fs.existsSync = jest.fn(() => true)
+  fs.readdir = jest.fn((paht, callback) => callback(undefined, dummyFilesInFolder))
+  fs.unlink = jest.fn()
+
+  const targetStorage = findStorage.findStorage(storageKey)
+
+  const attachments = await systemUnderTest.getAttachmentsPathAndStatus(markdownContent, storageKey, noteKey)
+  expect(attachments.length).toBe(3)
+  expect(attachments[0].isInUse).toBe(false)
+  expect(attachments[1].isInUse).toBe(true)
+  expect(attachments[2].isInUse).toBe(false)
+
+  expect(attachments[0].path).toBe(
+    path.join(
+      targetStorage.path,
+      systemUnderTest.DESTINATION_FOLDER,
+      noteKey,
+      dummyFilesInFolder[0]
+    )
+  )
+  expect(attachments[1].path).toBe(
+    path.join(
+      targetStorage.path,
+      systemUnderTest.DESTINATION_FOLDER,
+      noteKey,
+      dummyFilesInFolder[1]
+    )
+  )
+  expect(attachments[2].path).toBe(
+    path.join(
+      targetStorage.path,
+      systemUnderTest.DESTINATION_FOLDER,
+      noteKey,
+      dummyFilesInFolder[2]
+    )
+  )
+})
+
 it('should test that moveAttachments moves attachments only if the source folder existed', function () {
   fse.existsSync = jest.fn(() => false)
   fse.moveSync = jest.fn()