dewailly.py

#!/usr/bin/env python
"""
Dewailly is a tool to help you add files to your personal archive.  It checks the integrity of your archive.  It also helps you process new files by checking if they already exist in your archive (duplicates) or if you have previously processed them and decided not to add them (deletions).

To verify your archive:
$ cd /my_archive
$ ls
archive_metadata.json.txt Movies/ Music/ Photos/ Projects/
$ dewailly.py --verify_archive_and_detect_added_files=.
Read 12891 files from /my_archive/dewailly.archive_metadata.json.txt
Verified all files exist and their contents and metadata are unchanged.
No new files found.
$

To prepare to add some new files to your archive:
$ cd /new_files
$ dwailly.py --verify_archive_and_detect_added_files=/my_archive --rename_known_files_check_new_and_remember_deletions=.
Read 12891 files from /my_archive/dewailly.archive_metadata.json.txt
Read 6273 deletions from /my_archive/dewailly.deletions.json.txt
File /new_files/dewailly.new_files_metadata.json.txt does not exist.  Creating it.
Found 276 files under /new_files/
Renamed 132 known files: 129 deleted, 3 changed, 0 content changed, and 0 metadata changed.
Remembered 144 new files in /new_files/dewailly.new_files_metadata.json.txt
$

Now go through through the new and renamed files.  Move some to /my_archive/ and delete the rest.  Then run the program again to update your archive's metdata file and remember the deletions.

$ dwailly.py --verify_archive_and_detect_added_files=/my_archive --rename_known_files_check_new_and_remember_deletions=.
Read 12891 files from /my_archive/dewailly.archive_metadata.json.txt
Verified all files exist and their contents and metadata are unchanged.
Found 76 new files under /my_archive.  Remembered them in /my_archive/dewailly.archive_metadata.json.txt
Read 6273 deletions from /my_archive/dewailly.deletions.json.txt
Read 144 files from /new_files/dewailly.new_files_metadata.json.txt
Found 0 files under /new_files/
Remembering 68 deletions in /my_archive/dewailly.deletions.json.txt
Deleting /new_files/new_files_metadata.json.txt since it is now empty.
$
"""
import argparse
import getopt
import hashlib
import json
import os
import os.path
import sys

class Record:
    def __init__(self, root_dir_path, relative_path, sha256, ctime, mtime, size):
        self.root_dir_path = root_dir_path
        self.relative_path = relative_path
        self.sha256 = sha256
        self.ctime = ctime
        self.mtime = mtime
        self.size = size

    def __str__(self):
        return 'Record{root_dir_path=%s relative_path=%s sha256=%s ctime=%s mtime=%s size=%s}' % (
          self.root_dir_path, self.relative_path, self.sha256, self.ctime, self.mtime, self.size)

    def toDict(self):
        return {
            'root_dir_path': self.root_dir_path,
            'relative_path': self.relative_path,
            'sha256': self.sha256,
            'ctime': self.ctime,
            'mtime': self.mtime,
            'size': self.size
            }

    @staticmethod
    def fromFile(self, root_dir_path, relative_path):
        try:
            filepath = os.path.join(root_dir_path, relative_path)
            with open(filepath, "rb") as f:
                return Record(
                    root_dir_path,
                    relative_path,
                    hashlib.sha256(f.read()).hexdigest(),
                    int(os.path.getctime(filepath)),
                    int(os.path.getmtime(filepath)),
                    os.path.getsize(filepath))
        except (e):
            raise Exception('Exception for %s %s: %s', root_dir_path, relative_path, e)


#     @staticmethod
#     def fromJsonObject(self, root_dir_path, json_object):
#         relative_path = json_object['relative_path']
#         assert relative_path is string
#         assert !relative_path.empty
#         assert !relative_path.contains('..')
#
#                                     json_object['sha256'],
#                                     json_object['ctime'],
#                                     json_object['mtime'],
#                                     json_object['size'],

def parseArgs(argv):
    parser = argparse.ArgumentParser(prog=argv[0])
    parser.add_argument(
        '--verify_archive_and_detect_added_files',
        dest='archive_dir',
        required=True,
        help='path of archive directory')
    parser.add_argument(
        '--rename_known_files_check_new_and_remember_deletions',
        dest='new_files_dir',
        required=False,
        help='path of directory with new files')
    parsed = parser.parse_args(argv[1:])
    return (os.path.abspath(parsed.archive_dir),
        (os.path.abspath(parsed.new_files_dir) if parsed.new_files_dir else None))

def raiseError(e):
    raise e

def scanDirTree(top):
    entries = []
    for dirpath, _, filenames in os.walk(top, onerror=raiseError):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            if os.path.islink(filepath) or not os.path.isfile(filepath):
                sys.stderr.write('Skipping non-file %s\n' % (repr(filepath),))
                continue
            with open(filepath, "rb") as f:
                entries.append({
                    'sha256': hashlib.sha256(f.read()).hexdigest(),
                    'filepath': os.path.relpath(filepath, top),
                    'ctime': int(os.path.getctime(filepath)),
                    'mtime': int(os.path.getmtime(filepath)),
                    'size': os.path.getsize(filepath),
                    })
    return entries

def doArchive(archive_dir):
    if not os.path.isdir(archive_dir):
        raise ValueError('archive_dir is not a directory', archive_dir)
    metadata_path = os.path.join(archive_dir, 'dewailly.archive_metadata.json.txt')
    if os.path.exists(metadata_path):
        with open(metadata_path) as f:
            metadata = json.load(f)
            print 'Read %s files from %s' % (len(metadata), metadata_path)
    else:
        print 'Creating file %s' % (metadata_path,)
        metadata = {}
    for entry in scanDirTree(archive_dir):
        
    # Verified all files exist and their contents and metadata are unchanged.
    # No new files found.


def doNew(arg):
    print 'do new', arg


if __name__ == '__main__':
    archive_dir, new_files_dir = parseArgs(sys.argv)
    if new_files_dir and (
        not os.path.relpath(archive_dir, new_files_dir).startswith('..') or
        not os.path.relpath(new_files_dir, archive_dir).startswith('..')):
        raise ValueError('one directory contains the other', archive_dir, new_files_dir)
    doArchive(archive_dir)
    if new_files_dir:
        doNew(archive_dir, new_files_dir)