Refactor sage-uncompress-spkg, add tests

Author: vbraun

build/bin/sage-uncompress-spkg

@@ -1,264 +1,23 @@
 #!/usr/bin/env python
 
-"""
-USAGE:
-
-  sage-uncompress-spkg [-d DIR] PKG [FILE]
-
-With a single argument, unpack the file PKG to the current directory.
-
-If a directory is specified with the -d option the contents of
-the archive are extracted into that directory using the following
-rules:
-
-  1. If the archive contains (like most) a top-level directory
-     which contains all other files in the archive, the contents
-     of that directory are extracted into DIR, ignoring the name
-     of the top-level directory in the archive.
-
-  2. If the archive does not contain a single top-level directory
-     then all the contents of the archive are extracted into DIR.
-
-The directory must not already exist.
-
-If FILE is specified, extract FILE from PKG and send it to
-stdout. (This option is present only for backwards compatibility:
-printing the SPKG.txt file from an old-style spkg.)
-"""
-
-from __future__ import print_function
-
-import argparse
-import copy
-import os
-import stat
-import sys
-import tarfile
-import zipfile
-
-
-def filter_os_files(filenames):
-    """
-    Given a list of filenames, returns a filtered list with OS-specific
-    special files removed.
-
-    Currently removes OSX .DS_Store files and AppleDouble format ._ files.
-    """
-
-    files_set = set(filenames)
-
-    def is_os_file(path):
-        dirname, name = os.path.split(path)
-
-        if name == '.DS_Store':
-            return True
-
-        if name.startswith('._'):
-            name = os.path.join(dirname, name[2:])
-            # These files store extended attributes on OSX
-            # In principle this could be a false positive but it's
-            # unlikely, and to be really sure we'd have to extract the file
-            # (or at least the first four bytes to check for the magic number
-            # documented in
-            # http://kaiser-edv.de/documents/AppleSingle_AppleDouble.pdf)
-            if name in files_set or os.path.normpath(name) in files_set:
-                return True
-
-        return False
-
-    filenames = filter(lambda f: not is_os_file(f), filenames)
-
-    if sys.version_info[0] == 2:
-        return filenames
-    else:
-        # Make sure to return a list on Python >= 3
-        return list(filenames)
-
-
-class SageTarFile(tarfile.TarFile):
-    """
-    Sage as tarfile.TarFile, but applies the user's current umask to the
-    permissions of all extracted files and directories.
-
-    This mimics the default behavior of the ``tar`` utility.
-
-    See http://trac.sagemath.org/ticket/20218#comment:16 for more background.
-    """
-
-    def __new__(cls, *args, **kwargs):
-        # This is is that SageTarFile() is equivalent to TarFile.open() which
-        # is more flexible than the basic TarFile.__init__
-        inst = tarfile.TarFile.open(*args, **kwargs)
-        inst.__class__ = cls
-        return inst
-
-    def __init__(self, *args, **kwargs):
-        # Unfortunately the only way to get the current umask is to set it
-        # and then restore it
-        self.umask = os.umask(0o777)
-        os.umask(self.umask)
-
-    @classmethod
-    def can_read(cls, filename):
-        """
-        Given an archive filename, returns True if this class can read and
-        process the archive format of that file.
-        """
-
-        return tarfile.is_tarfile(filename)
-
-    @property
-    def names(self):
-        """
-        List of filenames in the archive.
-
-        Filters out names of OS-related files that shouldn't be in the
-        archive (.DS_Store, etc.)
-        """
-
-        return filter_os_files(self.getnames())
-
-    def chmod(self, tarinfo, target):
-        tarinfo = copy.copy(tarinfo)
-        tarinfo.mode &= ~self.umask
-        tarinfo.mode &= ~(stat.S_ISUID | stat.S_ISGID)
-        return super(SageTarFile, self).chmod(tarinfo, target)
-
-    def extractall(self, path='.', members=None):
-        """
-        Same as tarfile.TarFile.extractall but allows filenames for
-        the members argument (like zipfile.ZipFile).
-        """
-        if members:
-            name_to_member = dict([member.name, member] for member in self.getmembers())
-            members = [m if isinstance(m, tarfile.TarInfo)
-                       else name_to_member[m]
-                       for m in members]
-        return super(SageTarFile, self).extractall(path=path, members=members)
-
-    def extractbytes(self, member):
-        """
-        Return the contents of the specified archive member as bytes.
-
-        If the member does not exist, returns None.
-        """
-
-        if member in self.getnames():
-            reader = self.extractfile(member)
-            return reader.read()
-
-
-class SageZipFile(zipfile.ZipFile):
-    """
-    Wrapper for zipfile.ZipFile to provide better API fidelity with
-    SageTarFile insofar as it's used by this script.
-    """
-
-    @classmethod
-    def can_read(cls, filename):
-        """
-        Given an archive filename, returns True if this class can read and
-        process the archive format of that file.
-        """
-
-        return zipfile.is_zipfile(filename)
-
-    @property
-    def names(self):
-        """
-        List of filenames in the archive.
-
-        Filters out names of OS-related files that shouldn't be in the
-        archive (.DS_Store, etc.)
-        """
-
-        return filter_os_files(self.namelist())
-
-    def extractbytes(self, member):
-        """
-        Return the contents of the specified archive member as bytes.
-
-        If the member does not exist, returns None.
-        """
-
-        if member in self.namelist():
-            return self.read(member)
-
-
-ARCHIVE_TYPES = [SageTarFile, SageZipFile]
-
-
-def main(argv=None):
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-d', dest='dir', metavar='DIR',
-                        help='directory to extract archive contents into')
-    parser.add_argument('pkg', nargs=1, metavar='PKG',
-                        help='the archive to extract')
-    parser.add_argument('file', nargs='?', metavar='FILE',
-                        help='(deprecated) print the contents of the given '
-                             'archive member to stdout')
-
-    args = parser.parse_args(argv)
-
-    filename = args.pkg[0]
-    dirname = args.dir
-
-    for cls in ARCHIVE_TYPES:
-        if cls.can_read(filename):
-            break
-    else:
-        print('Error: Unknown file type: {}'.format(filename),
-              file=sys.stderr)
-        return 1
-
-    # For now ZipFile and TarFile both have default open modes that are
-    # acceptable
-    archive = cls(filename)
-
-    if args.file:
-        contents = archive.extractbytes(args.file)
-        if contents:
-            print(contents, end='')
-            return 0
-        else:
-            return 1
-
-    top_level = None
-
-    if dirname:
-        if os.path.exists(dirname):
-            print('Error: Directory {} already exists'.format(dirname),
-                  file=sys.stderr)
-            return 1
-
-        top_levels = set()
-        for member in archive.names:
-            # Zip and tar files all use forward slashes as separators
-            # internally
-            top_levels.add(member.split('/', 1)[0])
-
-        if len(top_levels) == 1:
-            top_level = top_levels.pop()
-        else:
-            os.makedirs(dirname)
-
-    prev_cwd = os.getcwd()
-
-    if dirname and not top_level:
-        # We want to extract content into dirname, but there is not
-        # a single top-level directory for the tarball, so we cd into
-        # the extraction target first
-        os.chdir(dirname)
-
-    try:
-        archive.extractall(members=archive.names)
-        if dirname and top_level:
-            os.rename(top_level, dirname)
-    finally:
-        os.chdir(prev_cwd)
-
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())
+# usage: sage-uncompress-spkg [-h] [-d DIR] PKG [FILE]
+# 
+# positional arguments:
+#   PKG         the archive to extract
+#   FILE        (deprecated) print the contents of the given archive member to
+#               stdout
+# 
+# optional arguments:
+#   -h, --help  show this help message and exit
+#   -d DIR      directory to extract archive contents into
+
+
+try:
+    import sage_bootstrap
+except ImportError:
+    import os, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+    import sage_bootstrap
+
+from sage_bootstrap.uncompress.cmdline import run
+run()

build/sage_bootstrap/cmdline.py

@@ -22,8 +22,12 @@
 import logging
 log = logging.getLogger()
 
+
 # Note that argparse is not part of Python 2.6, so we bundle it
-from sage_bootstrap.compat import argparse
+try:
+    import argparse
+except ImportError:
+    from sage_bootstrap.compat import argparse
 
 from sage_bootstrap.app import Application
 

build/sage_bootstrap/download/cmdline.py

@@ -22,7 +22,10 @@
 log = logging.getLogger()
 
 # Note that argparse is not part of Python 2.6, so we bundle it
-from sage_bootstrap.compat import argparse
+try:
+    import argparse
+except ImportError:
+    from sage_bootstrap.compat import argparse
 
 from sage_bootstrap.download.app import Application
 from sage_bootstrap.env import SAGE_DISTFILES

build/sage_bootstrap/uncompress/__init__.py

@@ -0,0 +1,71 @@
+"""
+"""
+
+#*****************************************************************************
+#       Copyright (C) 2016 Volker Braun <vbraun.name@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+#*****************************************************************************
+
+from __future__ import print_function
+
+import os
+
+from sage_bootstrap.uncompress.tar_file import SageTarFile
+from sage_bootstrap.uncompress.zip_file import SageZipFile
+
+ARCHIVE_TYPES = [SageTarFile, SageZipFile]
+
+
+
+def open_archive(filename):
+    """
+    Automatically detect archive type
+    """
+    for cls in ARCHIVE_TYPES:
+        if cls.can_read(filename):
+            break
+    else:
+        raise ValueError
+
+    # For now ZipFile and TarFile both have default open modes that are
+    # acceptable
+    return cls(filename)
+
+
+def unpack_archive(archive, dirname=None):
+    """
+    Unpack archive
+    """
+    top_level = None
+
+    if dirname:
+        top_levels = set()
+        for member in archive.names:
+            # Zip and tar files all use forward slashes as separators
+            # internally
+            top_levels.add(member.split('/', 1)[0])
+
+        if len(top_levels) == 1:
+            top_level = top_levels.pop()
+        else:
+            os.makedirs(dirname)
+
+    prev_cwd = os.getcwd()
+
+    if dirname and not top_level:
+        # We want to extract content into dirname, but there is not
+        # a single top-level directory for the tarball, so we cd into
+        # the extraction target first
+        os.chdir(dirname)
+
+    try:
+        archive.extractall(members=archive.names)
+        if dirname and top_level:
+            os.rename(top_level, dirname)
+    finally:
+        os.chdir(prev_cwd)