Support Zip64 when compressing iterables and strings

.bumpversion.cfg

@@ -0,0 +1,11 @@
+[bumpversion]
+current_version = 1.1.8
+commit = true
+message = Release {new_version}
+tag = true
+
+[bumpversion:file:CHANGELOG.md]
+search = Unreleased
+replace = v{new_version} ({now:%Y-%m-%d})
+
+[bumpversion:file:setup.py]

CHANGELOG.md

@@ -0,0 +1,15 @@
+# ChangeLog
+
+This file details the changes that were made after forking v1.1.4 from https://github.com/allanlei/python-zipstream.
+
+## v1.1.8 (2020-09-14)
+* New datetime parameter in write_iter (https://github.com/arjan-s/python-zipstream/pull/8)
+
+## v1.1.7 (2019-10-22)
+* Stream data in the order it was received (https://github.com/arjan-s/python-zipstream/pull/4)
+
+## v1.1.6 (2019-06-06)
+* Add partial flushing of ZipStreams (https://github.com/arjan-s/python-zipstream/pull/1)
+
+## v1.1.5 (2019-03-18)
+* Support Zip64 when compressing iterables and strings (https://github.com/allanlei/python-zipstream/pull/25)

README.markdown → README.md

@@ -1,9 +1,6 @@
 
 # python-zipstream
 
-[![Build Status](https://travis-ci.org/allanlei/python-zipstream.png?branch=master)](https://travis-ci.org/allanlei/python-zipstream)
-[![Coverage Status](https://coveralls.io/repos/allanlei/python-zipstream/badge.png)](https://coveralls.io/r/allanlei/python-zipstream)
-
 zipstream.py is a zip archive generator based on python 3.3's zipfile.py. It was created to
 generate a zip file generator for streaming (ie web apps). This is beneficial for when you
 want to provide a downloadable archive of a large collection of regular files, which would be infeasible to
@@ -78,12 +75,12 @@ archives.
 ## Installation
 
 ```
-pip install zipstream
+pip install zipstream-new
 ```
 
 ## Requirements
 
- * Python 2.6, 2.7, 3.2, 3.3, pypy
+ * Python 2.6+, 3.2+, pypy
 
 ## Examples
 
@@ -95,7 +92,7 @@ from flask import Response
 @app.route('/package.zip', methods=['GET'], endpoint='zipball')
 def zipball():
  def generator():
- z = zipstream.ZipFile(mode='w', compression=ZIP_DEFLATED)
+ z = zipstream.ZipFile(mode='w', compression=zipstream.ZIP_DEFLATED)
 
  z.write('/path/to/file')
 
@@ -110,12 +107,33 @@ def zipball():
 
 @app.route('/package.zip', methods=['GET'], endpoint='zipball')
 def zipball():
- z = zipstream.ZipFile(mode='w', compression=ZIP_DEFLATED)
+ z = zipstream.ZipFile(mode='w', compression=zipstream.ZIP_DEFLATED)
  z.write('/path/to/file')
 
  response = Response(z, mimetype='application/zip')
  response.headers['Content-Disposition'] = 'attachment; filename={}'.format('files.zip')
  return response
+
+# Partial flushing of the zip before closing
+
+@app.route('/package.zip', methods=['GET'], endpoint='zipball')
+def zipball():
+ def generate_zip_with_manifest():
+ z = zipstream.ZipFile(mode='w', compression=zipstream.ZIP_DEFLATED)
+
+ manifest = []
+ for filename in os.listdir('/path/to/files'):
+ z.write(os.path.join('/path/to/files', filename), arcname=filename)
+ yield from z.flush()
+ manifest.append(filename)
+
+ z.write_str('manifest.json', json.dumps(manifest).encode())
+
+ yield from z
+
+ response = Response(generate_zip_with_manifest(), mimetype='application/zip')
+ response.headers['Content-Disposition'] = 'attachment; filename={}'.format('files.zip')
+ return response
 ```
 
 ### django 1.5+
@@ -124,7 +142,7 @@ def zipball():
 from django.http import StreamingHttpResponse
 
 def zipball(request):
- z = zipstream.ZipFile(mode='w', compression=ZIP_DEFLATED)
+ z = zipstream.ZipFile(mode='w', compression=zipstream.ZIP_DEFLATED)
  z.write('/path/to/file')
 
  response = StreamingHttpResponse(z, content_type='application/zip')
@@ -149,3 +167,5 @@ def GET(self):
 With python version > 2.6, just run the following command: `python -m unittest discover`
 
 Alternatively, you can use `nose`.
+
+If you want to run the tests on all supported Python versions, run `tox`.

setup.py

@@ -2,15 +2,26 @@
 from setuptools import setup, find_packages
 
 
+with open("README.md", "r") as fh:
+ long_description = fh.read()
+
 setup(
- name='zipstream',
- version='1.1.4',
- description='Zipfile generator',
- author='Allan Lei',
- author_email='allanlei@helveticode.com',
- url='https://github.com/allanlei/python-zipstream',
+ name='zipstream-new',
+ version='1.1.8',
+ description='Zipfile generator that takes input files as well as streams',
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ author='arjan5',
+ author_email='arjan@anymore.nl',
+ url='https://github.com/arjan-s/python-zipstream',
  packages=find_packages(exclude=['tests']),
  keywords='zip streaming',
  test_suite='nose.collector',
  tests_require=['nose'],
+ classifiers=[
+ "Programming Language :: Python",
+ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+ "Operating System :: OS Independent",
+ "Topic :: System :: Archiving :: Compression",
+ ],
 )

tests/test_zipstream.py

@@ -3,6 +3,7 @@
 
 import os
 import tempfile
+import time
 import unittest
 import zipstream
 import zipfile
@@ -76,6 +77,28 @@ def string_generator():
 
  os.remove(f.name)
 
+ def test_write_iterable_with_date_time(self):
+ file_name_in_zip = "data_datetime"
+ file_date_time_in_zip = time.strptime("2011-04-19 22:30:21", "%Y-%m-%d %H:%M:%S")
+
+ z = zipstream.ZipFile(mode='w')
+ def string_generator():
+ for _ in range(10):
+ yield b'zipstream\x01\n'
+ z.write_iter(iterable=string_generator(), arcname=file_name_in_zip, date_time=file_date_time_in_zip)
+
+ f = tempfile.NamedTemporaryFile(suffix='zip', delete=False)
+ for chunk in z:
+ f.write(chunk)
+ f.close()
+
+ z2 = zipfile.ZipFile(f.name, 'r')
+ self.assertFalse(z2.testzip())
+
+ self.assertEqual(file_date_time_in_zip[0:5], z2.getinfo(file_name_in_zip).date_time[0:5])
+
+ os.remove(f.name)
+
  def test_writestr(self):
  z = zipstream.ZipFile(mode='w')
 
@@ -92,6 +115,31 @@ def test_writestr(self):
 
  os.remove(f.name)
 
+ def test_partial_writes(self):
+ z = zipstream.ZipFile(mode='w')
+ f = tempfile.NamedTemporaryFile(suffix='zip', delete=False)
+
+ with open(SAMPLE_FILE_RTF, 'rb') as fp:
+ z.writestr('sample1.rtf', fp.read())
+
+ for chunk in z.flush():
+ f.write(chunk)
+
+ with open(SAMPLE_FILE_RTF, 'rb') as fp:
+ z.writestr('sample2.rtf', fp.read())
+
+ for chunk in z.flush():
+ f.write(chunk)
+
+ for chunk in z:
+ f.write(chunk)
+
+ f.close()
+ z2 = zipfile.ZipFile(f.name, 'r')
+ self.assertFalse(z2.testzip())
+
+ os.remove(f.name)
+
  def test_write_iterable_no_archive(self):
  z = zipstream.ZipFile(mode='w')
  self.assertRaises(TypeError, z.write_iter, iterable=range(10))

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26, py27, py32, py33, py34, py35, pypy
+envlist = py26, py27, py32, py33, py34, py35, py36, py37, py38, pypy, pypy3
 
 [testenv]
 deps=nose

zipstream/__init__.py

@@ -178,9 +178,8 @@ def __init__(self, fileobj=None, mode='w', compression=ZIP_STORED, allowZip64=Fa
  self.paths_to_write = []
 
  def __iter__(self):
- for kwargs in self.paths_to_write:
- for data in self.__write(**kwargs):
- yield data
+ for data in self.flush():
+ yield data
  for data in self.__close():
  yield data
 
@@ -190,6 +189,12 @@ def __enter__(self):
  def __exit__(self, type, value, traceback):
  self.close()
 
+ def flush(self):
+ while self.paths_to_write:
+ kwargs = self.paths_to_write.pop(0)
+ for data in self.__write(**kwargs):
+ yield data
+
  @property
  def comment(self):
  """The comment text associated with the ZIP file."""
@@ -215,20 +220,20 @@ def write(self, filename, arcname=None, compress_type=None):
  kwargs = {'filename': filename, 'arcname': arcname, 'compress_type': compress_type}
  self.paths_to_write.append(kwargs)
 
- def write_iter(self, arcname, iterable, compress_type=None):
+ def write_iter(self, arcname, iterable, compress_type=None, buffer_size=None, date_time=None):
  """Write the bytes iterable `iterable` to the archive under the name `arcname`."""
- kwargs = {'arcname': arcname, 'iterable': iterable, 'compress_type': compress_type}
+ kwargs = {'arcname': arcname, 'iterable': iterable, 'compress_type': compress_type, 'buffer_size': buffer_size, 'date_time': date_time}
  self.paths_to_write.append(kwargs)
 
- def writestr(self, arcname, data, compress_type=None):
+ def writestr(self, arcname, data, compress_type=None, buffer_size=None, date_time=None):
  """
  Writes a str into ZipFile by wrapping data as a generator
  """
  def _iterable():
  yield data
- return self.write_iter(arcname, _iterable(), compress_type=compress_type)
+ return self.write_iter(arcname, _iterable(), compress_type=compress_type, buffer_size=buffer_size, date_time=date_time)
 
- def __write(self, filename=None, iterable=None, arcname=None, compress_type=None):
+ def __write(self, filename=None, iterable=None, arcname=None, compress_type=None, buffer_size=None, date_time=None):
  """Put the bytes from filename into the archive under the name
  `arcname`."""
  if not self.fp:
@@ -243,7 +248,11 @@ def __write(self, filename=None, iterable=None, arcname=None, compress_type=None
  mtime = time.localtime(st.st_mtime)
  date_time = mtime[0:6]
  else:
- st, isdir, date_time = None, False, time.localtime()[0:6]
+ st, isdir = None, False
+ if date_time is not None and isinstance(date_time, time.struct_time):
+ date_time = date_time[0:6]
+ if date_time is None:
+ date_time = time.localtime()[0:6]
  # Create ZipInfo instance to store file information
  if arcname is None:
  arcname = filename
@@ -265,7 +274,7 @@ def __write(self, filename=None, iterable=None, arcname=None, compress_type=None
  if st:
  zinfo.file_size = st[6]
  else:
- zinfo.file_size = 0
+ zinfo.file_size = buffer_size or 0
  zinfo.flag_bits = 0x00
  zinfo.flag_bits |= 0x08 # ZIP flag bits, bit 3 indicates presence of data descriptor
  zinfo.header_offset = self.fp.tell() # Start of header bytes