[WIP] Moving to Py3

.coveragerc

@@ -0,0 +1,24 @@
+[run]
+source = imgee
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code
+    def __repr__
+    if self\.debug
+
+    # Don't complain about importerror handlers
+    except ImportError
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if False:
+    if __name__ == .__main__.:

imgee/models/__init__.py

@@ -6,7 +6,7 @@
 
 db = SQLAlchemy(app)
 
-from imgee.models.user import *  # NOQA  # isort:skip
-from imgee.models.stored_file import *  # NOQA  # isort:skip
-from imgee.models.thumbnail import *  # NOQA  # isort:skip
-from imgee.models.profile import *  # NOQA  # isort:skip
+from .user import *  # NOQA  # isort:skip
+from .stored_file import *  # NOQA  # isort:skip
+from .thumbnail import *  # NOQA  # isort:skip
+from .profile import *  # NOQA  # isort:skip

imgee/storage.py

@@ -90,7 +90,7 @@ def save_file(fp, profile, title=None):
     img_name = "%s%s" % (id_, extn)
     local_path = path_for(img_name)
 
-    with open(local_path, 'w') as img:
+    with open(local_path, 'wb') as img:
         img.write(fp.read())
 
     stored_file = save_img_in_db(
@@ -153,23 +153,24 @@ def save_tn_in_db(img, tn_name, size):
     return name
 
 
-def save_on_s3(filename, remotename='', content_type='', bucket='', folder=''):
+def save_on_s3(filename, remotename='', content_type=''):
     """
     Save contents from file named `filename` to `remotename` on S3.
     """
-    b = bucket or get_s3_bucket()
-    folder = get_s3_folder(folder)
+    bucket = get_s3_bucket()
+    folder = get_s3_folder()
+    key = os.path.join(folder, filename)
 
-    with open(path_for(filename)) as fp:
+    with open(path_for(filename), 'rb') as fp:
         filename = remotename or filename
-        k = b.new_key(folder + filename)
-        headers = {
-            'Cache-Control': 'max-age=31536000',  # 60*60*24*365
-            'Content-Type': get_file_type(fp, filename),
-            # once cached, it is set to expire after a year
-            'Expires': datetime.utcnow() + timedelta(days=365),
-        }
-        k.set_contents_from_file(fp, policy='public-read', headers=headers)
+        bucket.put_object(
+            ACL='public-read',
+            Key=key,
+            Body=fp.read(),
+            CacheControl='max-age=31536000',
+            ContentType=content_type or get_file_type(fp, filename),
+            Expires=datetime.utcnow() + timedelta(days=365),
+        )
     return filename
 
 
@@ -353,7 +354,9 @@ def delete(stored_file, commit=True):
     keys = [(get_s3_folder() + thumbnail.name + extn) for thumbnail in thumbnails]
     keys.append(get_s3_folder() + stored_file.name + extn)
     bucket = get_s3_bucket()
-    bucket.delete_keys(keys)
+    bucket.delete_objects(
+        Delete={'Objects': [{'Key': key} for key in keys], 'Quiet': True}
+    )
 
     # remove from the db
     # remove thumbnails explicitly.

imgee/utils.py

@@ -8,9 +8,8 @@
 
 from flask import request
 
-from boto import connect_s3
-from boto.s3.bucket import Bucket
-from boto.s3.key import Key
+import boto3
+import botocore
 from PIL import Image
 import defusedxml.cElementTree as ElementTree
 import magic
@@ -19,6 +18,7 @@
 
 from . import app
 
+
 THUMBNAIL_COMMANDS = {
     'inkscape': "inkscape -z -f {src} -e {src}.original.png && convert -quiet -thumbnail {width}x{height} {src}.original.png -colorspace sRGB -quality 75% {dest}",
     'rsvg-convert': "rsvg-convert --width={width} --height={height} --keep-aspect-ratio=TRUE --format={format} {src} > {dest}",
@@ -303,9 +303,8 @@ def is_animated_gif(local_path):
 
 def get_file_type(fp, filename=None):
     fp.seek(0)
-    data = fp.read(1024)  # https://github.com/ahupp/python-magic#usage
+    result = magic.from_buffer(fp.read(), mime=True)
     fp.seek(0)
-    result = magic.from_buffer(data, mime=True)
     if result in ('text/plain', 'text/xml', 'application/xml'):
         if is_svg(fp):
             return 'image/svg+xml'
@@ -322,14 +321,18 @@ def is_file_allowed(fp, provided_mimetype=None, filename=None):
 
 
 # -- s3 related --
-def get_s3_connection():
-    return connect_s3(app.config['AWS_ACCESS_KEY'], app.config['AWS_SECRET_KEY'])
+
+
+def get_s3_client():
+    return boto3.resource(
+        's3',
+        aws_access_key_id=app.config['AWS_ACCESS_KEY'],
+        aws_secret_access_key=app.config['AWS_SECRET_KEY'],
+    )
 
 
 def get_s3_bucket():
-    conn = get_s3_connection()
-    bucket = Bucket(conn, app.config['AWS_BUCKET'])
-    return bucket
+    return get_s3_client().Bucket(app.config['AWS_BUCKET'])
 
 
 def get_s3_folder(f=''):
@@ -347,12 +350,10 @@ def exists_in_s3(thumb):
     if 'thumb_extn' in ALLOWED_MIMETYPES[thumb.stored_file.mimetype]:
         extn = ALLOWED_MIMETYPES[thumb.stored_file.mimetype]['thumb_extn']
     key = os.path.join(folder, thumb.name + extn)
-    # print("checking whether exists in s3: {}".format(key))
-    resp = bucket.get_key(key)
-    if not resp:
-        # print("does not exist")
+    try:
+        bucket.Object(key).load()
+    except botocore.exceptions.ClientError:
         return False
-    # print("exists")
     return True
 
 
@@ -361,9 +362,8 @@ def download_from_s3(img_name):
     if not os.path.exists(local_path):
         bucket = get_s3_bucket()
         folder = get_s3_folder()
-        k = Key(bucket)
-        k.key = os.path.join(folder, img_name)
-        k.get_contents_to_filename(local_path)
+        key = os.path.join(folder, img_name)
+        bucket.Object(key).download_file(local_path)
     return local_path
 
 
@@ -378,6 +378,7 @@ def get_width_height(img_path):
             o = check_output(
                 'identify -quiet -ping -format "%wx%h" {}[0]'.format(img_path),
                 shell=True,
+                universal_newlines=True
             )
             w, h = o.split('x')
         elif extn in ['.cdr']:
@@ -394,7 +395,8 @@ def get_width_height(img_path):
                 w, h = int(round(float(wo))), int(round(float(ho)))
         else:
             o = check_output(
-                'identify -quiet -ping -format "%wx%h" {}'.format(img_path), shell=True
+                'identify -quiet -ping -format "%wx%h" {}'.format(img_path), shell=True,
+                universal_newlines=True
             )
             w, h = o.split('x')
         return (w, h)

imgee/views/files.py

@@ -39,10 +39,11 @@ def _redirect_url_frm_upload(profile_name):
 def upload_file(profile):
     upload_form = UploadImageForm()
     if upload_form.validate_on_submit():
-        file_ = request.files['upload_file']
-        title, stored_file = save_file(file_, profile=profile)
-        flash('"%s" uploaded successfully.' % title)
-        return redirect(_redirect_url_frm_upload(profile.name))
+        imgfile = request.files['upload_file']
+        if imgfile.filename != '':
+            title, stored_file = save_file(imgfile, profile=profile)
+            flash('"%s" uploaded successfully.' % title)
+            return redirect(_redirect_url_frm_upload(profile.name))
     return render_template('form.html.jinja2', form=upload_form, profile=profile)
 
 

instance/testing.py

@@ -8,12 +8,12 @@
 UNKNOWN_FILE_THUMBNAIL = 'unknown.jpeg'
 
 #: S3 Configuration; AWS credentials with restricted access
-AWS_ACCESS_KEY = environ.get('AWS_ACCESS_KEY')
-AWS_SECRET_KEY = environ.get('AWS_SECRET_KEY')
-AWS_BUCKET = environ.get('AWS_BUCKET', 'imgee-testing')
+#: S3 Configuration
+AWS_ACCESS_KEY = 'AKIAIRIRVOH3RE5SKR4A'
+AWS_SECRET_KEY = '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'
+AWS_BUCKET = 'imgee'
 AWS_FOLDER = 'test'
-
-MEDIA_DOMAIN = 'https://%s.s3.amazonaws.com' % AWS_BUCKET
+MEDIA_DOMAIN = 'http://%s.s3.amazonaws.com/' % AWS_BUCKET
 
 SQLALCHEMY_DATABASE_URI = 'sqlite://'
 WTF_CSRF_ENABLED = False

requirements.txt

@@ -7,9 +7,8 @@ Flask-SQLAlchemy==2.4.1
 psycopg2==2.8.4
 https://github.com/hasgeek/flask-lastuser/zipball/master
 https://github.com/jace/flask-alembic/archive/master.zip
-boto==2.49.0
+boto3==1.10.46
 redis==3.3.11
-xml4h==0.2.0
 python-magic==0.4.15
 defusedxml==0.6.0
 Flask-Migrate==2.5.2

runtests.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 set -e
+export PYTHONIOENCODING="UTF-8"
 export FLASK_ENV="TESTING"
-coverage run `which nosetests` "$@"
+coverage run -m pytest "$@"
 coverage report -m

setup.cfg

@@ -30,12 +30,3 @@ sections = FUTURE, STDLIB, SQLALCHEMY, FLASK, THIRDPARTY, FIRSTPARTY, LOCALFOLDE
 # Bandit config for flake8-bandit. There's another copy in .pre-commit-config.yaml
 [bandit]
 exclude = tests, migrations, instance
-
-[nosetests]
-match=^test
-nocapture=1
-cover-package=imgee
-with-coverage=1
-cover-erase=1
-with-doctest=1
-where=tests

tests/fixtures.py

@@ -11,7 +11,7 @@ class ImgeeTestCase(unittest.TestCase):
     def get_test_user(self, name='testuser', uid=1):
         u = User.query.filter_by(username=str(name)).first()
         if not u:
-            userid = ''.join(random.sample(string.letters, 22))
+            userid = ''.join(random.sample(string.ascii_letters, 22))
             u = User(
                 username=str(name),
                 userid=str(userid),

tests/test_labels.py

@@ -14,13 +14,13 @@ class LabelTestCase(ImgeeTestCase):
     def setUp(self):
         super(LabelTestCase, self).setUp()
         self.img_id = None
-        self.test_files = ['../imgee/static/img/imgee.png']
+        self.test_files = ['imgee/static/img/imgee.png']
         self.test_labels = ['logos', 'banners', 'profile-photos']
 
     def upload(self, path=None):
         profile = self.get_test_profile()
         sf = None
-        with open(path or self.test_files[0]) as fp:
+        with open(path or self.test_files[0], 'rb') as fp:
             fs = FileStorage(fp)
             title, sf = save_file(fs, profile)
         return sf

tests/test_upload.py

@@ -17,10 +17,10 @@ def setUp(self):
         super(UploadTestCase, self).setUp()
         self.img_id = None
         self.test_files = [
-            '../imgee/static/img/imgee.png',
-            '../imgee/static/img/imgee.jpeg',
-            '../imgee/static/img/imgee.svg',
-            '../imgee/static/img/loading.gif',
+            'imgee/static/img/imgee.png',
+            'imgee/static/img/imgee.jpeg',
+            'imgee/static/img/imgee.svg',
+            'imgee/static/img/loading.gif',
         ]
         self.test_labels = ['logos', 'banners', 'profile-photos']
         self.files = self.upload_all()
@@ -38,7 +38,7 @@ def upload(self, path=None):
         profile = self.get_test_profile()
         sf = None
         if path:
-            with open(path) as fp:
+            with open(path, 'rb') as fp:
                 fs = FileStorage(fp)
                 title, sf = save_file(fs, profile)
             return sf