no-if-six.py* (#2840)

* no-if-six.py* * no unused imports * Update .travis.yml * no py27 in travis (#2836) * no py27 in travis * py37 * flake8 and docs back * py36 py37 postgres * py36-apache * MixedUnicodeBytes removed * unused imports removed * no-if-py3 * space * space * MixedUnicodeBytesFormat returned * postgres toxenv for py3 * postgres toxenv for py3 * postgres toxenv for py3 * \n * no-unused-import * test_multiple_workers back
spotify · Dec 3, 2019 · f42cdc3 · f42cdc3
1 parent ce6d4b8
commit f42cdc3
Show file tree

Hide file tree

Showing 15 changed files with 50 additions and 110 deletions.
diff --git a/luigi/contrib/bigquery_avro.py b/luigi/contrib/bigquery_avro.py
@@ -1,7 +1,6 @@
 """Specialized tasks for handling Avro data in BigQuery from GCS.
 """
 import logging
-import six
 
 from luigi.contrib.bigquery import BigQueryLoadTask, SourceFormat
 from luigi.contrib.gcs import GCSClient
@@ -85,10 +84,7 @@ def _get_writer_schema(datum_reader):
         Returns:
         Returning correct attribute name depending on Python version.
         """
-        if six.PY2:
-            return datum_reader.writers_schema
-        else:
-            return datum_reader.writer_schema
+        return datum_reader.writer_schema
 
     def _set_output_doc(self, avro_schema):
         bq_client = self.output().client.client

diff --git a/luigi/contrib/external_daily_snapshot.py b/luigi/contrib/external_daily_snapshot.py
@@ -19,10 +19,6 @@
 import logging
 
 import luigi
-from luigi import six
-
-if six.PY3:
-    xrange = range
 
 logger = logging.getLogger('luigi-interface')
 
@@ -69,7 +65,7 @@ def latest(cls, *args, **kwargs):
     def __latest(cls, date, lookback, args, kwargs):
         assert lookback > 0
         t = None
-        for i in xrange(lookback):
+        for i in range(lookback):
             d = date - datetime.timedelta(i)
             t = cls(date=d, *args, **kwargs)
             if t.complete():

diff --git a/luigi/contrib/hadoop.py b/luigi/contrib/hadoop.py
@@ -49,9 +49,6 @@
 import luigi.contrib.s3
 from luigi.contrib import mrrunner
 
-if six.PY2:
-    from itertools import imap as map
-
 try:
     # See benchmark at https://gist.github.com/mvj3/02dca2bcc8b0ef1bbfb5
     import ujson as json

diff --git a/luigi/contrib/hive.py b/luigi/contrib/hive.py
@@ -34,9 +34,6 @@
 from luigi.target import FileAlreadyExists, FileSystemTarget
 from luigi.task import flatten
 
-if six.PY3:
-    unicode = str
-
 logger = logging.getLogger('luigi-interface')
 
 
@@ -460,7 +457,7 @@ def run_job(self, job, tracking_url_callback=None):
         self.prepare_outputs(job)
         with tempfile.NamedTemporaryFile() as f:
             query = job.query()
-            if isinstance(query, unicode):
+            if isinstance(query, str):
                 query = query.encode('utf8')
             f.write(query)
             f.flush()

diff --git a/luigi/date_interval.py b/luigi/date_interval.py
@@ -31,14 +31,9 @@ class MyTask(luigi.Task):
 ``--date-interval 2014`` (for a year) and some other notations.
 """
 
-from luigi import six
-
 import datetime
 import re
 
-if six.PY3:
-    xrange = range
-
 
 class DateInterval(object):
 
@@ -75,7 +70,7 @@ def dates(self):
     def hours(self):
         ''' Same as dates() but returns 24 times more info: one for each hour.'''
         for date in self.dates():
-            for hour in xrange(24):
+            for hour in range(24):
                 yield datetime.datetime.combine(date, datetime.time(hour))
 
     def __str__(self):
@@ -188,7 +183,7 @@ class Week(DateInterval):
     '''
     def __init__(self, y, w):
         ''' Python datetime does not have a method to convert from ISO weeks, so the constructor uses some stupid brute force'''
-        for d in xrange(-10, 370):
+        for d in range(-10, 370):
             date = datetime.date(y, 1, 1) + datetime.timedelta(d)
             if date.isocalendar() == (y, w, 1):
                 date_a = date

diff --git a/luigi/rpc.py b/luigi/rpc.py
@@ -79,8 +79,7 @@ def _create_request(self, full_url, body=None):
         if url.username:
             # base64 encoding of username:password
             auth = base64.b64encode(six.b('{}:{}'.format(url.username, url.password or '')))
-            if six.PY3:
-                auth = auth.decode('utf-8')
+            auth = auth.decode('utf-8')
 
             # update full_url and create a request object with the auth header set
             full_url = url._replace(netloc=url.netloc.split('@', 1)[-1]).geturl()

diff --git a/luigi/task.py b/luigi/task.py
@@ -792,10 +792,7 @@ class RequiringTask(luigi.Task):
 
         externalize(MyTask)  # BAD: This does nothing (as after luigi 2.4.0)
     """
-    # Seems like with python < 3.3 copy.copy can't copy classes
-    # and objects with specified metaclass http://bugs.python.org/issue11480
-    compatible_copy = copy.copy if six.PY3 else copy.deepcopy
-    copied_value = compatible_copy(taskclass_or_taskobject)
+    copied_value = copy.copy(taskclass_or_taskobject)
     if copied_value is taskclass_or_taskobject:
         # Assume it's a class
         clazz = taskclass_or_taskobject

diff --git a/luigi/util.py b/luigi/util.py
@@ -220,13 +220,9 @@ class TaskB(luigi.Task):
 import datetime
 import logging
 
-from luigi import six
-
 from luigi import task
 from luigi import parameter
 
-if six.PY3:
-    xrange = range
 
 logger = logging.getLogger('luigi-interface')
 
@@ -459,7 +455,7 @@ def previous(task):
 
 def get_previous_completed(task, max_steps=10):
     prev = task
-    for _ in xrange(max_steps):
+    for _ in range(max_steps):
         prev = previous(prev)
         logger.debug("Checking if %s is complete", prev)
         if prev.complete():

diff --git a/test/contrib/bigquery_avro_test.py b/test/contrib/bigquery_avro_test.py
@@ -19,7 +19,6 @@
 These are the unit tests for the BigQueryLoadAvro class.
 """
 
-import six
 import unittest
 import avro
 import avro.schema
@@ -41,15 +40,9 @@ def test_writer_schema_method_existence(self):
             ]
         }
         """
-        avro_schema = self._parse_schema(schema_json)
+        avro_schema = avro.schema.Parse(schema_json)
         reader = avro.io.DatumReader(avro_schema, avro_schema)
         actual_schema = BigQueryLoadAvro._get_writer_schema(reader)
         self.assertEqual(actual_schema, avro_schema,
                          "writer(s) avro_schema attribute not found")
         # otherwise AttributeError is thrown
-
-    def _parse_schema(self, schema_json):
-        if six.PY2:
-            return avro.schema.parse(schema_json)
-        else:
-            return avro.schema.Parse(schema_json)
diff --git a/test/contrib/opener_test.py b/test/contrib/opener_test.py
@@ -1,7 +1,6 @@
 import luigi
 import mock
 import random
-import six
 import unittest
 
 from luigi.contrib.opener import OpenerTarget, NoOpenerError
@@ -117,16 +116,15 @@ def test_s3_parse_param(self, s3_init_patch):
                                          bar='true')
 
     def test_binary_support(self):
-        '''Make sure keyword arguments are preserved through the OpenerTarget
-
-        '''
-        if six.PY3:
-            # Verify we can't normally write binary data
-            fp = OpenerTarget("mock://file.txt").open('w')
-            self.assertRaises(TypeError, fp.write, b'\x07\x08\x07')
-
-            # Verify the format is passed to the target and write binary data
-            fp = OpenerTarget("mock://file.txt",
-                              format=luigi.format.MixedUnicodeBytes).open('w')
-            fp.write(b'\x07\x08\x07')
-            fp.close()
+        """
+        Make sure keyword arguments are preserved through the OpenerTarget
+        """
+        # Verify we can't normally write binary data
+        fp = OpenerTarget("mock://file.txt").open('w')
+        self.assertRaises(TypeError, fp.write, b'\x07\x08\x07')
+
+        # Verify the format is passed to the target and write binary data
+        fp = OpenerTarget("mock://file.txt",
+                          format=luigi.format.MixedUnicodeBytes).open('w')
+        fp.write(b'\x07\x08\x07')
+        fp.close()
diff --git a/test/contrib/sqla_test.py b/test/contrib/sqla_test.py
@@ -24,18 +24,13 @@
 import tempfile
 import unittest
 
-from luigi import six
-
 import luigi
 import sqlalchemy
 from luigi.contrib import sqla
 from luigi.mock import MockTarget
 from nose.plugins.attrib import attr
 from helpers import skipOnTravis
 
-if six.PY3:
-    unicode = str
-
 
 class BaseTask(luigi.Task):
 
@@ -149,7 +144,7 @@ def _check_entries(self, engine):
             result = conn.execute(s).fetchall()
             for i in range(len(BaseTask.TASK_LIST)):
                 given = BaseTask.TASK_LIST[i].strip("\n").split("\t")
-                given = (unicode(given[0]), unicode(given[1]))
+                given = (str(given[0]), str(given[1]))
                 self.assertEqual(given, tuple(result[i]))
 
     def test_rows(self):

diff --git a/test/date_parameter_test.py b/test/date_parameter_test.py
@@ -20,7 +20,6 @@
 
 import luigi
 import luigi.interface
-from luigi import six
 
 
 class DateTask(luigi.Task):
@@ -97,10 +96,8 @@ def test_parse_padding_zero(self):
         self.assertEqual(dm, datetime.datetime(2013, 2, 1, 18, 7, 0))
 
     def test_parse_deprecated(self):
-        if six.PY3:
-            with self.assertWarnsRegex(DeprecationWarning, 'Using "H" between hours and minutes is deprecated, omit it instead.'):
-                dm = luigi.DateMinuteParameter().parse('2013-02-01T18H42')
-        else:
+        with self.assertWarnsRegex(DeprecationWarning,
+                                   'Using "H" between hours and minutes is deprecated, omit it instead.'):
             dm = luigi.DateMinuteParameter().parse('2013-02-01T18H42')
         self.assertEqual(dm, datetime.datetime(2013, 2, 1, 18, 42, 0))
 

diff --git a/test/parameter_test.py b/test/parameter_test.py
@@ -28,7 +28,6 @@
 import luigi.notifications
 from luigi.mock import MockTarget
 from luigi.parameter import ParameterException
-from luigi import six
 from worker_test import email_patch
 
 luigi.notifications.DEBUG = True
@@ -616,16 +615,15 @@ class CatsWithoutSection(luigi.Config):
             self.assertEqual(Dogs().n_dogs, 654)
             self.assertEqual(CatsWithoutSection().n_cats, 321)
 
-    if six.PY3:
-        def test_global_significant_param_warning(self):
-            """ We don't want any kind of global param to be positional """
-            with self.assertWarnsRegex(DeprecationWarning, 'is_global support is removed. Assuming positional=False'):
-                class MyTask(luigi.Task):
-                    # This could typically be called "--test-dry-run"
-                    x_g1 = luigi.Parameter(default='y', is_global=True, significant=True)
+    def test_global_significant_param_warning(self):
+        """ We don't want any kind of global param to be positional """
+        with self.assertWarnsRegex(DeprecationWarning, 'is_global support is removed. Assuming positional=False'):
+            class MyTask(luigi.Task):
+                # This could typically be called "--test-dry-run"
+                x_g1 = luigi.Parameter(default='y', is_global=True, significant=True)
 
-            self.assertRaises(luigi.parameter.UnknownParameterException,
-                              lambda: MyTask('arg'))
+        self.assertRaises(luigi.parameter.UnknownParameterException,
+                          lambda: MyTask('arg'))
 
         def test_global_insignificant_param_warning(self):
             """ We don't want any kind of global param to be positional """
@@ -678,10 +676,8 @@ def testDateWithMinuteInterval(self):
     @with_config({"foo": {"bar": "2001-02-03T04H30"}})
     def testDateMinuteDeprecated(self):
         p = luigi.DateMinuteParameter(config_path=dict(section="foo", name="bar"))
-        if six.PY3:
-            with self.assertWarnsRegex(DeprecationWarning, 'Using "H" between hours and minutes is deprecated, omit it instead.'):
-                self.assertEqual(datetime.datetime(2001, 2, 3, 4, 30, 0), _value(p))
-        else:
+        with self.assertWarnsRegex(DeprecationWarning,
+                                   'Using "H" between hours and minutes is deprecated, omit it instead.'):
             self.assertEqual(datetime.datetime(2001, 2, 3, 4, 30, 0), _value(p))
 
     @with_config({"foo": {"bar": "2001-02-03T040506"}})
@@ -1000,19 +996,13 @@ class OverrideEnvStuff(LuigiTestCase):
 
     @with_config({"core": {"default-scheduler-port": '6543'}})
     def testOverrideSchedulerPort(self):
-        if six.PY3:
-            with self.assertWarnsRegex(DeprecationWarning, r'default-scheduler-port is deprecated'):
-                env_params = luigi.interface.core()
-        else:
+        with self.assertWarnsRegex(DeprecationWarning, r'default-scheduler-port is deprecated'):
             env_params = luigi.interface.core()
-        self.assertEqual(env_params.scheduler_port, 6543)
+            self.assertEqual(env_params.scheduler_port, 6543)
 
     @with_config({"core": {"scheduler-port": '6544'}})
     def testOverrideSchedulerPort2(self):
-        if six.PY3:
-            with self.assertWarnsRegex(DeprecationWarning, r'scheduler-port \(with dashes\) should be avoided'):
-                env_params = luigi.interface.core()
-        else:
+        with self.assertWarnsRegex(DeprecationWarning, r'scheduler-port \(with dashes\) should be avoided'):
             env_params = luigi.interface.core()
         self.assertEqual(env_params.scheduler_port, 6544)
 

diff --git a/test/task_test.py b/test/task_test.py
@@ -17,7 +17,6 @@
 
 import doctest
 import pickle
-import six
 import warnings
 
 from helpers import unittest, LuigiTestCase
@@ -137,23 +136,22 @@ def test_no_warn_if_param_types_ok(self):
             DummyTask(**DUMMY_TASK_OK_PARAMS)
         self.assertEqual(len(w), 0, msg='No warning should be raised when correct parameter types are used')
 
-    if six.PY3:  # assertWarnsRegex was introduced in Python 3.2
-        def test_warn_on_non_str_param(self):
-            params = dict(**DUMMY_TASK_OK_PARAMS)
-            params['param'] = 42
-            with self.assertWarnsRegex(UserWarning, 'Parameter "param" with value "42" is not of type string.'):
-                DummyTask(**params)
+    def test_warn_on_non_str_param(self):
+        params = dict(**DUMMY_TASK_OK_PARAMS)
+        params['param'] = 42
+        with self.assertWarnsRegex(UserWarning, 'Parameter "param" with value "42" is not of type string.'):
+            DummyTask(**params)
 
-        def test_warn_on_non_timedelta_param(self):
-            params = dict(**DUMMY_TASK_OK_PARAMS)
+    def test_warn_on_non_timedelta_param(self):
+        params = dict(**DUMMY_TASK_OK_PARAMS)
 
-            class MockTimedelta(object):
-                days = 1
-                seconds = 1
+        class MockTimedelta(object):
+            days = 1
+            seconds = 1
 
-            params['timedelta_param'] = MockTimedelta()
-            with self.assertWarnsRegex(UserWarning, 'Parameter "timedelta_param" with value ".*" is not of type timedelta.'):
-                DummyTask(**params)
+        params['timedelta_param'] = MockTimedelta()
+        with self.assertWarnsRegex(UserWarning, 'Parameter "timedelta_param" with value ".*" is not of type timedelta.'):
+            DummyTask(**params)
 
 
 class ExternalizeTaskTest(LuigiTestCase):

diff --git a/test/worker_test.py b/test/worker_test.py
@@ -38,7 +38,6 @@
 from luigi.scheduler import Scheduler
 from luigi.worker import Worker
 from luigi.rpc import RPCError
-from luigi import six
 from luigi.cmdline import luigi_run
 
 luigi.notifications.DEBUG = True
@@ -1484,9 +1483,6 @@ def complete(self):
 class MultipleWorkersTest(unittest.TestCase):
 
     @unittest.skip('Always skip. There are many intermittent failures')
-    # This pass under python3 when run as `nosetests test/worker_test.py`
-    # but not as `nosetests test`. Probably some side effect on previous tests
-    @unittest.skipIf(six.PY3, 'This test fail on python3 when run with tox.')
     def test_multiple_workers(self):
         # Test using multiple workers
         # Also test generating classes dynamically since this may reflect issues with