diff --git a/luigi/contrib/bigquery_avro.py b/luigi/contrib/bigquery_avro.py index 6e92a6acf9..7350da1f42 100644 --- a/luigi/contrib/bigquery_avro.py +++ b/luigi/contrib/bigquery_avro.py @@ -1,7 +1,6 @@ """Specialized tasks for handling Avro data in BigQuery from GCS. """ import logging -import six from luigi.contrib.bigquery import BigQueryLoadTask, SourceFormat from luigi.contrib.gcs import GCSClient @@ -85,10 +84,7 @@ def _get_writer_schema(datum_reader): Returns: Returning correct attribute name depending on Python version. """ - if six.PY2: - return datum_reader.writers_schema - else: - return datum_reader.writer_schema + return datum_reader.writer_schema def _set_output_doc(self, avro_schema): bq_client = self.output().client.client diff --git a/luigi/contrib/external_daily_snapshot.py b/luigi/contrib/external_daily_snapshot.py index 8c5fcf6ee0..307cd112e2 100644 --- a/luigi/contrib/external_daily_snapshot.py +++ b/luigi/contrib/external_daily_snapshot.py @@ -19,10 +19,6 @@ import logging import luigi -from luigi import six - -if six.PY3: - xrange = range logger = logging.getLogger('luigi-interface') @@ -69,7 +65,7 @@ def latest(cls, *args, **kwargs): def __latest(cls, date, lookback, args, kwargs): assert lookback > 0 t = None - for i in xrange(lookback): + for i in range(lookback): d = date - datetime.timedelta(i) t = cls(date=d, *args, **kwargs) if t.complete(): diff --git a/luigi/contrib/hadoop.py b/luigi/contrib/hadoop.py index 7310e14c35..1e48d27456 100644 --- a/luigi/contrib/hadoop.py +++ b/luigi/contrib/hadoop.py @@ -49,9 +49,6 @@ import luigi.contrib.s3 from luigi.contrib import mrrunner -if six.PY2: - from itertools import imap as map - try: # See benchmark at https://gist.github.com/mvj3/02dca2bcc8b0ef1bbfb5 import ujson as json diff --git a/luigi/contrib/hive.py b/luigi/contrib/hive.py index 96c8247282..255eebd98f 100644 --- a/luigi/contrib/hive.py +++ b/luigi/contrib/hive.py @@ -34,9 +34,6 @@ from luigi.target import FileAlreadyExists, FileSystemTarget from luigi.task import flatten -if six.PY3: - unicode = str - logger = logging.getLogger('luigi-interface') @@ -460,7 +457,7 @@ def run_job(self, job, tracking_url_callback=None): self.prepare_outputs(job) with tempfile.NamedTemporaryFile() as f: query = job.query() - if isinstance(query, unicode): + if isinstance(query, str): query = query.encode('utf8') f.write(query) f.flush() diff --git a/luigi/date_interval.py b/luigi/date_interval.py index 8a7bcfd0f1..302f9798e0 100644 --- a/luigi/date_interval.py +++ b/luigi/date_interval.py @@ -31,14 +31,9 @@ class MyTask(luigi.Task): ``--date-interval 2014`` (for a year) and some other notations. """ -from luigi import six - import datetime import re -if six.PY3: - xrange = range - class DateInterval(object): @@ -75,7 +70,7 @@ def dates(self): def hours(self): ''' Same as dates() but returns 24 times more info: one for each hour.''' for date in self.dates(): - for hour in xrange(24): + for hour in range(24): yield datetime.datetime.combine(date, datetime.time(hour)) def __str__(self): @@ -188,7 +183,7 @@ class Week(DateInterval): ''' def __init__(self, y, w): ''' Python datetime does not have a method to convert from ISO weeks, so the constructor uses some stupid brute force''' - for d in xrange(-10, 370): + for d in range(-10, 370): date = datetime.date(y, 1, 1) + datetime.timedelta(d) if date.isocalendar() == (y, w, 1): date_a = date diff --git a/luigi/rpc.py b/luigi/rpc.py index 5975fcfbd9..1ef6c1d6bb 100644 --- a/luigi/rpc.py +++ b/luigi/rpc.py @@ -79,8 +79,7 @@ def _create_request(self, full_url, body=None): if url.username: # base64 encoding of username:password auth = base64.b64encode(six.b('{}:{}'.format(url.username, url.password or ''))) - if six.PY3: - auth = auth.decode('utf-8') + auth = auth.decode('utf-8') # update full_url and create a request object with the auth header set full_url = url._replace(netloc=url.netloc.split('@', 1)[-1]).geturl() diff --git a/luigi/task.py b/luigi/task.py index c434a3683c..5e6e727842 100644 --- a/luigi/task.py +++ b/luigi/task.py @@ -792,10 +792,7 @@ class RequiringTask(luigi.Task): externalize(MyTask) # BAD: This does nothing (as after luigi 2.4.0) """ - # Seems like with python < 3.3 copy.copy can't copy classes - # and objects with specified metaclass http://bugs.python.org/issue11480 - compatible_copy = copy.copy if six.PY3 else copy.deepcopy - copied_value = compatible_copy(taskclass_or_taskobject) + copied_value = copy.copy(taskclass_or_taskobject) if copied_value is taskclass_or_taskobject: # Assume it's a class clazz = taskclass_or_taskobject diff --git a/luigi/util.py b/luigi/util.py index aae3ce3248..83a66baa09 100644 --- a/luigi/util.py +++ b/luigi/util.py @@ -220,13 +220,9 @@ class TaskB(luigi.Task): import datetime import logging -from luigi import six - from luigi import task from luigi import parameter -if six.PY3: - xrange = range logger = logging.getLogger('luigi-interface') @@ -459,7 +455,7 @@ def previous(task): def get_previous_completed(task, max_steps=10): prev = task - for _ in xrange(max_steps): + for _ in range(max_steps): prev = previous(prev) logger.debug("Checking if %s is complete", prev) if prev.complete(): diff --git a/test/contrib/bigquery_avro_test.py b/test/contrib/bigquery_avro_test.py index c79a630044..1e13e1471b 100644 --- a/test/contrib/bigquery_avro_test.py +++ b/test/contrib/bigquery_avro_test.py @@ -19,7 +19,6 @@ These are the unit tests for the BigQueryLoadAvro class. """ -import six import unittest import avro import avro.schema @@ -41,15 +40,9 @@ def test_writer_schema_method_existence(self): ] } """ - avro_schema = self._parse_schema(schema_json) + avro_schema = avro.schema.Parse(schema_json) reader = avro.io.DatumReader(avro_schema, avro_schema) actual_schema = BigQueryLoadAvro._get_writer_schema(reader) self.assertEqual(actual_schema, avro_schema, "writer(s) avro_schema attribute not found") # otherwise AttributeError is thrown - - def _parse_schema(self, schema_json): - if six.PY2: - return avro.schema.parse(schema_json) - else: - return avro.schema.Parse(schema_json) diff --git a/test/contrib/opener_test.py b/test/contrib/opener_test.py index 014c82ab2b..33e0de421f 100644 --- a/test/contrib/opener_test.py +++ b/test/contrib/opener_test.py @@ -1,7 +1,6 @@ import luigi import mock import random -import six import unittest from luigi.contrib.opener import OpenerTarget, NoOpenerError @@ -117,16 +116,15 @@ def test_s3_parse_param(self, s3_init_patch): bar='true') def test_binary_support(self): - '''Make sure keyword arguments are preserved through the OpenerTarget - - ''' - if six.PY3: - # Verify we can't normally write binary data - fp = OpenerTarget("mock://file.txt").open('w') - self.assertRaises(TypeError, fp.write, b'\x07\x08\x07') - - # Verify the format is passed to the target and write binary data - fp = OpenerTarget("mock://file.txt", - format=luigi.format.MixedUnicodeBytes).open('w') - fp.write(b'\x07\x08\x07') - fp.close() + """ + Make sure keyword arguments are preserved through the OpenerTarget + """ + # Verify we can't normally write binary data + fp = OpenerTarget("mock://file.txt").open('w') + self.assertRaises(TypeError, fp.write, b'\x07\x08\x07') + + # Verify the format is passed to the target and write binary data + fp = OpenerTarget("mock://file.txt", + format=luigi.format.MixedUnicodeBytes).open('w') + fp.write(b'\x07\x08\x07') + fp.close() diff --git a/test/contrib/sqla_test.py b/test/contrib/sqla_test.py index db265a9742..ef50124457 100644 --- a/test/contrib/sqla_test.py +++ b/test/contrib/sqla_test.py @@ -24,8 +24,6 @@ import tempfile import unittest -from luigi import six - import luigi import sqlalchemy from luigi.contrib import sqla @@ -33,9 +31,6 @@ from nose.plugins.attrib import attr from helpers import skipOnTravis -if six.PY3: - unicode = str - class BaseTask(luigi.Task): @@ -149,7 +144,7 @@ def _check_entries(self, engine): result = conn.execute(s).fetchall() for i in range(len(BaseTask.TASK_LIST)): given = BaseTask.TASK_LIST[i].strip("\n").split("\t") - given = (unicode(given[0]), unicode(given[1])) + given = (str(given[0]), str(given[1])) self.assertEqual(given, tuple(result[i])) def test_rows(self): diff --git a/test/date_parameter_test.py b/test/date_parameter_test.py index 6438996b01..0c705bb53a 100644 --- a/test/date_parameter_test.py +++ b/test/date_parameter_test.py @@ -20,7 +20,6 @@ import luigi import luigi.interface -from luigi import six class DateTask(luigi.Task): @@ -97,10 +96,8 @@ def test_parse_padding_zero(self): self.assertEqual(dm, datetime.datetime(2013, 2, 1, 18, 7, 0)) def test_parse_deprecated(self): - if six.PY3: - with self.assertWarnsRegex(DeprecationWarning, 'Using "H" between hours and minutes is deprecated, omit it instead.'): - dm = luigi.DateMinuteParameter().parse('2013-02-01T18H42') - else: + with self.assertWarnsRegex(DeprecationWarning, + 'Using "H" between hours and minutes is deprecated, omit it instead.'): dm = luigi.DateMinuteParameter().parse('2013-02-01T18H42') self.assertEqual(dm, datetime.datetime(2013, 2, 1, 18, 42, 0)) diff --git a/test/parameter_test.py b/test/parameter_test.py index 03b07c3154..d6cdb254c3 100644 --- a/test/parameter_test.py +++ b/test/parameter_test.py @@ -28,7 +28,6 @@ import luigi.notifications from luigi.mock import MockTarget from luigi.parameter import ParameterException -from luigi import six from worker_test import email_patch luigi.notifications.DEBUG = True @@ -616,16 +615,15 @@ class CatsWithoutSection(luigi.Config): self.assertEqual(Dogs().n_dogs, 654) self.assertEqual(CatsWithoutSection().n_cats, 321) - if six.PY3: - def test_global_significant_param_warning(self): - """ We don't want any kind of global param to be positional """ - with self.assertWarnsRegex(DeprecationWarning, 'is_global support is removed. Assuming positional=False'): - class MyTask(luigi.Task): - # This could typically be called "--test-dry-run" - x_g1 = luigi.Parameter(default='y', is_global=True, significant=True) + def test_global_significant_param_warning(self): + """ We don't want any kind of global param to be positional """ + with self.assertWarnsRegex(DeprecationWarning, 'is_global support is removed. Assuming positional=False'): + class MyTask(luigi.Task): + # This could typically be called "--test-dry-run" + x_g1 = luigi.Parameter(default='y', is_global=True, significant=True) - self.assertRaises(luigi.parameter.UnknownParameterException, - lambda: MyTask('arg')) + self.assertRaises(luigi.parameter.UnknownParameterException, + lambda: MyTask('arg')) def test_global_insignificant_param_warning(self): """ We don't want any kind of global param to be positional """ @@ -678,10 +676,8 @@ def testDateWithMinuteInterval(self): @with_config({"foo": {"bar": "2001-02-03T04H30"}}) def testDateMinuteDeprecated(self): p = luigi.DateMinuteParameter(config_path=dict(section="foo", name="bar")) - if six.PY3: - with self.assertWarnsRegex(DeprecationWarning, 'Using "H" between hours and minutes is deprecated, omit it instead.'): - self.assertEqual(datetime.datetime(2001, 2, 3, 4, 30, 0), _value(p)) - else: + with self.assertWarnsRegex(DeprecationWarning, + 'Using "H" between hours and minutes is deprecated, omit it instead.'): self.assertEqual(datetime.datetime(2001, 2, 3, 4, 30, 0), _value(p)) @with_config({"foo": {"bar": "2001-02-03T040506"}}) @@ -1000,19 +996,13 @@ class OverrideEnvStuff(LuigiTestCase): @with_config({"core": {"default-scheduler-port": '6543'}}) def testOverrideSchedulerPort(self): - if six.PY3: - with self.assertWarnsRegex(DeprecationWarning, r'default-scheduler-port is deprecated'): - env_params = luigi.interface.core() - else: + with self.assertWarnsRegex(DeprecationWarning, r'default-scheduler-port is deprecated'): env_params = luigi.interface.core() - self.assertEqual(env_params.scheduler_port, 6543) + self.assertEqual(env_params.scheduler_port, 6543) @with_config({"core": {"scheduler-port": '6544'}}) def testOverrideSchedulerPort2(self): - if six.PY3: - with self.assertWarnsRegex(DeprecationWarning, r'scheduler-port \(with dashes\) should be avoided'): - env_params = luigi.interface.core() - else: + with self.assertWarnsRegex(DeprecationWarning, r'scheduler-port \(with dashes\) should be avoided'): env_params = luigi.interface.core() self.assertEqual(env_params.scheduler_port, 6544) diff --git a/test/task_test.py b/test/task_test.py index 74d579c40e..2fb9f33b84 100644 --- a/test/task_test.py +++ b/test/task_test.py @@ -17,7 +17,6 @@ import doctest import pickle -import six import warnings from helpers import unittest, LuigiTestCase @@ -137,23 +136,22 @@ def test_no_warn_if_param_types_ok(self): DummyTask(**DUMMY_TASK_OK_PARAMS) self.assertEqual(len(w), 0, msg='No warning should be raised when correct parameter types are used') - if six.PY3: # assertWarnsRegex was introduced in Python 3.2 - def test_warn_on_non_str_param(self): - params = dict(**DUMMY_TASK_OK_PARAMS) - params['param'] = 42 - with self.assertWarnsRegex(UserWarning, 'Parameter "param" with value "42" is not of type string.'): - DummyTask(**params) + def test_warn_on_non_str_param(self): + params = dict(**DUMMY_TASK_OK_PARAMS) + params['param'] = 42 + with self.assertWarnsRegex(UserWarning, 'Parameter "param" with value "42" is not of type string.'): + DummyTask(**params) - def test_warn_on_non_timedelta_param(self): - params = dict(**DUMMY_TASK_OK_PARAMS) + def test_warn_on_non_timedelta_param(self): + params = dict(**DUMMY_TASK_OK_PARAMS) - class MockTimedelta(object): - days = 1 - seconds = 1 + class MockTimedelta(object): + days = 1 + seconds = 1 - params['timedelta_param'] = MockTimedelta() - with self.assertWarnsRegex(UserWarning, 'Parameter "timedelta_param" with value ".*" is not of type timedelta.'): - DummyTask(**params) + params['timedelta_param'] = MockTimedelta() + with self.assertWarnsRegex(UserWarning, 'Parameter "timedelta_param" with value ".*" is not of type timedelta.'): + DummyTask(**params) class ExternalizeTaskTest(LuigiTestCase): diff --git a/test/worker_test.py b/test/worker_test.py index ae5e449e02..69796a8065 100644 --- a/test/worker_test.py +++ b/test/worker_test.py @@ -38,7 +38,6 @@ from luigi.scheduler import Scheduler from luigi.worker import Worker from luigi.rpc import RPCError -from luigi import six from luigi.cmdline import luigi_run luigi.notifications.DEBUG = True @@ -1484,9 +1483,6 @@ def complete(self): class MultipleWorkersTest(unittest.TestCase): @unittest.skip('Always skip. There are many intermittent failures') - # This pass under python3 when run as `nosetests test/worker_test.py` - # but not as `nosetests test`. Probably some side effect on previous tests - @unittest.skipIf(six.PY3, 'This test fail on python3 when run with tox.') def test_multiple_workers(self): # Test using multiple workers # Also test generating classes dynamically since this may reflect issues with