From 516ede9af60217d947740d46df3693f40557188a Mon Sep 17 00:00:00 2001 From: drowoseque Date: Mon, 2 Dec 2019 16:33:43 +0300 Subject: [PATCH] [luigi.six.string_types] and [luigi.six.binary_type] removed (#2838) * no string_types * no-binary_type * Update .travis.yml * Update .travis.yml * no py27 in travis (#2836) * no py27 in travis * py37 * flake8 and docs back * py36 py37 postgres * py36-apache * MixedUnicodeBytes removed * unused imports removed * no-if-py3 * space * space * MixedUnicodeBytesFormat returned * original * no ununsed import * Update .travis.yml * Fix params hashing (#2540) * move params freezing to separated module * freeze dict for toml parser * FIX FrozenOrderedDict name * Freeze only params of config, not all config * +docstring * [luigi.mock.MockFile] removed (#2839) * no-mock-file * MockFile removed * unused import * no py27 in travis (#2836) * no py27 in travis * py37 * flake8 and docs back * py36 py37 postgres * py36-apache * MixedUnicodeBytes removed * unused imports removed * no-if-py3 * space * space * MixedUnicodeBytesFormat returned * new string_types * no string_types * no-binary_type * original * no ununsed import * no py27 in travis (#2836) * no py27 in travis * py37 * flake8 and docs back * py36 py37 postgres * py36-apache * MixedUnicodeBytes removed * unused imports removed * no-if-py3 * space * space * MixedUnicodeBytesFormat returned * new string_types * snakebite * no-unused-import --- luigi/contrib/docker_runner.py | 3 +-- luigi/contrib/esindex.py | 4 +--- luigi/contrib/gcs.py | 4 ++-- luigi/contrib/postgres.py | 2 +- luigi/contrib/spark.py | 5 ++--- luigi/mock.py | 3 +-- luigi/parameter.py | 6 +++--- luigi/six.py | 4 ---- luigi/task.py | 4 ++-- luigi/tools/range.py | 2 +- test/contrib/hdfs_test.py | 3 +-- 11 files changed, 15 insertions(+), 25 deletions(-) diff --git a/luigi/contrib/docker_runner.py b/luigi/contrib/docker_runner.py index 7e0113db1c..9655190760 100644 --- a/luigi/contrib/docker_runner.py +++ b/luigi/contrib/docker_runner.py @@ -40,7 +40,6 @@ import luigi from luigi.local_target import LocalFileSystem -from luigi import six logger = logging.getLogger('luigi-interface') @@ -158,7 +157,7 @@ def __init__(self, *args, **kwargs): self.environment['LUIGI_TMP_DIR'] = self.container_tmp_dir # add additional volume binds specified by the user to the tmp_Dir bind - if isinstance(self.binds, six.string_types): + if isinstance(self.binds, str): self._binds.append(self.binds) elif isinstance(self.binds, list): self._binds.extend(self.binds) diff --git a/luigi/contrib/esindex.py b/luigi/contrib/esindex.py index fc046b2652..f1b799a819 100644 --- a/luigi/contrib/esindex.py +++ b/luigi/contrib/esindex.py @@ -90,8 +90,6 @@ def docs(self): import luigi -from luigi import six - logger = logging.getLogger('luigi-interface') try: @@ -366,7 +364,7 @@ def _docs(self): iterdocs = iter(self.docs()) first = next(iterdocs) needs_parsing = False - if isinstance(first, six.string_types): + if isinstance(first, str): needs_parsing = True elif isinstance(first, dict): pass diff --git a/luigi/contrib/gcs.py b/luigi/contrib/gcs.py index ff40e9a3c6..8c983fc25b 100644 --- a/luigi/contrib/gcs.py +++ b/luigi/contrib/gcs.py @@ -282,8 +282,8 @@ def put_multiple(self, filepaths, remote_directory, mimetype=None, chunksize=Non def put_string(self, contents, dest_path, mimetype=None): mimetype = mimetype or mimetypes.guess_type(dest_path)[0] or DEFAULT_MIMETYPE - assert isinstance(mimetype, six.string_types) - if not isinstance(contents, six.binary_type): + assert isinstance(mimetype, str) + if not isinstance(contents, bytes): contents = contents.encode("utf-8") media = http.MediaIoBaseUpload(six.BytesIO(contents), mimetype, resumable=bool(contents)) self._do_put(media, dest_path) diff --git a/luigi/contrib/postgres.py b/luigi/contrib/postgres.py index 15c15ba7d4..0e0e8d9eb4 100644 --- a/luigi/contrib/postgres.py +++ b/luigi/contrib/postgres.py @@ -282,7 +282,7 @@ def output(self): ) def copy(self, cursor, file): - if isinstance(self.columns[0], six.string_types): + if isinstance(self.columns[0], str): column_names = self.columns elif len(self.columns[0]) == 2: column_names = [c[0] for c in self.columns] diff --git a/luigi/contrib/spark.py b/luigi/contrib/spark.py index a2ec73dc16..28950828ea 100644 --- a/luigi/contrib/spark.py +++ b/luigi/contrib/spark.py @@ -28,7 +28,6 @@ import pickle -from luigi import six from luigi.contrib.external_program import ExternalProgramTask from luigi import configuration @@ -234,11 +233,11 @@ def app_command(self): return [self.app] + self.app_options() def _list_config(self, config): - if config and isinstance(config, six.string_types): + if config and isinstance(config, str): return list(map(lambda x: x.strip(), config.split(','))) def _dict_config(self, config): - if config and isinstance(config, six.string_types): + if config and isinstance(config, str): return dict(map(lambda i: i.split('=', 1), config.split('|'))) def _text_arg(self, name, value): diff --git a/luigi/mock.py b/luigi/mock.py index 730c7a3c1c..55af319b8b 100644 --- a/luigi/mock.py +++ b/luigi/mock.py @@ -25,7 +25,6 @@ import sys -from luigi import six from luigi import target from luigi.format import get_default_format @@ -139,7 +138,7 @@ def write(self, data): if mock_target._mirror_on_stderr: if self._write_line: sys.stderr.write(fn + ": ") - if six.binary_type: + if bytes: sys.stderr.write(data.decode('utf8')) else: sys.stderr.write(data) diff --git a/luigi/parameter.py b/luigi/parameter.py index 6ae82e03e0..5646044857 100644 --- a/luigi/parameter.py +++ b/luigi/parameter.py @@ -276,7 +276,7 @@ def serialize(self, x): def _warn_on_wrong_param_type(self, param_name, param_value): if self.__class__ != Parameter: return - if not isinstance(param_value, six.string_types): + if not isinstance(param_value, str): warnings.warn('Parameter "{}" with value "{}" is not of type string.'.format(param_name, param_value)) def normalize(self, x): @@ -339,7 +339,7 @@ def parse(self, x): def _warn_on_wrong_param_type(self, param_name, param_value): if self.__class__ != OptionalParameter: return - if not isinstance(param_value, six.string_types) and param_value is not None: + if not isinstance(param_value, str) and param_value is not None: warnings.warn('OptionalParameter "{}" with value "{}" is not of type string or None.'.format( param_name, param_value)) @@ -949,7 +949,7 @@ def parse(self, source): :param s: String to be parse """ # TOML based config convert params to python types itself. - if not isinstance(source, six.string_types): + if not isinstance(source, str): return source return json.loads(source, object_pairs_hook=FrozenOrderedDict) diff --git a/luigi/six.py b/luigi/six.py index da09da4350..4dbd932468 100644 --- a/luigi/six.py +++ b/luigi/six.py @@ -36,19 +36,15 @@ PY34 = sys.version_info[0:2] >= (3, 4) if PY3: - string_types = str, integer_types = int, class_types = type, text_type = str - binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, integer_types = (int, long) class_types = (type, types.ClassType) text_type = unicode - binary_type = str if sys.platform.startswith("java"): # Jython always uses 32 bits. diff --git a/luigi/task.py b/luigi/task.py index 110420036a..c434a3683c 100644 --- a/luigi/task.py +++ b/luigi/task.py @@ -237,7 +237,7 @@ def _owner_list(self): owner_email = self.owner_email if owner_email is None: return [] - elif isinstance(owner_email, six.string_types): + elif isinstance(owner_email, str): return owner_email.split(',') else: return owner_email @@ -870,7 +870,7 @@ def flatten(struct): for _, result in six.iteritems(struct): flat += flatten(result) return flat - if isinstance(struct, six.string_types): + if isinstance(struct, str): return [struct] try: diff --git a/luigi/tools/range.py b/luigi/tools/range.py index 9553dd082b..54d2613388 100755 --- a/luigi/tools/range.py +++ b/luigi/tools/range.py @@ -118,7 +118,7 @@ def of_cls(self): """ DONT USE. Will be deleted soon. Use ``self.of``! """ - if isinstance(self.of, six.string_types): + if isinstance(self.of, str): warnings.warn('When using Range programatically, dont pass "of" param as string!') return Register.get_task_cls(self.of) return self.of diff --git a/test/contrib/hdfs_test.py b/test/contrib/hdfs_test.py index 420bbd06d2..f2cb1c48f2 100644 --- a/test/contrib/hdfs_test.py +++ b/test/contrib/hdfs_test.py @@ -22,7 +22,6 @@ import luigi import luigi.format from luigi.contrib import hdfs -from luigi import six import luigi.contrib.hdfs.clients from target_test import FileSystemTargetTestMixin @@ -171,7 +170,7 @@ def test_glob_exists(self): def assertRegexpMatches(self, text, expected_regexp, msg=None): """Python 2.7 backport.""" - if isinstance(expected_regexp, six.string_types): + if isinstance(expected_regexp, str): expected_regexp = re.compile(expected_regexp) if not expected_regexp.search(text): msg = msg or "Regexp didn't match"