From e0de2e416cb418cb696eebb690f6c890ee15003c Mon Sep 17 00:00:00 2001 From: drowoseque Date: Tue, 3 Dec 2019 13:51:19 +0300 Subject: [PATCH] abstractproperty -> property + abstractmethod (#2847) * Update .travis.yml * no py27 in travis (#2836) * no py27 in travis * py37 * flake8 and docs back * py36 py37 postgres * py36-apache * MixedUnicodeBytes removed * unused imports removed * no-if-py3 * space * space * MixedUnicodeBytesFormat returned * abstractproperty -> property + abstractmethod * Update beam_dataflow.py * Update beam_dataflow.py --- luigi/contrib/beam_dataflow.py | 57 ++++++++++++++++++++++------------ luigi/contrib/esindex.py | 3 +- luigi/contrib/pai.py | 9 ++++-- luigi/contrib/rdbms.py | 33 +++++++++++++------- luigi/contrib/redshift.py | 21 ++++++++----- luigi/contrib/salesforce.py | 6 ++-- luigi/contrib/sqla.py | 6 ++-- luigi/parameter.py | 9 ++++-- luigi/target.py | 3 +- test/subtask_test.py | 3 +- 10 files changed, 99 insertions(+), 51 deletions(-) diff --git a/luigi/contrib/beam_dataflow.py b/luigi/contrib/beam_dataflow.py index 386d361177..eec77baaa0 100644 --- a/luigi/contrib/beam_dataflow.py +++ b/luigi/contrib/beam_dataflow.py @@ -16,7 +16,6 @@ # import abc -from abc import abstractmethod, abstractproperty import logging import json import os @@ -36,75 +35,93 @@ class DataflowParamKeys(metaclass=abc.ABCMeta): the Python implementation expects snake case. """ - @abstractproperty + @property + @abc.abstractmethod def runner(self): pass - @abstractproperty + @property + @abc.abstractmethod def project(self): pass - @abstractproperty + @property + @abc.abstractmethod def zone(self): pass - @abstractproperty + @property + @abc.abstractmethod def region(self): pass - @abstractproperty + @property + @abc.abstractmethod def staging_location(self): pass - @abstractproperty + @property + @abc.abstractmethod def temp_location(self): pass - @abstractproperty + @property + @abc.abstractmethod def gcp_temp_location(self): pass - @abstractproperty + @property + @abc.abstractmethod def num_workers(self): pass - @abstractproperty + @property + @abc.abstractmethod def autoscaling_algorithm(self): pass - @abstractproperty + @property + @abc.abstractmethod def max_num_workers(self): pass - @abstractproperty + @property + @abc.abstractmethod def disk_size_gb(self): pass - @abstractproperty + @property + @abc.abstractmethod def worker_machine_type(self): pass - @abstractproperty + @property + @abc.abstractmethod def worker_disk_type(self): pass - @abstractproperty + @property + @abc.abstractmethod def job_name(self): pass - @abstractproperty + @property + @abc.abstractmethod def service_account(self): pass - @abstractproperty + @property + @abc.abstractmethod def network(self): pass - @abstractproperty + @property + @abc.abstractmethod def subnetwork(self): pass - @abstractproperty + @property + @abc.abstractmethod def labels(self): pass @@ -218,7 +235,7 @@ def __init__(self): raise ValueError("dataflow_params must be of type DataflowParamKeys") super(BeamDataflowJobTask, self).__init__() - @abstractmethod + @abc.abstractmethod def dataflow_executable(self): """ Command representing the Dataflow executable to be run. diff --git a/luigi/contrib/esindex.py b/luigi/contrib/esindex.py index f1b799a819..d7f71dfbd3 100644 --- a/luigi/contrib/esindex.py +++ b/luigi/contrib/esindex.py @@ -272,7 +272,8 @@ def http_auth(self): """ return None - @abc.abstractproperty + @property + @abc.abstractmethod def index(self): """ The target index. diff --git a/luigi/contrib/pai.py b/luigi/contrib/pai.py index 280b7f952a..792ef49177 100644 --- a/luigi/contrib/pai.py +++ b/luigi/contrib/pai.py @@ -181,17 +181,20 @@ class OpenPai(luigi.Config): class PaiTask(luigi.Task): __POLL_TIME = 5 - @abc.abstractproperty + @property + @abc.abstractmethod def name(self): """Name for the job, need to be unique, required""" return 'SklearnExample' - @abc.abstractproperty + @property + @abc.abstractmethod def image(self): """URL pointing to the Docker image for all tasks in the job, required""" return 'openpai/pai.example.sklearn' - @abc.abstractproperty + @property + @abc.abstractmethod def tasks(self): """List of taskRole, one task role at least, required""" return [] diff --git a/luigi/contrib/rdbms.py b/luigi/contrib/rdbms.py index 7af6f9d0da..52777693e9 100644 --- a/luigi/contrib/rdbms.py +++ b/luigi/contrib/rdbms.py @@ -162,23 +162,28 @@ class CopyToTable(luigi.task.MixinNaiveBulkComplete, _MetadataColumnsMixin, luig * `port` """ - @abc.abstractproperty + @property + @abc.abstractmethod def host(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def database(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def user(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def password(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def table(self): return None @@ -287,7 +292,8 @@ class Query(luigi.task.MixinNaiveBulkComplete, luigi.Task): * `output` """ - @abc.abstractproperty + @property + @abc.abstractmethod def host(self): """ Host of the RDBMS. Implementation should support `hostname:port` @@ -302,23 +308,28 @@ def port(self): """ return None - @abc.abstractproperty + @property + @abc.abstractmethod def database(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def user(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def password(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def table(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def query(self): return None diff --git a/luigi/contrib/redshift.py b/luigi/contrib/redshift.py index 7b79dcdf9f..06d18e6a7c 100644 --- a/luigi/contrib/redshift.py +++ b/luigi/contrib/redshift.py @@ -169,7 +169,8 @@ def s3_load_path(self): """ return None - @abc.abstractproperty + @property + @abc.abstractmethod def copy_options(self): """ Add extra copy options, for example: @@ -526,14 +527,16 @@ class S3CopyJSONToTable(S3CopyToTable, _CredentialsMixin): configuration or environment variables. """ - @abc.abstractproperty + @property + @abc.abstractmethod def jsonpath(self): """ Override the jsonpath schema location for the table. """ return '' - @abc.abstractproperty + @property + @abc.abstractmethod def copy_json_options(self): """ Add extra copy options, for example: @@ -622,19 +625,23 @@ class KillOpenRedshiftSessions(luigi.Task): # 30 seconds is usually fine; 60 is conservative connection_reset_wait_seconds = luigi.IntParameter(default=60) - @abc.abstractproperty + @property + @abc.abstractmethod def host(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def database(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def user(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def password(self): return None diff --git a/luigi/contrib/salesforce.py b/luigi/contrib/salesforce.py index c20904de6d..194fd8e45b 100644 --- a/luigi/contrib/salesforce.py +++ b/luigi/contrib/salesforce.py @@ -105,7 +105,8 @@ class salesforce(luigi.Config): class QuerySalesforce(Task): - @abc.abstractproperty + @property + @abc.abstractmethod def object_name(self): """ Override to return the SF object we are querying. @@ -126,7 +127,8 @@ def sandbox_name(self): """Override to specify the sandbox name if it is intended to be used.""" return None - @abc.abstractproperty + @property + @abc.abstractmethod def soql(self): """Override to return the raw string SOQL or the path to it.""" return None diff --git a/luigi/contrib/sqla.py b/luigi/contrib/sqla.py index 948ecc4f05..9537826ad3 100644 --- a/luigi/contrib/sqla.py +++ b/luigi/contrib/sqla.py @@ -284,11 +284,13 @@ class CopyToTable(luigi.Task): echo = False connect_args = {} - @abc.abstractproperty + @property + @abc.abstractmethod def connection_string(self): return None - @abc.abstractproperty + @property + @abc.abstractmethod def table(self): return None diff --git a/luigi/parameter.py b/luigi/parameter.py index c8869913ac..c9c3def4d8 100644 --- a/luigi/parameter.py +++ b/luigi/parameter.py @@ -357,7 +357,8 @@ def __init__(self, interval=1, start=None, **kwargs): self.interval = interval self.start = start if start is not None else _UNIX_EPOCH.date() - @abc.abstractproperty + @property + @abc.abstractmethod def date_format(self): """ Override me with a :py:meth:`~datetime.date.strftime` string. @@ -501,14 +502,16 @@ def __init__(self, interval=1, start=None, **kwargs): self.interval = interval self.start = start if start is not None else _UNIX_EPOCH - @abc.abstractproperty + @property + @abc.abstractmethod def date_format(self): """ Override me with a :py:meth:`~datetime.date.strftime` string. """ pass - @abc.abstractproperty + @property + @abc.abstractmethod def _timedelta(self): """ How to move one interval of this type forward (i.e. not counting self.interval). diff --git a/luigi/target.py b/luigi/target.py index 1a6794e381..904dcfe834 100644 --- a/luigi/target.py +++ b/luigi/target.py @@ -215,7 +215,8 @@ def __init__(self, path): # cast to str to allow path to be objects like pathlib.PosixPath and py._path.local.LocalPath self.path = str(path) - @abc.abstractproperty + @property + @abc.abstractmethod def fs(self): """ The :py:class:`FileSystem` associated with this FileSystemTarget. diff --git a/test/subtask_test.py b/test/subtask_test.py index 98640be8c1..2e197f312e 100644 --- a/test/subtask_test.py +++ b/test/subtask_test.py @@ -24,7 +24,8 @@ class AbstractTask(luigi.Task): k = luigi.IntParameter() - @abc.abstractproperty + @property + @abc.abstractmethod def foo(self): raise NotImplementedError