snakebite removed (#2833)

* snamebite removed

* no minicluster

* no minicluster

* flake8

* branches - master

* no snakebite in tox.ini

luigi/contrib/hdfs/__init__.py

@@ -18,7 +18,7 @@
 """
 Provides access to HDFS using the :py:class:`HdfsTarget`, a subclass of :py:class:`~luigi.target.Target`.
 You can configure what client by setting the "client" config under the "hdfs" section in the configuration, or using the ``--hdfs-client`` command line option.
-"hadoopcli" is the slowest, but should work out of the box. "snakebite" is the fastest, but requires Snakebite to be installed.
+"hadoopcli" is the slowest, but should work out of the box.
 
 Since the hdfs functionality is quite big in luigi, it's split into smaller
 files under ``luigi/contrib/hdfs/*.py``. But for the sake of convenience and
@@ -29,7 +29,6 @@
 from luigi.contrib.hdfs import config as hdfs_config
 from luigi.contrib.hdfs import clients as hdfs_clients
 from luigi.contrib.hdfs import error as hdfs_error
-from luigi.contrib.hdfs import snakebite_client as hdfs_snakebite_client
 from luigi.contrib.hdfs import hadoopcli_clients as hdfs_hadoopcli_clients
 from luigi.contrib.hdfs import webhdfs_client as hdfs_webhdfs_client
 from luigi.contrib.hdfs import format as hdfs_format
@@ -47,9 +46,7 @@
 # clients
 HDFSCliError = hdfs_error.HDFSCliError
 call_check = hdfs_hadoopcli_clients.HdfsClient.call_check
-list_path = hdfs_snakebite_client.SnakebiteHdfsClient.list_path
 HdfsClient = hdfs_hadoopcli_clients.HdfsClient
-SnakebiteHdfsClient = hdfs_snakebite_client.SnakebiteHdfsClient
 WebHdfsClient = hdfs_webhdfs_client.WebHdfsClient
 HdfsClientCdh3 = hdfs_hadoopcli_clients.HdfsClientCdh3
 HdfsClientApache1 = hdfs_hadoopcli_clients.HdfsClientApache1

luigi/contrib/hdfs/clients.py

@@ -16,17 +16,14 @@
 #
 
 """
-The implementations of the hdfs clients. The hadoop cli client and the
-snakebite client.
+The implementations of the hdfs clients.
 """
 import logging
 import threading
 
 from luigi.contrib.hdfs import config as hdfs_config
-from luigi.contrib.hdfs import snakebite_client as hdfs_snakebite_client
 from luigi.contrib.hdfs import webhdfs_client as hdfs_webhdfs_client
 from luigi.contrib.hdfs import hadoopcli_clients as hdfs_hadoopcli_clients
-import luigi.contrib.target
 
 logger = logging.getLogger('luigi-interface')
 
@@ -43,13 +40,6 @@ def get_autoconfig_client(client_cache=_AUTOCONFIG_CLIENT):
         configured_client = hdfs_config.get_configured_hdfs_client()
         if configured_client == "webhdfs":
             client_cache.client = hdfs_webhdfs_client.WebHdfsClient()
-        elif configured_client == "snakebite":
-            client_cache.client = hdfs_snakebite_client.SnakebiteHdfsClient()
-        elif configured_client == "snakebite_with_hadoopcli_fallback":
-            client_cache.client = luigi.contrib.target.CascadingClient([
-                hdfs_snakebite_client.SnakebiteHdfsClient(),
-                hdfs_hadoopcli_clients.create_hadoopcli_client(),
-            ])
         elif configured_client == "hadoopcli":
             client_cache.client = hdfs_hadoopcli_clients.create_hadoopcli_client()
         else:

luigi/contrib/hdfs/config.py

@@ -17,14 +17,12 @@
 
 """
 You can configure what client by setting the "client" config under the "hdfs" section in the configuration, or using the ``--hdfs-client`` command line option.
-"hadoopcli" is the slowest, but should work out of the box. "snakebite" is the fastest, but requires Snakebite to be installed.
+"hadoopcli" is the slowest, but should work out of the box.
 """
 
 import random
 import luigi
 import luigi.configuration
-from luigi import six
-import warnings
 import os
 import getpass
 
@@ -33,12 +31,6 @@
 
 class hdfs(luigi.Config):
     client_version = luigi.IntParameter(default=None)
-    effective_user = luigi.OptionalParameter(
-        default=os.getenv('HADOOP_USER_NAME'),
-        description="Optionally specifies the effective user for snakebite. "
-                    "If not set the environment variable HADOOP_USER_NAME is "
-                    "used, else USER")
-    snakebite_autoconfig = luigi.BoolParameter(default=False)
     namenode_host = luigi.OptionalParameter(default=None)
     namenode_port = luigi.IntParameter(default=None)
     client = luigi.Parameter(default='hadoopcli')
@@ -52,7 +44,7 @@ class hadoopcli(luigi.Config):
     command = luigi.Parameter(default="hadoop",
                               config_path=dict(section="hadoop", name="command"),
                               description='The hadoop command, will run split() on it, '
-                              'so you can pass something like "hadoop --param"')
+                                          'so you can pass something like "hadoop --param"')
     version = luigi.Parameter(default="cdh4",
                               config_path=dict(section="hadoop", name="version"),
                               description='Can also be cdh3 or apache1')
@@ -80,20 +72,7 @@ def get_configured_hdfs_client():
     the [hdfs] section. It will return the client that retains backwards
     compatibility when 'client' isn't configured.
     """
-    config = hdfs()
-    custom = config.client
-    conf_usinf_snakebite = [
-        "snakebite_with_hadoopcli_fallback",
-        "snakebite",
-    ]
-    if six.PY3 and (custom in conf_usinf_snakebite):
-        warnings.warn(
-            "snakebite client not compatible with python3 at the moment"
-            "falling back on hadoopcli",
-            stacklevel=2
-        )
-        return "hadoopcli"
-    return custom
+    return hdfs().client
 
 
 def tmppath(path=None, include_unix_username=True):

luigi/contrib/hdfs/error.py

@@ -16,8 +16,7 @@
 #
 
 """
-The implementations of the hdfs clients. The hadoop cli client and the
-snakebite client.
+The implementations of the hdfs clients.
 """
 
 

luigi/contrib/hdfs/hadoopcli_clients.py

@@ -16,8 +16,7 @@
 #
 
 """
-The implementations of the hdfs clients. The hadoop cli client and the
-snakebite client.
+The implementations of the hdfs clients.
 """
 
 
@@ -38,7 +37,7 @@
 
 def create_hadoopcli_client():
     """
-    Given that we want one of the hadoop cli clients (unlike snakebite),
+    Given that we want one of the hadoop cli clients,
     this one will return the right one.
     """
     version = hdfs_config.get_configured_hadoop_version()