Merge pull request #36 from xmnlab/master

from master:  ENH: Refactor the pandas backend

ci/requirements-dev-2.7.yml

@@ -27,6 +27,7 @@ dependencies:
   - pymysql
   - pytables
   - pytest
+  - pytest-xdist
   - python=2.7
   - python-graphviz
   - python-hdfs>=2.0.16

ci/requirements-dev-3.5.yml

@@ -21,6 +21,7 @@ dependencies:
   - pymapd
   - pymysql
   - pytest
+  - pytest-xdist
   - python=3.5
   - python-graphviz
   - python-hdfs>=2.0.16

ci/requirements-dev-3.6.yml

@@ -22,6 +22,7 @@ dependencies:
   - pymysql
   - pytables
   - pytest
+  - pytest-xdist
   - python=3.6
   - python-graphviz
   - python-hdfs>=2.0.16

ci/requirements-docs-3.6.yml

@@ -26,6 +26,7 @@ dependencies:
   - pymysql
   - pytables
   - pytest
+  - pytest-xdist
   - python=3.6
   - python-graphviz
   - python-hdfs>=2.0.16

ibis/bigquery/udf/tests/test_find.py

@@ -1,6 +1,6 @@
 import ast
 from ibis.bigquery.udf.find import find_names
-from ibis.util import is_sequence
+from ibis.util import is_iterable
 
 
 def parse_expr(expr):
@@ -17,7 +17,7 @@ def eq(left, right):
     if type(left) != type(right):
         return False
 
-    if is_sequence(left) and is_sequence(right):
+    if is_iterable(left) and is_iterable(right):
         return all(map(eq, left, right))
 
     if not isinstance(left, ast.AST) and not isinstance(right, ast.AST):

ibis/compat.py

@@ -28,6 +28,7 @@ def viewkeys(x):
     from inspect import signature, Parameter, _empty
     import unittest.mock as mock
     range = range
+    map = map
     import builtins
     import pickle
     maketrans = str.maketrans
@@ -45,6 +46,7 @@ def viewkeys(x):
     lzip = zip
     zip = itertools.izip
     zip_longest = itertools.izip_longest
+    map = itertools.imap
 
     def viewkeys(x):
         return x.viewkeys()

ibis/expr/api.py

@@ -924,17 +924,15 @@ def _case(arg):
         string_col = Column[string*] 'string_col' from table
           ref_0
       cases:
-        ValueList[string*]
-          Literal[string]
-            a
-          Literal[string]
-            b
+        Literal[string]
+          a
+        Literal[string]
+          b
       results:
-        ValueList[string*]
-          Literal[string]
-            an a
-          Literal[string]
-            a b
+        Literal[string]
+          an a
+        Literal[string]
+          a b
       default:
         Literal[string]
           null or (not a and not b)

ibis/expr/format.py

@@ -218,7 +218,7 @@ def visit(what, extra_indents=0):
 
         if not arg_names:
             for arg in op.args:
-                if isinstance(arg, list):
+                if util.is_iterable(arg):
                     for x in arg:
                         visit(x)
                 else:
@@ -230,7 +230,7 @@ def visit(what, extra_indents=0):
                     name = None
                 if name is not None:
                     name = self._indent('{0}:'.format(name))
-                if isinstance(arg, list):
+                if util.is_iterable(arg):
                     if name is not None and len(arg) > 0:
                         formatted_args.append(name)
                         indents = 1

ibis/expr/operations.py

@@ -4,16 +4,16 @@
 import itertools
 import collections
 
-from functools import partial
 from ibis.expr.schema import HasSchema, Schema
 
-import ibis.util as util
 import ibis.common as com
-import ibis.compat as compat
 import ibis.expr.types as ir
 import ibis.expr.rules as rlz
 import ibis.expr.schema as sch
 import ibis.expr.datatypes as dt
+
+from ibis import util, compat
+from ibis.compat import functools, map
 from ibis.expr.signature import Annotable, Argument as Arg
 
 
@@ -56,6 +56,10 @@ def _pp(x):
 
         return '%s(%s)' % (opname, ', '.join(pprint_args))
 
+    @property
+    def inputs(self):
+        return tuple(self.args)
+
     def blocks(self):
         # The contents of this node at referentially distinct and may not be
         # analyzed deeper
@@ -134,13 +138,9 @@ def has_resolved_name(self):
 
 
 def all_equal(left, right, cache=None):
-    if isinstance(left, list):
-        if not isinstance(right, list):
-            return False
-        for a, b in zip(left, right):
-            if not all_equal(a, b, cache=cache):
-                return False
-        return True
+    if util.is_iterable(left):
+        return util.is_iterable(right) and all(
+            map(functools.partial(all_equal, cache=cache), left, right))
 
     if hasattr(left, 'equals'):
         return left.equals(right, cache=cache)
@@ -781,7 +781,7 @@ class Count(Reduction):
     where = Arg(rlz.boolean, default=None)
 
     def output_type(self):
-        return partial(ir.IntegerScalar, dtype=dt.int64)
+        return functools.partial(ir.IntegerScalar, dtype=dt.int64)
 
 
 class Arbitrary(Reduction):
@@ -911,7 +911,7 @@ class HLLCardinality(Reduction):
     def output_type(self):
         # Impala 2.0 and higher returns a DOUBLE
         # return ir.DoubleScalar
-        return partial(ir.IntegerScalar, dtype=dt.int64)
+        return functools.partial(ir.IntegerScalar, dtype=dt.int64)
 
 
 class GroupConcat(Reduction):
@@ -967,6 +967,10 @@ def over(self, window):
         new_window = self.window.combine(window)
         return WindowOp(self.expr, new_window)
 
+    @property
+    def inputs(self):
+        return self.expr.op().inputs[0], self.window
+
     def root_tables(self):
         result = list(toolz.unique(
             toolz.concatv(
@@ -985,7 +989,7 @@ def root_tables(self):
 
 class ShiftBase(AnalyticOp):
     arg = Arg(rlz.column(rlz.any))
-    offset = Arg(rlz.integer, default=None)
+    offset = Arg(rlz.one_of((rlz.integer, rlz.interval)), default=None)
     default = Arg(rlz.any, default=None)
     output_type = rlz.typeof('arg')
 
@@ -2299,7 +2303,7 @@ def output_type(self):
 class E(Constant):
 
     def output_type(self):
-        return partial(ir.FloatingScalar, dtype=dt.float64)
+        return functools.partial(ir.FloatingScalar, dtype=dt.float64)
 
 
 class Pi(Constant):
@@ -2771,8 +2775,7 @@ class ExpressionList(Node):
     exprs = Arg(rlz.noop)
 
     def __init__(self, values):
-        values = list(map(rlz.any, values))
-        super(ExpressionList, self).__init__(values)
+        super(ExpressionList, self).__init__(list(map(rlz.any, values)))
 
     def root_tables(self):
         return distinct_roots(self.exprs)
@@ -2788,16 +2791,15 @@ class ValueList(ValueOp):
     display_argnames = False  # disable showing argnames in repr
 
     def __init__(self, values):
-        values = list(map(rlz.any, values))
-        super(ValueList, self).__init__(values)
+        super(ValueList, self).__init__(tuple(map(rlz.any, values)))
+
+    def output_type(self):
+        dtype = rlz.highest_precedence_dtype(self.values)
+        return functools.partial(ir.ListExpr, dtype=dtype)
 
     def root_tables(self):
         return distinct_roots(*self.values)
 
-    def _make_expr(self):
-        dtype = rlz.highest_precedence_dtype(self.values)
-        return ir.ListExpr(self, dtype=dtype)
-
 
 # GEOMETRIC OPERATIONS
 

ibis/expr/types.py

@@ -680,11 +680,11 @@ def __getitem__(self, key):
         return self.values[key]
 
     def __add__(self, other):
-        other_values = getattr(other, 'values', other)
+        other_values = tuple(getattr(other, 'values', other))
         return type(self.op())(self.values + other_values).to_expr()
 
     def __radd__(self, other):
-        other_values = getattr(other, 'values', other)
+        other_values = tuple(getattr(other, 'values', other))
         return type(self.op())(other_values + self.values).to_expr()
 
     def __bool__(self):

ibis/expr/window.py

@@ -1,17 +1,3 @@
-# Copyright 2014 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import ibis.expr.types as ir
 import ibis.expr.operations as ops
 import ibis.util as util
@@ -58,38 +44,50 @@ def __init__(self, group_by=None, order_by=None,
         self._validate_frame()
 
     def _validate_frame(self):
-        p_tuple = has_p = False
-        f_tuple = has_f = False
+        preceding_tuple = has_preceding = False
+        following_tuple = has_following = False
         if self.preceding is not None:
-            p_tuple = isinstance(self.preceding, tuple)
-            has_p = True
+            preceding_tuple = isinstance(self.preceding, tuple)
+            has_preceding = True
 
         if self.following is not None:
-            f_tuple = isinstance(self.following, tuple)
-            has_f = True
-
-        if ((p_tuple and has_f) or (f_tuple and has_p)):
-            raise com.IbisInputError('Can only specify one window side '
-                                     ' when you want an off-center '
-                                     'window')
-        elif p_tuple:
+            following_tuple = isinstance(self.following, tuple)
+            has_following = True
+
+        if ((preceding_tuple and has_following) or
+                (following_tuple and has_preceding)):
+            raise com.IbisInputError(
+                'Can only specify one window side when you want an '
+                'off-center window'
+            )
+        elif preceding_tuple:
             start, end = self.preceding
             if start is None:
                 assert end >= 0
             else:
                 assert start > end
-        elif f_tuple:
+        elif following_tuple:
             start, end = self.following
             if end is None:
                 assert start >= 0
             else:
                 assert start < end
         else:
-            if has_p and self.preceding < 0:
-                raise com.IbisInputError('Window offset must be positive')
-
-            if has_f and self.following < 0:
-                raise com.IbisInputError('Window offset must be positive')
+            if not isinstance(self.preceding, ir.Expr):
+                if has_preceding and self.preceding < 0:
+                    raise com.IbisInputError(
+                        "'preceding' must be positive, got {}".format(
+                            self.preceding
+                        )
+                    )
+
+            if not isinstance(self.following, ir.Expr):
+                if has_following and self.following < 0:
+                    raise com.IbisInputError(
+                        "'following' must be positive, got {}".format(
+                            self.following
+                        )
+                    )
 
     def bind(self, table):
         # Internal API, ensure that any unresolved expr references (as strings,
@@ -128,32 +126,36 @@ def equals(self, other, cache=None):
         if cache is None:
             cache = {}
 
-        if (self, other) in cache:
-            return cache[(self, other)]
-
-        if id(self) == id(other):
-            cache[(self, other)] = True
+        if self is other:
+            cache[self, other] = True
             return True
 
         if not isinstance(other, Window):
-            cache[(self, other)] = False
+            cache[self, other] = False
             return False
 
+        try:
+            return cache[self, other]
+        except KeyError:
+            pass
+
         if (len(self._group_by) != len(other._group_by) or
                 not ops.all_equal(self._group_by, other._group_by,
                                   cache=cache)):
-            cache[(self, other)] = False
+            cache[self, other] = False
             return False
 
         if (len(self._order_by) != len(other._order_by) or
                 not ops.all_equal(self._order_by, other._order_by,
                                   cache=cache)):
-            cache[(self, other)] = False
+            cache[self, other] = False
             return False
 
-        equal = (self.preceding == other.preceding and
-                 self.following == other.following)
-        cache[(self, other)] = equal
+        equal = (
+            ops.all_equal(self.preceding, other.preceding, cache=cache) and
+            ops.all_equal(self.following, other.following, cache=cache)
+        )
+        cache[self, other] = equal
         return equal