Merge pull request #60 from xmnlab/master

 Corrections from the revision

ci/datamgr.py

@@ -76,16 +76,18 @@ def convert_to_database_compatible_value(value):
     """
     if pd.isnull(value):
         return None
-    elif isinstance(value, pd.Timestamp):
+    if isinstance(value, pd.Timestamp):
         return value.to_pydatetime()
-    else:
+    try:
+        return value.item()
+    except AttributeError:
         return value
 
 
 def insert(engine, tablename, df):
     keys = df.columns
     rows = [
-        dict(zip(keys, tuple(map(convert_to_database_compatible_value, row))))
+        dict(zip(keys, map(convert_to_database_compatible_value, row)))
         for row in df.itertuples(index=False, name=None)
     ]
     t = sa.Table(tablename, sa.MetaData(bind=engine), autoload=True)

ci/requirements-dev-2.7.yml

@@ -18,7 +18,7 @@ dependencies:
   - mock
   - multipledispatch
   - numpy=1.11.*
-  - pandas
+  - pandas=0.20
   - pathlib2
   - plumbum
   - psycopg2

ci/requirements-dev-3.5.yml

@@ -14,13 +14,12 @@ dependencies:
   - lz4
   - multipledispatch
   - numpy=1.12.0
-  - pandas
+  - pandas=0.22
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
   - pymapd>=0.3.2
   - pymysql
-  - pytables
   - pytest
   - pytest-xdist
   - python=3.5
@@ -31,7 +30,6 @@ dependencies:
   - ruamel.yaml
   - six
   - sqlalchemy>=1.0.0,<1.1.15
-  - thrift
   - toolz
   - xorg-libxpm
   - xorg-libxrender

ci/requirements-docs-3.6.yml

@@ -22,7 +22,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
-  - pymapd
+  - pymapd>=0.3.2
   - pymysql
   - pytables
   - pytest

docs/source/api.rst

@@ -440,6 +440,7 @@ All string operations are valid either on scalar or array values
    StringValue.capitalize
    StringValue.contains
    StringValue.like
+   StringValue.to_timestamp
    StringValue.parse_url
    StringValue.substr
    StringValue.left

ibis/__init__.py

@@ -74,8 +74,8 @@
 
 with suppress(ImportError):
     # pip install ibis-framework[mapd]
-    if sys.version_info[0] < 3:
-        raise ImportError('ibis.mapd is not allowed it for Python 2.')
+    if sys.version_info.major < 3:
+        raise ImportError('The MapD backend is not supported under Python 2.')
     import ibis.mapd.api as mapd
 
 restart_ordering()

ibis/bigquery/compiler.py

@@ -454,6 +454,21 @@ def compiles_strftime(translator, expr):
     )
 
 
+@compiles(ops.StringToTimestamp)
+def compiles_string_to_timestamp(translator, expr):
+    arg, format_string, timezone_arg = expr.op().args
+    fmt_string = translator.translate(format_string)
+    arg_formatted = translator.translate(arg)
+    if timezone_arg is not None:
+        timezone_str = translator.translate(timezone_arg)
+        return 'PARSE_TIMESTAMP({}, {}, {})'.format(
+            fmt_string,
+            arg_formatted,
+            timezone_str
+        )
+    return 'PARSE_TIMESTAMP({}, {})'.format(fmt_string, arg_formatted)
+
+
 @rewrites(ops.Any)
 def bigquery_rewrite_any(expr):
     arg, = expr.op().args

ibis/bigquery/tests/test_client.py

@@ -1,6 +1,7 @@
 import collections
 
 from datetime import date, datetime
+import pytz
 
 import pytest
 
@@ -529,6 +530,20 @@ def test_large_timestamp(client):
     assert result == huge_timestamp
 
 
+def test_string_to_timestamp(client):
+    timestamp = pd.Timestamp(datetime(year=2017, month=2, day=6),
+                             tz=pytz.timezone('UTC'))
+    expr = ibis.literal('2017-02-06').to_timestamp('%F')
+    result = client.execute(expr)
+    assert result == timestamp
+
+    timestamp_tz = pd.Timestamp(datetime(year=2017, month=2, day=6, hour=5),
+                                tz=pytz.timezone('UTC'))
+    expr_tz = ibis.literal('2017-02-06').to_timestamp('%F', 'America/New_York')
+    result_tz = client.execute(expr_tz)
+    assert result_tz == timestamp_tz
+
+
 def test_client_sql_query(client):
     expr = client.sql('select * from testing.functional_alltypes limit 20')
     result = expr.execute()

ibis/bigquery/tests/test_compiler.py

@@ -55,3 +55,24 @@ def test_identical_to(alltypes):
 WHERE (((`string_col` IS NULL) AND ('a' IS NULL)) OR (`string_col` = 'a')) AND
       (((`date_string_col` IS NULL) AND ('b' IS NULL)) OR (`date_string_col` = 'b'))"""  # noqa: E501
     assert result == expected
+
+
+@pytest.mark.parametrize(
+    'timezone',
+    [
+        None,
+        'America/New_York'
+    ]
+)
+def test_to_timestamp(alltypes, timezone):
+    expr = alltypes.date_string_col.to_timestamp('%F', timezone)
+    result = expr.compile()
+    if timezone:
+        expected = """\
+SELECT PARSE_TIMESTAMP('%F', `date_string_col`, 'America/New_York') AS `tmp`
+FROM `ibis-gbq.testing.functional_alltypes`"""
+    else:
+        expected = """\
+SELECT PARSE_TIMESTAMP('%F', `date_string_col`) AS `tmp`
+FROM `ibis-gbq.testing.functional_alltypes`"""
+    assert result == expected

ibis/expr/api.py

@@ -66,7 +66,7 @@
     'negate', 'ifelse',
     'Expr', 'Schema',
     'window', 'trailing_window', 'cumulative_window',
-    'pi', 'distance'
+    'pi'
 ]
 
 
@@ -1890,6 +1890,29 @@ def _string_replace(arg, pattern, replacement):
     return ops.StringReplace(arg, pattern, replacement).to_expr()
 
 
+def to_timestamp(arg, format_str, timezone=None):
+    """
+    Parses a string and returns a timestamp.
+
+    Parameters
+    ----------
+    format_str : A format string potentially of the type '%Y-%m-%d'
+    timezone : An optional string indicating the timezone,
+        i.e. 'America/New_York'
+
+    Examples
+    --------
+    >>> import ibis
+    >>> date_as_str = ibis.literal('20170206')
+    >>> result = date_as_str.to_timestamp('%Y%m%d')
+
+    Returns
+    -------
+    parsed : TimestampValue
+    """
+    return ops.StringToTimestamp(arg, format_str, timezone).to_expr()
+
+
 def parse_url(arg, extract, key=None):
     """
     Returns the portion of a URL corresponding to a part specified
@@ -1998,6 +2021,7 @@ def _string_getitem(self, key):
     re_search=re_search,
     re_extract=regex_extract,
     re_replace=regex_replace,
+    to_timestamp=to_timestamp,
     parse_url=parse_url,
 
     substr=_string_substr,
@@ -3191,26 +3215,3 @@ def _table_drop(self, fields):
 
 
 _add_methods(ir.TableExpr, _table_methods)
-
-
-# geometric operation
-
-def distance(from_lon, from_lat, to_lon, to_lat):
-    """
-    Distance between origin longitude and latitude and
-    destination longitude and latitude
-
-    Parameters
-    ----------
-    from_lon : numeric column expr or float
-    from_lat : numeric column expr or float
-    to_lon : numeric column expr or float
-    to_lat : numeric column expr or float
-
-    Returns
-    -------
-    expr :
-        if scalar input, scalar type, same as input
-        if array input, list of scalar type
-    """
-    return ops.Distance(from_lon, from_lat, to_lon, to_lat).to_expr()

ibis/expr/operations.py

@@ -524,14 +524,14 @@ class Log10(Logarithm):
 
 class Degrees(UnaryOp):
     """Converts radians to degrees"""
-    arg = Arg(rlz.floating)
-    output_type = rlz.shape_like('arg', dt.float)
+    arg = Arg(rlz.numeric)
+    output_type = rlz.shape_like('arg', dt.float64)
 
 
 class Radians(UnaryOp):
     """Converts radians to degrees"""
-    arg = Arg(rlz.floating)
-    output_type = rlz.shape_like('arg', dt.float)
+    arg = Arg(rlz.numeric)
+    output_type = rlz.shape_like('arg', dt.float64)
 
 
 # TRIGONOMETRIC OPERATIONS
@@ -865,11 +865,7 @@ class Correlation(Reduction):
     where = Arg(rlz.boolean, default=None)
 
     def output_type(self):
-        if isinstance(self.left, ir.DecimalValue):
-            dtype = self.left.type().largest
-        else:
-            dtype = dt.float64
-        return dtype.scalar_type()
+        return dt.float64.scalar_type()
 
 
 class Covariance(Reduction):
@@ -880,11 +876,7 @@ class Covariance(Reduction):
     where = Arg(rlz.boolean, default=None)
 
     def output_type(self):
-        if isinstance(self.left, ir.DecimalValue):
-            dtype = self.left.type().largest
-        else:
-            dtype = dt.float64
-        return dtype.scalar_type()
+        return dt.float64.scalar_type()
 
 
 class Max(Reduction):
@@ -2410,6 +2402,13 @@ class Strftime(ValueOp):
     output_type = rlz.shape_like('arg', dt.string)
 
 
+class StringToTimestamp(ValueOp):
+    arg = Arg(rlz.string)
+    format_str = Arg(rlz.string)
+    timezone = Arg(rlz.string, default=None)
+    output_type = rlz.shape_like('arg', dt.Timestamp(timezone='UTC'))
+
+
 class ExtractTemporalField(TemporalUnaryOp):
     output_type = rlz.shape_like('arg', dt.int32)
 
@@ -2800,17 +2799,3 @@ def output_type(self):
 
     def root_tables(self):
         return distinct_roots(*self.values)
-
-
-# GEOMETRIC OPERATIONS
-
-class Distance(ValueOp):
-    """
-    Calculates distance in meters between two WGS-84 positions.
-
-    """
-    from_lon = Arg(rlz.numeric)
-    from_lat = Arg(rlz.numeric)
-    to_lon = Arg(rlz.numeric)
-    to_lat = Arg(rlz.numeric)
-    output_type = rlz.shape_like('args', dt.float64)

ibis/mapd/compiler.py

@@ -201,8 +201,6 @@ class MapDDialect(compiles.Dialect):
 
 mapd_reg = mapd_ops._operation_registry
 
-compiles(ops.Distance, mapd_ops.distance)
-
 
 @rewrites(ops.All)
 def mapd_rewrite_all(expr):

ibis/mapd/operations.py

@@ -222,13 +222,11 @@ def _corr(translator, expr):
 
     x, y, how, where = args
 
-    f_type = '' if not _is_floating(x, y) else '_FLOAT'
-
     # compile the argument
     compiled_x = translator.translate(x)
     compiled_y = translator.translate(y)
 
-    return 'CORR{}({}, {})'.format(f_type, compiled_x, compiled_y)
+    return 'CORR({}, {})'.format(compiled_x, compiled_y)
 
 
 def _cov(translator, expr):
@@ -237,14 +235,12 @@ def _cov(translator, expr):
 
     x, y, how, where = args
 
-    f_type = '' if not _is_floating(x, y) else '_FLOAT'
-
     # compile the argument
     compiled_x = translator.translate(x)
     compiled_y = translator.translate(y)
 
-    return 'COVAR_{}{}({}, {})'.format(
-        how[:4].upper(), f_type, compiled_x, compiled_y
+    return 'COVAR_{}({}, {})'.format(
+        how[:4].upper(), compiled_x, compiled_y
     )
 
 
@@ -483,12 +479,6 @@ def _table_column(translator, expr):
 count = _reduction('count')
 
 
-def distance(translator, expr):
-    op = expr.op()
-    values = map(translator.translate, op.args)
-    return 'DISTANCE_IN_METERS({0})'.format(', '.join(values))
-
-
 # MATH
 
 class NumericTruncate(ops.NumericBinaryOp):

ibis/mapd/tests/test_client.py

@@ -2,7 +2,6 @@
 
 import ibis
 import ibis.expr.types as ir
-import os
 import pandas as pd
 import pytest
 
@@ -11,12 +10,6 @@
 pytest.importorskip('pymapd')
 
 
-MAPD_TEST_DB = os.environ.get('IBIS_TEST_MAPD_DATABASE', 'mapd')
-IBIS_MAPD_HOST = os.environ.get('IBIS_TEST_MAPD_HOST', 'localhost')
-IBIS_MAPD_USER = os.environ.get('IBIS_TEST_MAPD_USER', 'mapd')
-IBIS_MAPD_PASS = os.environ.get('IBIS_TEST_MAPD_PASSWORD', 'HyperInteractive')
-
-
 def test_table(alltypes):
     assert isinstance(alltypes, ir.TableExpr)