Merge pull request #21 from xmnlab/master

Code refactored; add tests, ci and docs;

ci/datamgr.py

@@ -7,6 +7,7 @@
 
 import pandas as pd
 import sqlalchemy as sa
+import pymapd
 
 from toolz import dissoc
 from plumbum import local
@@ -188,6 +189,44 @@ def sqlite(database, schema, tables, data_directory, **params):
     insert_tables(engine, tables, data_directory)
 
 
+@cli.command()
+@click.option('-h', '--host', default='localhost')
+@click.option('-P', '--port', default=9091, type=int)
+@click.option('-u', '--user', default='mapd')
+@click.option('-p', '--password', default='HyperInteractive')
+@click.option('-D', '--database', default='mapd')
+@click.option('-S', '--schema', type=click.File('rt'),
+              default=str(SCRIPT_DIR / 'schema' / 'mapd.sql'))
+@click.option('-t', '--tables', multiple=True, default=TEST_TABLES)
+@click.option('-d', '--data-directory', default=DATA_DIR)
+def mapd(schema, tables, data_directory, **params):
+    data_directory = Path(data_directory)
+    click.echo('Initializing MapD...')
+
+    # connection
+    conn = pymapd.connect(
+        host=params['host'], user=params['user'],
+        password=params['password'],
+        port=params['port'], dbname=params['database']
+    )
+
+    # create database
+    for stmt in schema.read().split(';'):
+        stmt = stmt.strip()
+        if len(stmt):
+            conn.execute(stmt)
+
+    # import data
+    query = 'COPY {} FROM \'{}\' WITH(delimiter=\',\', header=\'true\')'
+
+    click.echo('Loading data ...')
+    for table in tables:
+        src = data_directory / '{}.csv'.format(table)
+        click.echo(src)
+        conn.execute(query.format(table, src))
+    conn.close()
+
+
 @cli.command()
 @click.option('-h', '--host', default='localhost')
 @click.option('-P', '--port', default=3306, type=int)

ci/docker-compose.yml

@@ -8,6 +8,16 @@ services:
     environment:
       POSTGRES_PASSWORD: postgres
 
+  mapd:
+    image: mapd/mapd-ce-cpu
+    ports:
+      - 9090-9092:9090-9092
+    environment:
+      - MAPD_HOST=mapd
+      - MAPD_PORT=9091
+      - MAPD_DATABASE=mapd
+      - MAPD_USER=mapd
+
   mysql:
     image: mariadb:10.2
     ports:
@@ -52,7 +62,8 @@ services:
   waiter:
     image: jwilder/dockerize
     command: |
-      dockerize -wait tcp://mysql:3306
+      dockerize -wait tcp://mapd:9091
+                -wait tcp://mysql:3306
                 -wait tcp://postgres:5432
                 -wait tcp://impala:21050
                 -wait tcp://impala:50070
@@ -84,6 +95,11 @@ services:
       - IBIS_TEST_CLICKHOUSE_HOST=clickhouse
       - IBIS_TEST_CLICKHOUSE_PORT=9000
       - IBIS_TEST_CLICKHOUSE_DATABASE=ibis_testing
+      - IBIS_TEST_MAPD_HOST=mapd
+      - IBIS_TEST_MAPD_PORT=9091
+      - IBIS_TEST_MAPD_DATABASE=mapd
+      - IBIS_TEST_MAPD_USER=mapd
+      - IBIS_TEST_MAPD_PASSWORD=HyperInteractive
       - GOOGLE_BIGQUERY_PROJECT_ID=ibis-gbq
       - GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-service-key.json
     volumes:

ci/requirements-dev-2.7.yml

@@ -10,7 +10,7 @@ dependencies:
   - flake8
   - funcsigs
   - functools32
-  - google-cloud-bigquery<0.28
+  - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.14.0
   - jinja2
@@ -23,6 +23,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/requirements-dev-3.5.yml

@@ -7,7 +7,7 @@ dependencies:
   - clickhouse-sqlalchemy
   - cmake
   - flake8
-  - google-cloud-bigquery<0.28
+  - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.14.0
   - jinja2
@@ -18,6 +18,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytest
   - python=3.5

ci/requirements-dev-3.6.yml

@@ -7,7 +7,7 @@ dependencies:
   - clickhouse-sqlalchemy
   - cmake
   - flake8
-  - google-cloud-bigquery<0.28
+  - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.14.0
   - jinja2
@@ -18,6 +18,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/requirements-docs-3.6.yml

@@ -7,7 +7,7 @@ dependencies:
   - clickhouse-sqlalchemy
   - cmake
   - flake8
-  - google-cloud-bigquery<0.28
+  - google-cloud-bigquery>=1.0.0
   - graphviz
   - impyla>=0.14.0
   - ipython
@@ -22,6 +22,7 @@ dependencies:
   - plumbum
   - psycopg2
   - pyarrow>=0.6.0
+  - pymapd
   - pymysql
   - pytables
   - pytest

ci/schema/mapd.sql

@@ -0,0 +1,72 @@
+DROP TABLE IF EXISTS diamonds;
+
+CREATE TABLE diamonds (
+    carat FLOAT,
+    cut TEXT,
+    color TEXT,
+    clarity TEXT,
+    depth FLOAT,
+    table_ FLOAT,
+    price BIGINT,
+    x FLOAT,
+    y FLOAT,
+    z FLOAT
+);
+
+DROP TABLE IF EXISTS batting;
+
+CREATE TABLE batting (
+    playerID VARCHAR(255),
+    yearID BIGINT,
+    stint BIGINT,
+    teamID VARCHAR(7),
+    lgID VARCHAR(7),
+    G BIGINT,
+    AB BIGINT,
+    R BIGINT,
+    H BIGINT,
+    X2B BIGINT,
+    X3B BIGINT,
+    HR BIGINT,
+    RBI BIGINT,
+    SB BIGINT,
+    CS BIGINT,
+    BB BIGINT,
+    SO BIGINT,
+    IBB BIGINT,
+    HBP BIGINT,
+    SH BIGINT,
+    SF BIGINT,
+    GIDP BIGINT
+);
+
+DROP TABLE IF EXISTS awards_players;
+
+CREATE TABLE awards_players (
+    playerID VARCHAR(255),
+    awardID VARCHAR(255),
+    yearID BIGINT,
+    lgID VARCHAR(7),
+    tie VARCHAR(7),
+    notes VARCHAR(255)
+);
+
+DROP TABLE IF EXISTS functional_alltypes;
+
+CREATE TABLE functional_alltypes (
+    index BIGINT,
+    Unnamed_ BIGINT,
+    id INTEGER,
+    bool_col BOOLEAN,
+    tinyint_col SMALLINT,
+    smallint_col SMALLINT,
+    int_col INTEGER,
+    bigint_col BIGINT,
+    float_col FLOAT,
+    double_col DOUBLE,
+    date_string_col TEXT,
+    string_col TEXT,
+    timestamp_col TIMESTAMP,
+    year_ INTEGER,
+    month_ INTEGER
+);

docs/source/developer.rst

@@ -139,6 +139,25 @@ instructions above, then SQLite will be available in the conda environment.
 
       ci/datamgr.py sqlite
 
+MapD
+^^^^
+
+MapD can be used from either a docker image or from your machine directly.
+
+#. **Start the MapD Server docker image in another terminal**:
+
+   .. code:: sh
+
+      # Keeping this running as long as you want to test ibis
+      docker run -d -v $HOME/mapd-docker-storage:/mapd-storage -p 9090-9092:9090-9092 mapd/mapd-ce-cpu
+
+
+Here's how to load test data into MapD:
+
+   .. code:: sh
+
+      ci/datamgr.py mapd
+
 
 Running Tests
 -------------

docs/source/getting-started.rst

@@ -149,6 +149,27 @@ with:
 
    >>> con = ibis.bigquery.connect(project_id='ibis-gbq', dataset_id='testing')
 
+By default ibis assumes that the BigQuery project that's billed for queries is
+also the project where the data lives.
+
+However, it's very easy to query data that does **not** live in the billing
+project.
+
+.. note::
+
+   When you run queries against data from other projects **the billing project
+   will still be billed for any and all queries**.
+
+If you want to query data that lives in a different project than the billing
+project you can use the :meth:`~ibis.bigquery.client.BigQueryClient.database`
+method of :class:`~ibis.bigquery.client.BigQueryClient` objects:
+
+.. code-block:: python
+
+   >>> db = con.database('other-data-project.other-dataset')
+   >>> t = db.my_awesome_table
+   >>> t.sweet_column.sum().execute()  # runs against the billing project
+
 Learning resources
 ------------------
 

docs/source/release.rst

@@ -25,6 +25,8 @@ New Features
 * Add support for ``UNION`` in the BigQuery backend (:issue:`1408`, :issue:`1409`)
 * Support for writing UDFs in BigQuery (:issue:`1377`). See :ref:`the BigQuery
   UDF docs <udf.bigquery>` for more details.
+* Support for cross-project expressions in the BigQuery backend.
+  (:issue:`1428`)
 
 Bug Fixes
 ~~~~~~~~~