Add system test for insert/fetch of rows w/ nested schema.

Derived from schema / data from @tswast. Toward googleapis#3029.
richkadel · Mar 15, 2017 · e16833f · e16833f
1 parent 0fa8823
commit e16833f
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 10 deletions.
diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py
@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import datetime
 import json
 import operator
 import os
-
+import time
 import unittest
 
 from google.cloud import bigquery
+from google.cloud._helpers import UTC
 from google.cloud.exceptions import Forbidden
 
 from retry import RetryErrors
@@ -40,6 +43,7 @@ def _make_dataset_name(prefix):
 
 def _load_json_schema(filename='bigquery_test_schema.json'):
     from google.cloud.bigquery.table import _parse_schema_resource
+
     json_filename = os.path.join(WHERE, filename)
 
     with open(json_filename, 'r') as schema_file:
@@ -289,9 +293,6 @@ def _fetch_single_page(table):
         return list(page)
 
     def test_insert_data_then_dump_table(self):
-        import datetime
-        from google.cloud._helpers import UTC
-
         NOW_SECONDS = 1448911495.484366
         NOW = datetime.datetime.utcfromtimestamp(
             NOW_SECONDS).replace(tzinfo=UTC)
@@ -493,8 +494,6 @@ def _job_done(instance):
         # above).
 
     def test_sync_query_w_legacy_sql_types(self):
-        import datetime
-        from google.cloud._helpers import UTC
         naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
         stamp = '%s %s' % (naive.date().isoformat(), naive.time().isoformat())
         zoned = naive.replace(tzinfo=UTC)
@@ -533,8 +532,6 @@ def test_sync_query_w_legacy_sql_types(self):
             self.assertEqual(query.rows[0][0], example['expected'])
 
     def test_sync_query_w_standard_sql_types(self):
-        import datetime
-        from google.cloud._helpers import UTC
         from google.cloud.bigquery._helpers import ArrayQueryParameter
         from google.cloud.bigquery._helpers import ScalarQueryParameter
         from google.cloud.bigquery._helpers import StructQueryParameter
@@ -664,7 +661,7 @@ def test_dump_table_w_public_data(self):
         table = dataset.table(TABLE_NAME)
         self._fetch_single_page(table)
 
-    def test_create_table_nested_schema(self):
+    def test_create_table_insert_fetch_nested_schema(self):
         table_name = 'test_table'
         dataset = Config.CLIENT.dataset(
             _make_dataset_name('create_table_nested_schema'))
@@ -673,8 +670,58 @@ def test_create_table_nested_schema(self):
         retry_403(dataset.create)()
         self.to_delete.append(dataset)
 
-        table = dataset.table(table_name, schema=_load_json_schema())
+        schema = _load_json_schema()
+        table = dataset.table(table_name, schema=schema)
         table.create()
         self.to_delete.insert(0, table)
         self.assertTrue(table.exists())
         self.assertEqual(table.name, table_name)
+
+        to_insert = []
+        # Data is in "JSON Lines" format, see http://jsonlines.org/
+        json_filename = os.path.join(WHERE, 'bigquery_test_data.jsonl')
+        with open(json_filename) as rows_file:
+            for line in rows_file:
+                mapping = json.loads(line)
+                to_insert.append(
+                    tuple(mapping[field.name] for field in schema))
+
+        errors = table.insert_data(to_insert)
+        self.assertEqual(len(errors), 0)
+
+        retry = RetryResult(_has_rows, max_tries=8)
+        fetched = retry(self._fetch_single_page)(table)
+        self.assertEqual(len(fetched), len(to_insert))
+
+        for found, expected in zip(sorted(fetched), sorted(to_insert)):
+            self.assertEqual(found[0], expected[0])            # Name
+            self.assertEqual(found[1], int(expected[1]))       # Age
+            self.assertEqual(found[2], expected[2])            # Weight
+            self.assertEqual(found[3], expected[3])            # IsMagic
+
+            self.assertEqual(len(found[4]), len(expected[4]))  # Spells
+            for f_spell, e_spell in zip(found[4], expected[4]):
+                self.assertEqual(f_spell['Name'], e_spell['Name'])
+                parts = time.strptime(
+                    e_spell['LastUsed'], '%Y-%m-%d %H:%M:%S UTC')
+                e_used = datetime.datetime(*parts[0:6], tzinfo=UTC)
+                self.assertEqual(f_spell['LastUsed'], e_used)
+                self.assertEqual(f_spell['DiscoveredBy'],
+                                 e_spell['DiscoveredBy'])
+                self.assertEqual(f_spell['Properties'], e_spell['Properties'])
+
+                e_icon = base64.standard_b64decode(
+                    e_spell['Icon'].encode('ascii'))
+                self.assertEqual(f_spell['Icon'], e_icon)
+
+            parts = time.strptime(expected[5], '%H:%M:%S')
+            e_teatime = datetime.time(*parts[3:6])
+            self.assertEqual(found[5], e_teatime)              # TeaTime
+
+            parts = time.strptime(expected[6], '%Y-%m-%d')
+            e_nextvac = datetime.date(*parts[0:3])
+            self.assertEqual(found[6], e_nextvac)              # NextVacation
+
+            parts = time.strptime(expected[7], '%Y-%m-%dT%H:%M:%S')
+            e_favtime = datetime.datetime(*parts[0:6])
+            self.assertEqual(found[7], e_favtime)              # FavoriteTime
diff --git a/system_tests/bigquery_test_data.json b/system_tests/bigquery_test_data.json
@@ -0,0 +1,66 @@
+[
+   {
+      "Age" : "111",
+      "Spells" : [],
+      "Name" : "Bilbo",
+      "Weight" : 67.2,
+      "TeaTime" : "10:00:00",
+      "NextVacation" : "2017-09-22",
+      "FavoriteTime" : "2031-04-01T05:09:27",
+      "IsMagic" : false
+   },
+   {
+      "Age" : "1000",
+      "Name" : "Gandalf",
+      "Spells" : [
+         {
+            "Name" : "Skydragon",
+            "Properties" : [
+               {
+                  "Power" : 1,
+                  "Name" : "Flying"
+               },
+               {
+                  "Name" : "Creature",
+                  "Power" : 1
+               },
+               {
+                  "Power" : 11,
+                  "Name" : "Explodey"
+               }
+            ],
+            "LastUsed" : "2015-10-31 23:59:56 UTC",
+            "Icon" : "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAgCAYAAAAFQMh/AAAAAXNSR0IArs4c6QAAA9lJREFUSA21lk9OVEEQxvsRDImoiMG9mLjjCG5mEg7gEfQGsIcF7p0EDsBBSJiNO7ZsFRZqosb/QkSj7fer7ur33sw8GDFUUq+7q6vqq6qu7pkQzqG4EeI521e7FePVgM9cGPYwhCi6UO8qFOK+YY+Br66ujsmmxb84Yzwp6zCsxjJfWVkxnMsEMGuWHZ9Wcz11cM48hkq0vLwc1tbW4mAwqDpcdIqnMmgF0JMv2CiGnZ2dcHR0FA4PD8Pe3t5U/tx6bCSlb+JT8XfxT3HsUek0Li0tRdjWl+z6iRF+FNA1hXPDQ/IMNyRg3s8bD/OaZS+VP+9cOLSa64cA34oXZWagDkRzAaJxXaE+ufc4rCN7LrazZ2+8+STtpAL8WYDvpTaHKlkB2iQARMvb2+H27m4YaL7zaDtUw1BZAASi6T8T2UZnPZV2pvnJfCH5p8bewcGB6TrIfz8wBZgHQ83kjpuj6RBYQpuo09Tvmpd7TPe+ktZN8cKwS92KWXGuaqWowlYEwthtMcWOZUNJc8at+zuF/Xkqo69baS7P+AvWjYwJ4jyHXXsEnd74ZO/Pq+uXUuv6WNlso6cvnDsZB1V/unJab3D1/KrJDw9NCM9wHf2FK2ejTKMejnBHfGtfH7LGGCdQDqaqJgfgzWjXK1nYV4jRbPGnxUT7cqUaZfJrVZeOm9QmB21L6xXgbu/ScsYusJFMoU0x2fsamRJOd6kOYDRLUxv94ENZe8+0gM+0dyz+KgU7X8rLHHCIOZyrna4y6ykIu0YCs02TBXmk3PZssmEgaTxTo83xjCIjoE21h0Yah3MrV4+9kR8MaabGze+9NEILGAFE5nMOiiA32KnAr/sb7tED3nzlzC4dB38WMC+EjaqHfqvUKHi2gJPdWQ6AbH8hgyQ7QY6jvjj3QZWvX6pUAtduTX5Dss96Q7NI9RQRJeeKvRFbt0v2gb1Gx/PooJsztn1c1DqpAU3Hde2dB2aEHBhjgOFjMeDvxLafjQ3YZQSgOcHJZX611H45sGLHWvYTz9hiURlpNoBZvxb/Ft9lAQ1DmBfUiR+j1hAPkMBTE9L9+zLva1QvGFHurRBaZ5xLVitoBviiRkD/sIMDztKA5FA0b9/0OclzO2/XAQymJ0TcghZwEo9/AX8gMeAJMOvIsWWt5bwCoiFhVSllrdH0t5Q1JHAFlKJNkvTVdn2GHb9KdmacMT+d/Os05imJUccRX2YuZ93Sxf0Ilc4DPDeAq5SAvFEAY94cQc6BA26dzb4HWAJI4DPmQE5KCVUyvb2FcDZem7JdT2ggKUP3xX6n9XNq1DpzSf4Cy4ZqSlmM8d8AAAAASUVORK5CYII=",
+            "DiscoveredBy" : "Firebreather"
+         }
+      ],
+      "NextVacation" : "2666-06-06",
+      "TeaTime" : "15:00:00",
+      "Weight" : 198.6,
+      "FavoriteTime" : "2001-12-19T23:59:59",
+      "IsMagic" : true
+   },
+   {
+      "Weight" : 128.3,
+      "TeaTime" : "12:00:00",
+      "NextVacation" : "2017-03-14",
+      "IsMagic" : true,
+      "FavoriteTime" : "2000-10-31T23:27:46",
+      "Age" : "17",
+      "Spells" : [
+         {
+            "LastUsed" : "2017-02-14 12:07:23 UTC",
+            "Properties" : [
+               {
+                  "Name" : "Makes you look crazy",
+                  "Power" : 1
+               }
+            ],
+            "Icon" : "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAABxpRE9UAAAAAgAAAAAAAAAgAAAAKAAAACAAAAAgAAABxj2CfowAAAGSSURBVHgB7Jc9TsNAEIX3JDkCPUV6KlpKFHEGlD4nyA04ACUXQKTgCEipUnKGNEbP0otentayicZ24SlWs7tjO/N9u/5J2b2+NUtuZcnwYE8BuQPyGZAPwXwLLPk5kG+BJa9+fgfkh1B+CeancL4F8i2Q/wWm/S/w+XFoTseftn0dvhu0OXfhpM+AGvzcEiYVAFisPqE9zrETJhHAlXfg2lglMK9z0f3RBfB+ZyRUV3x+erzsEIjjOBqc1xtNAIrvguybV3A9lkVHxlEE6GrrPb/ZvAySwlUnfCmlPQ+R8JCExvGtcRQBLFwj4FGkznX1VYDKPG/f2/MjwCksXACgdNUxJjwK9xwl4JihOwTFR0kIF+CABEPRnvsvPFctMoYKqAFSAFaMwB4pp3Y+bodIYL9WmIAaIOHxo7W8wiHvAjTvhUeNwwSgeAeAABbqOewC5hBdwFD4+9+7puzXV9fS6/b1wwT4tsaYAhwOOQdUQch5vgZCeAhAv3ZM31yYAAUgvApQQQ6n5w6FB/RVe1jdJOAPAAD//1eMQwoAAAGQSURBVO1UMU4DQQy8X9AgWopIUINEkS4VlJQo4gvwAV7AD3gEH4iSgidESpWSXyyZExP5lr0c7K5PsXBhec/2+jzjuWtent9CLdtu1mG5+gjz+WNr7IsY7eH+tvO+xfuqk4vz7CH91edFaF5v9nb6dBKm13edvrL+0Lk5lMzJkQDeJSkkgHF6mR8CHwMHCQR/NAQQGD0BAlwK4FCefQiefq+A2Vn29tG7igLAfmwcnJu/nJy3BMQkMN9HEPr8AL3bfBv7Bp+7/SoExMDjZwKEJwmyhnnmQIQEBIlz2x0iKoAvJkAC6TsTIH6MqRrEWUMSZF2zAwqT4Eu/e6pzFAIkmNSZ4OFT+VYBIIF//UqbJwnF/4DU0GwOn8r/JQYCpPGufEfJuZiA37ycQw/5uFeqPq4pfR6FADmkBCXjfWdZj3NfXW58dAJyB9W65wRoMWulryvAyqa05nQFaDFrpa8rwMqmtOZ0BWgxa6WvK8DKprTmdAVoMWulryvAyqa05nQFaDFrpa8rwMqmtOb89wr4AtQ4aPoL6yVpAAAAAElFTkSuQmCC",
+            "Name" : "Talking cats",
+            "DiscoveredBy" : "Salem"
+         }
+      ],
+      "Name" : "Sabrina"
+   }
+]
diff --git a/system_tests/bigquery_test_data.jsonl b/system_tests/bigquery_test_data.jsonl
@@ -0,0 +1,3 @@
+{"Name":"Bilbo","Age":"111","Weight":67.2,"IsMagic":false,"Spells":[],"TeaTime":"10:00:00","NextVacation":"2017-09-22","FavoriteTime":"2031-04-01T05:09:27"}
+{"Name":"Gandalf","Age":"1000","Weight":198.6,"IsMagic":true,"Spells":[{"Name": "Skydragon", "Icon":"iVBORw0KGgoAAAANSUhEUgAAAB4AAAAgCAYAAAAFQMh/AAAAAXNSR0IArs4c6QAAA9lJREFUSA21lk9OVEEQxvsRDImoiMG9mLjjCG5mEg7gEfQGsIcF7p0EDsBBSJiNO7ZsFRZqosb/QkSj7fer7ur33sw8GDFUUq+7q6vqq6qu7pkQzqG4EeI521e7FePVgM9cGPYwhCi6UO8qFOK+YY+Br66ujsmmxb84Yzwp6zCsxjJfWVkxnMsEMGuWHZ9Wcz11cM48hkq0vLwc1tbW4mAwqDpcdIqnMmgF0JMv2CiGnZ2dcHR0FA4PD8Pe3t5U/tx6bCSlb+JT8XfxT3HsUek0Li0tRdjWl+z6iRF+FNA1hXPDQ/IMNyRg3s8bD/OaZS+VP+9cOLSa64cA34oXZWagDkRzAaJxXaE+ufc4rCN7LrazZ2+8+STtpAL8WYDvpTaHKlkB2iQARMvb2+H27m4YaL7zaDtUw1BZAASi6T8T2UZnPZV2pvnJfCH5p8bewcGB6TrIfz8wBZgHQ83kjpuj6RBYQpuo09Tvmpd7TPe+ktZN8cKwS92KWXGuaqWowlYEwthtMcWOZUNJc8at+zuF/Xkqo69baS7P+AvWjYwJ4jyHXXsEnd74ZO/Pq+uXUuv6WNlso6cvnDsZB1V/unJab3D1/KrJDw9NCM9wHf2FK2ejTKMejnBHfGtfH7LGGCdQDqaqJgfgzWjXK1nYV4jRbPGnxUT7cqUaZfJrVZeOm9QmB21L6xXgbu/ScsYusJFMoU0x2fsamRJOd6kOYDRLUxv94ENZe8+0gM+0dyz+KgU7X8rLHHCIOZyrna4y6ykIu0YCs02TBXmk3PZssmEgaTxTo83xjCIjoE21h0Yah3MrV4+9kR8MaabGze+9NEILGAFE5nMOiiA32KnAr/sb7tED3nzlzC4dB38WMC+EjaqHfqvUKHi2gJPdWQ6AbH8hgyQ7QY6jvjj3QZWvX6pUAtduTX5Dss96Q7NI9RQRJeeKvRFbt0v2gb1Gx/PooJsztn1c1DqpAU3Hde2dB2aEHBhjgOFjMeDvxLafjQ3YZQSgOcHJZX611H45sGLHWvYTz9hiURlpNoBZvxb/Ft9lAQ1DmBfUiR+j1hAPkMBTE9L9+zLva1QvGFHurRBaZ5xLVitoBviiRkD/sIMDztKA5FA0b9/0OclzO2/XAQymJ0TcghZwEo9/AX8gMeAJMOvIsWWt5bwCoiFhVSllrdH0t5Q1JHAFlKJNkvTVdn2GHb9KdmacMT+d/Os05imJUccRX2YuZ93Sxf0Ilc4DPDeAq5SAvFEAY94cQc6BA26dzb4HWAJI4DPmQE5KCVUyvb2FcDZem7JdT2ggKUP3xX6n9XNq1DpzSf4Cy4ZqSlmM8d8AAAAASUVORK5CYII=","DiscoveredBy":"Firebreather","Properties":[{"Name":"Flying","Power":1},{"Name":"Creature","Power":1},{"Name":"Explodey","Power":11}],"LastUsed":"2015-10-31 23:59:56 UTC"}],"TeaTime":"15:00:00","NextVacation":"2666-06-06","FavoriteTime":"2001-12-19T23:59:59"}
+{"Name":"Sabrina","Age":"17","Weight":128.3,"IsMagic":true,"Spells":[{"Name": "Talking cats", "Icon":"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAABxpRE9UAAAAAgAAAAAAAAAgAAAAKAAAACAAAAAgAAABxj2CfowAAAGSSURBVHgB7Jc9TsNAEIX3JDkCPUV6KlpKFHEGlD4nyA04ACUXQKTgCEipUnKGNEbP0otentayicZ24SlWs7tjO/N9u/5J2b2+NUtuZcnwYE8BuQPyGZAPwXwLLPk5kG+BJa9+fgfkh1B+CeancL4F8i2Q/wWm/S/w+XFoTseftn0dvhu0OXfhpM+AGvzcEiYVAFisPqE9zrETJhHAlXfg2lglMK9z0f3RBfB+ZyRUV3x+erzsEIjjOBqc1xtNAIrvguybV3A9lkVHxlEE6GrrPb/ZvAySwlUnfCmlPQ+R8JCExvGtcRQBLFwj4FGkznX1VYDKPG/f2/MjwCksXACgdNUxJjwK9xwl4JihOwTFR0kIF+CABEPRnvsvPFctMoYKqAFSAFaMwB4pp3Y+bodIYL9WmIAaIOHxo7W8wiHvAjTvhUeNwwSgeAeAABbqOewC5hBdwFD4+9+7puzXV9fS6/b1wwT4tsaYAhwOOQdUQch5vgZCeAhAv3ZM31yYAAUgvApQQQ6n5w6FB/RVe1jdJOAPAAD//1eMQwoAAAGQSURBVO1UMU4DQQy8X9AgWopIUINEkS4VlJQo4gvwAV7AD3gEH4iSgidESpWSXyyZExP5lr0c7K5PsXBhec/2+jzjuWtent9CLdtu1mG5+gjz+WNr7IsY7eH+tvO+xfuqk4vz7CH91edFaF5v9nb6dBKm13edvrL+0Lk5lMzJkQDeJSkkgHF6mR8CHwMHCQR/NAQQGD0BAlwK4FCefQiefq+A2Vn29tG7igLAfmwcnJu/nJy3BMQkMN9HEPr8AL3bfBv7Bp+7/SoExMDjZwKEJwmyhnnmQIQEBIlz2x0iKoAvJkAC6TsTIH6MqRrEWUMSZF2zAwqT4Eu/e6pzFAIkmNSZ4OFT+VYBIIF//UqbJwnF/4DU0GwOn8r/JQYCpPGufEfJuZiA37ycQw/5uFeqPq4pfR6FADmkBCXjfWdZj3NfXW58dAJyB9W65wRoMWulryvAyqa05nQFaDFrpa8rwMqmtOZ0BWgxa6WvK8DKprTmdAVoMWulryvAyqa05nQFaDFrpa8rwMqmtOb89wr4AtQ4aPoL6yVpAAAAAElFTkSuQmCC","DiscoveredBy":"Salem","Properties":[{"Name":"Makes you look crazy","Power":1}],"LastUsed":"2017-02-14 12:07:23 UTC"}],"TeaTime":"12:00:00","NextVacation":"2017-03-14","FavoriteTime":"2000-10-31T23:27:46"}