diff --git a/datalab/bigquery/_table.py b/datalab/bigquery/_table.py index 6467eb305..6b6203521 100644 --- a/datalab/bigquery/_table.py +++ b/datalab/bigquery/_table.py @@ -401,7 +401,7 @@ def insert_data(self, data, include_index=False, index_name=None): self._info = self._api.tables_get(self._name_parts) if 'streamingBuffer' not in self._info or \ 'estimatedRows' not in self._info['streamingBuffer'] or \ - self._info['streamingBuffer']['estimatedRows'] > 0: + int(self._info['streamingBuffer']['estimatedRows']) > 0: break time.sleep(2) diff --git a/datalab/context/_utils.py b/datalab/context/_utils.py index 577289831..c5dec5832 100644 --- a/datalab/context/_utils.py +++ b/datalab/context/_utils.py @@ -116,7 +116,7 @@ def get_project_id(): proc = subprocess.Popen(['gcloud', 'config', 'list', '--format', 'value(core.project)'], stdout=subprocess.PIPE) stdout, _ = proc.communicate() - value = stdout.strip() + value = stdout.decode().strip() if proc.poll() == 0 and value: return value except: diff --git a/datalab/utils/_http.py b/datalab/utils/_http.py index a161da14c..a07f51db7 100644 --- a/datalab/utils/_http.py +++ b/datalab/utils/_http.py @@ -53,6 +53,16 @@ class Http(object): """A helper class for making HTTP requests. """ + # Reuse one Http object across requests to take advantage of Keep-Alive, e.g. + # for BigQuery queries that requires at least ~5 sequential http requests. + # + # TODO(nikhilko): + # SSL cert validation seemingly fails, and workarounds are not amenable + # to implementing in library code. So configure the Http object to skip + # doing so, in the interim. + http = httplib2.Http() + http.disable_ssl_certificate_validation = True + def __init__(self): pass @@ -109,15 +119,8 @@ def request(url, args=None, data=None, headers=None, method=None, if method is None: method = 'GET' - # Create an Http object to issue requests. Associate the credentials - # with it if specified to perform authorization. - # - # TODO(nikhilko): - # SSL cert validation seemingly fails, and workarounds are not amenable - # to implementing in library code. So configure the Http object to skip - # doing so, in the interim. - http = httplib2.Http() - http.disable_ssl_certificate_validation = True + # Authorize with credentials if given. + http = Http.http if credentials is not None: http = credentials.authorize(http) if stats is not None: diff --git a/tests/_util/http_tests.py b/tests/_util/http_tests.py index 371a1e8a4..b620ffb9e 100644 --- a/tests/_util/http_tests.py +++ b/tests/_util/http_tests.py @@ -23,7 +23,7 @@ class TestCases(unittest.TestCase): @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_get_request_is_invoked(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -32,7 +32,7 @@ def test_get_request_is_invoked(self, mock_request, mock_response): self.assertEqual(mock_request.call_args[1]['method'], 'GET') @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_post_request_is_invoked(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -40,7 +40,7 @@ def test_post_request_is_invoked(self, mock_request, mock_response): self.assertEqual(mock_request.call_args[1]['method'], 'POST') @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_explicit_post_request_is_invoked(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -48,7 +48,7 @@ def test_explicit_post_request_is_invoked(self, mock_request, mock_response): self.assertEqual(mock_request.call_args[1]['method'], 'POST') @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_query_string_format(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -59,7 +59,7 @@ def test_query_string_format(self, mock_request, mock_response): self.assertTrue('b=a+b+c' in parts[1:]) @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_formats_json_request(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -71,7 +71,7 @@ def test_formats_json_request(self, mock_request, mock_response): 'application/json') @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_supports_custom_content(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{}') @@ -83,7 +83,7 @@ def test_supports_custom_content(self, mock_request, mock_response): self.assertEqual(mock_request.call_args[1]['headers']['Content-Type'], 'text/plain') @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_parses_json_response(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, '{"abc":123}') @@ -91,7 +91,7 @@ def test_parses_json_response(self, mock_request, mock_response): self.assertEqual(data['abc'], 123) @mock.patch('httplib2.Response') - @mock.patch('httplib2.Http.request') + @mock.patch('datalab.utils._http.Http.http.request') def test_raises_http_error(self, mock_request, mock_response): TestCases._setup_mocks(mock_request, mock_response, 'Not Found', 404)