OverflowError when using upload_analytic_image method

[mattphotonman]

from voxel51.api import API
api = API()
api.upload_analytic("/path/to/analytic.json")
api.upload_analytic_image("analytic id", "/path/to/image.tar.gz", "gpu")

---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
<ipython-input-9-2c6500bb68c5> in <module>
----> 1 api.upload_analytic_image("analytic id", "/path/to/image.tar.gz", "gpu")

~/api-py/voxel51/api.py in upload_analytic_image(self, analytic_id, image_tar_path, image_type)
    191             files = {"file": (filename, df, mime_type)}
    192             res = self._requests.post(
--> 193                 endpoint, headers=self._header, files=files, params=params)
    194         _validate_response(res)
    195 

~/myenv/lib/python3.5/site-packages/requests/api.py in post(url, data, json, **kwargs)
    114     """
    115 
--> 116     return request('post', url, data=data, json=json, **kwargs)
    117 
    118 

~/myenv/lib/python3.5/site-packages/requests/api.py in request(method, url, **kwargs)
     58     # cases, and look like a memory leak in others.
     59     with sessions.Session() as session:
---> 60         return session.request(method=method, url=url, **kwargs)
     61 
     62 

~/myenv/lib/python3.5/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    522         }
    523         send_kwargs.update(settings)
--> 524         resp = self.send(prep, **send_kwargs)
    525 
    526         return resp

~/myenv/lib/python3.5/site-packages/requests/sessions.py in send(self, request, **kwargs)
    635 
    636         # Send the request
--> 637         r = adapter.send(request, **kwargs)
    638 
    639         # Total elapsed time of the request (approximately)

~/myenv/lib/python3.5/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    447                     decode_content=False,
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )
    451 

~/myenv/lib/python3.5/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    598                                                   timeout=timeout_obj,
    599                                                   body=body, headers=headers,
--> 600                                                   chunked=chunked)
    601 
    602             # If we're going to release the connection in ``finally:``, then

~/myenv/lib/python3.5/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    352             conn.request_chunked(method, url, **httplib_request_kw)
    353         else:
--> 354             conn.request(method, url, **httplib_request_kw)
    355 
    356         # Reset the timeout for the recv() on the socket

/usr/lib/python3.5/http/client.py in request(self, method, url, body, headers)
   1104     def request(self, method, url, body=None, headers={}):
   1105         """Send a complete request to the server."""
-> 1106         self._send_request(method, url, body, headers)
   1107 
   1108     def _set_content_length(self, body, method):

/usr/lib/python3.5/http/client.py in _send_request(self, method, url, body, headers)
   1149             # default charset of iso-8859-1.
   1150             body = _encode(body, 'body')
-> 1151         self.endheaders(body)
   1152 
   1153     def getresponse(self):

/usr/lib/python3.5/http/client.py in endheaders(self, message_body)
   1100         else:
   1101             raise CannotSendHeader()
-> 1102         self._send_output(message_body)
   1103 
   1104     def request(self, method, url, body=None, headers={}):

/usr/lib/python3.5/http/client.py in _send_output(self, message_body)
    934         self.send(msg)
    935         if message_body is not None:
--> 936             self.send(message_body)
    937 
    938     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):

/usr/lib/python3.5/http/client.py in send(self, data)
    906             return
    907         try:
--> 908             self.sock.sendall(data)
    909         except TypeError:
    910             if isinstance(data, collections.Iterable):

/usr/lib/python3.5/ssl.py in sendall(self, data, flags)
    889             count = 0
    890             while (count < amount):
--> 891                 v = self.send(data[count:])
    892                 count += v
    893             return amount

/usr/lib/python3.5/ssl.py in send(self, data, flags)
    859                     "non-zero flags not allowed in calls to send() on %s" %
    860                     self.__class__)
--> 861             return self._sslobj.write(data)
    862         else:
    863             return socket.send(self, data, flags)

/usr/lib/python3.5/ssl.py in write(self, data)
    584         The 'data' argument must support the buffer interface.
    585         """
--> 586         return self._sslobj.write(data)
    587 
    588     def getpeercert(self, binary_form=False):

OverflowError: string longer than 2147483647 bytes

[brimoor]

OverflowError: string longer than 2147483647 bytes is exactly 2GB. @hoddr didn't we decide to lift the data size limit for now?

[mattphotonman]

@hoddr I get this error in the staging environment.

[mattphotonman]

Per @hoddr , need to refactor to support streaming uploads, or another alternative mentioned in this thread: https://stackoverflow.com/questions/53095132/how-to-upload-chunks-of-a-string-longer-than-2147483647-bytes

[brimoor]

No, this library is already using streaming uploads as recommended in the link above. This must be an API side (nginx) error.

[hoddr]

If it was an nginx issue, we wouldn't be able to upload via console either. I will test via curl momentarily, but there is no client max body size restriction in staging.

[mattphotonman]

The link says to try chunking uploads if streaming doesn't work. (Not sure why streaming wouldn't work.)

[mattphotonman]

@hoddr @brimoor , wondering if there was any progress on this. Thanks!

[brimoor]

no progress yet. Confusing because we are using streaming uploads, so no file size limitations are expected. I googled once more just now for possible problems, and this post https://github.com/kennethreitz/requests/issues/1584 seems to suggest the problem might be related to the fact that we are passing params= in addition to data=. Not sure though.

It's not too onerous to upload via the console for now, right?

[mattphotonman]

no progress yet. Confusing because we are using streaming uploads, so no file size limitations are expected. I googled once more just now for possible problems, and this post kennethreitz/requests#1584 seems to suggest the problem might be related to the fact that we are passing params= in addition to data=. Not sure though.

It's not too onerous to upload via the console for now, right?

It can take a long time to download from VM to laptop, and then upload to the platform, depending on internet speed. If we could do it via the API it would be a significant time saver.