fix Overflow in download analysis metadata

Author: antgonza

qiita_db/handlers/analysis.py

@@ -6,7 +6,10 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
+from tornado import gen
 from tornado.web import HTTPError
+from iostream import StreamClosedError
+from json import dumps
 
 import qiita_db as qdb
 from .oauth2 import OauthBaseHandler, authenticate_oauth
@@ -44,7 +47,7 @@ def _get_analysis(a_id):
 
 class APIAnalysisMetadataHandler(OauthBaseHandler):
     @authenticate_oauth
-    def get(self, analysis_id):
+    async def get(self, analysis_id):
         """Retrieves the analysis metadata
 
         Parameters
@@ -56,15 +59,34 @@ def get(self, analysis_id):
         -------
         dict
             The contents of the analysis keyed by sample id
+
+        Notes
+        -----
+        This response needed to be broken in chunks because we were hitting
+        the max size of a respose: 2G; based on: https://bit.ly/3CPvyjd
         """
+        chunk_len = 1024 * 1024 * 1  # 1 MiB
+
         with qdb.sql_connection.TRN:
             a = _get_analysis(analysis_id)
             mf_fp = qdb.util.get_filepath_information(
                 a.mapping_file)['fullpath']
-            response = None
             if mf_fp is not None:
                 df = qdb.metadata_template.util.load_template_to_dataframe(
                     mf_fp, index='#SampleID')
-                response = df.to_dict(orient='index')
+                response = dumps(df.to_dict(orient='index'))
 
-        self.write(response)
+                crange = range(chunk_len, len(response)+chunk_len, chunk_len)
+                for i, (win) in enumerate(crange):
+                    chunk = response[i*chunk_len:win]
+                    try:
+                        self.write(chunk)
+                        await self.flush()
+                    except StreamClosedError:
+                        break
+                    finally:
+                        del chunk
+                        # pause the coroutine so other handlers can run
+                        await gen.sleep(0.000000001)  # 1 nanosecond
+            else:
+                self.write(None)