diff --git a/qiita_db/handlers/analysis.py b/qiita_db/handlers/analysis.py index a3cf77d2e..1d4dddd5e 100644 --- a/qiita_db/handlers/analysis.py +++ b/qiita_db/handlers/analysis.py @@ -6,7 +6,9 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- +from tornado import gen from tornado.web import HTTPError +from json import dumps import qiita_db as qdb from .oauth2 import OauthBaseHandler, authenticate_oauth @@ -44,7 +46,7 @@ def _get_analysis(a_id): class APIAnalysisMetadataHandler(OauthBaseHandler): @authenticate_oauth - def get(self, analysis_id): + async def get(self, analysis_id): """Retrieves the analysis metadata Parameters @@ -56,15 +58,37 @@ def get(self, analysis_id): ------- dict The contents of the analysis keyed by sample id + + Notes + ----- + This response needed to be broken in chunks because we were hitting + the max size of a respose: 2G; based on: https://bit.ly/3CPvyjd """ + chunk_len = 1024 * 1024 * 1 # 1 MiB + + response = None with qdb.sql_connection.TRN: a = _get_analysis(analysis_id) mf_fp = qdb.util.get_filepath_information( a.mapping_file)['fullpath'] - response = None if mf_fp is not None: df = qdb.metadata_template.util.load_template_to_dataframe( mf_fp, index='#SampleID') - response = df.to_dict(orient='index') + response = dumps(df.to_dict(orient='index')) + + if response is not None: + crange = range(chunk_len, len(response)+chunk_len, chunk_len) + for i, (win) in enumerate(crange): + # sending the chunk and flushing + chunk = response[i*chunk_len:win] + self.write(chunk) + await self.flush() + + # cleaning chuck and pause the coroutine so other handlers + # can run, note that this is required/important based on the + # original implementation in https://bit.ly/3CPvyjd + del chunk + await gen.sleep(0.000000001) # 1 nanosecond - self.write(response) + else: + self.write(None) diff --git a/scripts/qiita-test-install b/scripts/qiita-test-install index b1f3a0c35..7ffa7d9b4 100755 --- a/scripts/qiita-test-install +++ b/scripts/qiita-test-install @@ -232,11 +232,11 @@ class QiitaConfig(TestCase): string_acceptable_version)) def test_redbiom_version(self): - acceptable_version = (0, 3, 5) + acceptable_version = (0, 3, 8) string_acceptable_version = '.'.join(map(str, acceptable_version)) version = tuple(map(int, redbiom_lib_version.split('.'))) - self.assertTrue(acceptable_version == version, + self.assertTrue(acceptable_version >= version, 'Unsupported redbiom version. You have %s but the ' 'minimum required version is %s' % ('.'.join(map(str, version)), diff --git a/setup.py b/setup.py index 0991acb6a..808791617 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ install_requires=['psycopg2', 'click', 'bcrypt', 'pandas', 'biom-format', 'tornado<6.0', 'toredis', 'redis', 'scp', 'pyparsing', 'h5py', 'natsort', 'nose', 'pep8', - 'networkx', 'humanize', 'scikit-bio', 'wtforms', + 'networkx', 'humanize', 'scikit-bio', 'wtforms<3.0.0', 'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx', 'nltk', 'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme', 'paramiko', 'seaborn', 'matplotlib', 'scipy', 'nose',