Skip to content

improving delete speed #2165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 7, 2017
Merged

improving delete speed #2165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ae8db1
improving delete speed
antgonza Jul 6, 2017
97cc95c
more sleep!
antgonza Jul 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions qiita_core/testing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ def wait_for_prep_information_job(prep_id, raise_if_none=True):
else:
redis_info = loads(r_client.get(job_id))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(job_id))
sleep(0.05)
sleep(0.1)


def wait_for_processing_job(job_id):
Expand Down
5 changes: 1 addition & 4 deletions qiita_db/metadata_template/base_metadata_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,10 +557,7 @@ def _common_creation_steps(cls, md_template, obj_id):
table_name = cls._table_name(obj_id)
column_datatype = ["%s varchar" % col for col in headers]
sql = """CREATE TABLE qiita.{0} (
sample_id varchar NOT NULL, {1},
CONSTRAINT fk_{0} FOREIGN KEY (sample_id)
REFERENCES qiita.study_sample (sample_id)
ON UPDATE CASCADE
sample_id varchar NOT NULL, {1}
)""".format(table_name, ', '.join(column_datatype))
qdb.sql_connection.TRN.add(sql)

Expand Down
21 changes: 21 additions & 0 deletions qiita_db/support_files/patches/55.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
-- Jul 6, 2017
-- DELETE all sample/prep CONSTRAINTs

-- Step 1, remove all FK constraints from study_sample
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this comment is out of date.

CREATE OR REPLACE FUNCTION qiita.delete_study_sample_constraints(cname text, tname text) RETURNS void AS $$
BEGIN
EXECUTE 'ALTER TABLE ' || tname || ' DROP CONSTRAINT ' || cname;
END;
$$ LANGUAGE plpgsql;

WITH query AS (
SELECT constraint_name AS cname, 'qiita.' || table_name AS tname
FROM information_schema.table_constraints
WHERE constraint_type ='FOREIGN KEY' AND (
(constraint_name LIKE 'fk_sample_%' AND table_name LIKE 'sample_%') OR
(constraint_name LIKE 'fk_prep_%' AND table_name LIKE 'prep_%')) AND
table_name NOT IN ('prep_template', 'prep_template_sample',
'prep_template_filepath', 'prep_template_processing_job'))
SELECT qiita.delete_study_sample_constraints(cname, tname) FROM query;

DROP FUNCTION qiita.delete_study_sample_constraints(cname text, tname text);
3 changes: 2 additions & 1 deletion qiita_pet/handlers/api_proxy/tests/test_prep_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,9 @@ def _wait_for_parallel_job(self, key):
obs = r_client.get(key)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)

def test_prep_template_graph_get_req(self):
obs = prep_template_graph_get_req(1, 'test@foo.bar')
Expand Down
12 changes: 8 additions & 4 deletions qiita_pet/handlers/api_proxy/tests/test_sample_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,8 +400,9 @@ def test_sample_template_post_req(self):
# the test database
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)

def test_sample_template_post_req_no_access(self):
obs = sample_template_post_req(1, 'demo@microbio.me', '16S',
Expand All @@ -426,8 +427,9 @@ def test_sample_template_put_req(self):
# the test database
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)

def test_sample_template_put_req_no_access(self):
obs = sample_template_put_req(1, 'demo@microbio.me', 'filepath')
Expand Down Expand Up @@ -456,8 +458,9 @@ def test_sample_template_delete_req(self):
# the test database
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)

def test_sample_template_delete_req_no_access(self):
obs = sample_template_delete_req(1, 'demo@microbio.me')
Expand Down Expand Up @@ -556,8 +559,9 @@ def test_sample_template_patch_request(self):
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.05)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)

ST = qdb.metadata_template.sample_template.SampleTemplate
self.assertNotIn("season_environment", ST(1).categories())
Expand Down
3 changes: 2 additions & 1 deletion qiita_pet/handlers/study_handlers/tests/test_sample_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,9 @@ def test_delete_sample_template(self):
self.assertIsNotNone(obs)
redis_info = loads(r_client.get(loads(obs)['job_id']))
while redis_info['status_msg'] == 'Running':
sleep(0.5)
sleep(0.1)
redis_info = loads(r_client.get(loads(obs)['job_id']))
sleep(0.1)


class TestSampleAJAXReadOnly(TestHandlerBase):
Expand Down