From 3d8b9c9f5a81b0c1850cbfe44625216908c41977 Mon Sep 17 00:00:00 2001 From: Kevin Tian Date: Wed, 16 Aug 2023 13:23:39 -0400 Subject: [PATCH] Fix broken integration/E2E tests (#1013) * wip fix broken integration tests * fix cancel method * temp enable tests on fork * update _run_program * revert cancel job change * remove integration test check * update pending job tests --- test/ibm_test_case.py | 11 +++----- test/integration/test_basic_server_paths.py | 10 +++++--- test/integration/test_job.py | 28 +++++++++++++-------- test/integration/test_results.py | 10 ++------ test/integration/test_retrieve_job.py | 9 ++++--- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/test/ibm_test_case.py b/test/ibm_test_case.py index 9be4028b8..2878597d4 100644 --- a/test/ibm_test_case.py +++ b/test/ibm_test_case.py @@ -162,10 +162,9 @@ def _run_program( self, service, program_id=None, - iterations=1, inputs=None, interim_results=None, - final_result=None, + circuits=None, callback=None, backend=None, log_level=None, @@ -173,7 +172,6 @@ def _run_program( max_execution_time=None, session_id=None, start_session=False, - sleep_per_iteration=0, ): """Run a program.""" self.log.debug("Running program on %s", service.channel) @@ -181,11 +179,8 @@ def _run_program( inputs if inputs is not None else { - "iterations": iterations, "interim_results": interim_results or {}, - "final_result": final_result or {}, - "sleep_per_iteration": sleep_per_iteration, - "circuits": ReferenceCircuits.bell(), + "circuits": circuits or ReferenceCircuits.bell(), } ) pid = program_id or self.program_ids[service.channel] @@ -206,7 +201,7 @@ def _run_program( if max_execution_time: options.max_execution_time = max_execution_time sampler = Sampler(backend=backend, options=options) - job = sampler.run(ReferenceCircuits.bell(), callback=callback) + job = sampler.run(circuits or ReferenceCircuits.bell(), callback=callback) else: job = service.run( program_id=pid, diff --git a/test/integration/test_basic_server_paths.py b/test/integration/test_basic_server_paths.py index e15d62f85..2c84d4a6b 100644 --- a/test/integration/test_basic_server_paths.py +++ b/test/integration/test_basic_server_paths.py @@ -56,8 +56,12 @@ def test_device_properties_and_defaults(self): def test_device_status(self): """Test device status.""" self._require_2_hgps() - for hgp in self._get_hgps(): with self.subTest(hgp=hgp): - backend = self.service.backends(simulator=False, operational=True, instance=hgp)[0] - self.assertTrue(backend.status()) + # check if hgp contains non simulator backends + backends = self.service.backends(simulator=False, operational=True, instance=hgp) + if backends: + backend = self.service.backends( + simulator=False, operational=True, instance=hgp + )[0] + self.assertTrue(backend.status()) diff --git a/test/integration/test_job.py b/test/integration/test_job.py index 330cb4a3d..4c568e4b4 100644 --- a/test/integration/test_job.py +++ b/test/integration/test_job.py @@ -53,7 +53,7 @@ def test_run_program(self, service): def test_run_program_real_device(self, service): """Test running a program.""" device = get_real_device(service) - job = self._run_program(service, final_result="foo", backend=device) + job = self._run_program(service, backend=device) result = job.result() self.assertEqual(JobStatus.DONE, job.status()) self.assertEqual("foo", result) @@ -139,8 +139,12 @@ def test_invalid_max_execution_time_fails(self, service): def test_cancel_job_queued(self, service): """Test canceling a queued job.""" real_device = get_real_device(service) - _ = self._run_program(service, iterations=10, backend=real_device) - job = self._run_program(service, iterations=2, backend=real_device) + _ = self._run_program( + service, circuits=[ReferenceCircuits.bell()] * 10, backend=real_device + ) + job = self._run_program( + service, circuits=[ReferenceCircuits.bell()] * 2, backend=real_device + ) wait_for_status(job, JobStatus.QUEUED) if not cancel_job_safe(job, self.log): return @@ -152,7 +156,10 @@ def test_cancel_job_queued(self, service): @quantum_only def test_cancel_job_running(self, service): """Test canceling a running job.""" - job = self._run_program(service, iterations=5) + job = self._run_program( + service, + circuits=[ReferenceCircuits.bell()] * 10, + ) if not cancel_job_safe(job, self.log): return time.sleep(10) # Wait a bit for DB to update. @@ -173,7 +180,7 @@ def test_delete_job(self, service): sub_tests = [JobStatus.RUNNING, JobStatus.DONE] for status in sub_tests: with self.subTest(status=status): - job = self._run_program(service, iterations=2) + job = self._run_program(service) wait_for_status(job, status) service.delete_job(job.job_id()) with self.assertRaises(RuntimeJobNotFound): @@ -184,8 +191,8 @@ def test_delete_job(self, service): def test_delete_job_queued(self, service): """Test deleting a queued job.""" real_device = get_real_device(service) - _ = self._run_program(service, iterations=10, backend=real_device) - job = self._run_program(service, iterations=2, backend=real_device) + _ = self._run_program(service, backend=real_device) + job = self._run_program(service, backend=real_device) wait_for_status(job, JobStatus.QUEUED) service.delete_job(job.job_id()) with self.assertRaises(RuntimeJobNotFound): @@ -196,7 +203,7 @@ def test_delete_job_queued(self, service): def test_final_result(self, service): """Test getting final result.""" final_result = get_complex_types() - job = self._run_program(service, final_result=final_result) + job = self._run_program(service) result = job.result(decoder=SerializableClassDecoder) self.assertEqual(final_result, result) @@ -206,7 +213,7 @@ def test_final_result(self, service): @run_integration_test def test_job_status(self, service): """Test job status.""" - job = self._run_program(service, iterations=1) + job = self._run_program(service) time.sleep(random.randint(1, 5)) self.assertTrue(job.status()) @@ -216,7 +223,6 @@ def test_job_inputs(self, service): """Test job inputs.""" interim_results = get_complex_types() inputs = { - "iterations": 1, "interim_results": interim_results, "circuits": ReferenceCircuits.bell(), } @@ -268,7 +274,7 @@ def test_wait_for_final_state_after_job_status(self, service): @run_integration_test def test_job_creation_date(self, service): """Test job creation date.""" - job = self._run_program(service, iterations=1) + job = self._run_program(service) self.assertTrue(job.creation_date) rjob = service.job(job.job_id()) self.assertTrue(rjob.creation_date) diff --git a/test/integration/test_results.py b/test/integration/test_results.py index e8677b53b..ad02a24da 100644 --- a/test/integration/test_results.py +++ b/test/integration/test_results.py @@ -49,7 +49,6 @@ def result_callback(job_id, result): job = self._run_program( service, backend="ibmq_qasm_simulator", - iterations=iterations, interim_results=int_res, callback=result_callback, ) @@ -83,7 +82,6 @@ def result_callback(job_id, result): job = self._run_program( service, backend="ibmq_qasm_simulator", - iterations=iterations, interim_results=int_res, callback=result_callback, ) @@ -115,7 +113,6 @@ def result_callback(job_id, result): job = self._run_program( service, backend="ibmq_qasm_simulator", - iterations=iterations, interim_results=int_res, ) job.stream_results(result_callback) @@ -139,7 +136,6 @@ def result_callback(job_id, result): service, backend="ibmq_qasm_simulator", interim_results="foobar", - sleep_per_iteration=10, ) job.wait_for_final_state() job._status = JobStatus.RUNNING # Allow stream_results() @@ -220,11 +216,10 @@ def result_callback(job_id, result): for status in sub_tests: with self.subTest(status=status): if status == JobStatus.QUEUED: - _ = self._run_program(service, iterations=10) + _ = self._run_program(service) job = self._run_program( service=service, - iterations=iterations, interim_results="foo", callback=result_callback, ) @@ -251,7 +246,6 @@ def result_callback(job_id, result): # pylint: disable=unused-argument job = self._run_program( service, backend="ibmq_qasm_simulator", - iterations=10, callback=result_callback, ) job.wait_for_final_state() @@ -275,6 +269,6 @@ def result_callback(job_id, result): # pylint: disable=unused-argument # TODO - verify WebsocketError in output log. For some reason self.assertLogs # doesn't always work even when the error is clearly logged. with use_proxies(service, invalid_proxy): - job = self._run_program(service, iterations=2, callback=result_callback) + job = self._run_program(service, callback=result_callback) job.wait_for_final_state() self.assertFalse(callback_called) diff --git a/test/integration/test_retrieve_job.py b/test/integration/test_retrieve_job.py index d1503a9d7..5997a9e62 100644 --- a/test/integration/test_retrieve_job.py +++ b/test/integration/test_retrieve_job.py @@ -30,8 +30,8 @@ class TestIntegrationRetrieveJob(IBMIntegrationJobTestCase): def test_retrieve_job_queued(self, service): """Test retrieving a queued job.""" real_device = get_real_device(service) - _ = self._run_program(service, iterations=10, backend=real_device) - job = self._run_program(service, iterations=2, backend=real_device) + _ = self._run_program(service, backend=real_device) + job = self._run_program(service, backend=real_device) wait_for_status(job, JobStatus.QUEUED) rjob = service.job(job.job_id()) self.assertEqual(job.job_id(), rjob.job_id()) @@ -40,7 +40,7 @@ def test_retrieve_job_queued(self, service): @run_integration_test def test_retrieve_job_running(self, service): """Test retrieving a running job.""" - job = self._run_program(service, iterations=10) + job = self._run_program(service) wait_for_status(job, JobStatus.RUNNING) rjob = service.job(job.job_id()) self.assertEqual(job.job_id(), rjob.job_id()) @@ -101,7 +101,8 @@ def test_retrieve_jobs_limit(self, service): @run_integration_test def test_retrieve_pending_jobs(self, service): """Test retrieving pending jobs (QUEUED, RUNNING).""" - job = self._run_program(service, iterations=20) + circuits = [ReferenceCircuits.bell()] * 20 + job = self._run_program(service, circuits=circuits) wait_for_status(job, JobStatus.RUNNING) rjobs = service.jobs(pending=True) after_status = job.status()