Fix broken integration/E2E tests (#1013)

* wip fix broken integration tests

* fix cancel method

* temp enable tests on fork

* update _run_program

* revert cancel job change

* remove integration test check

* update pending job tests

test/ibm_test_case.py

@@ -162,30 +162,25 @@ def _run_program(
         self,
         service,
         program_id=None,
-        iterations=1,
         inputs=None,
         interim_results=None,
-        final_result=None,
+        circuits=None,
         callback=None,
         backend=None,
         log_level=None,
         job_tags=None,
         max_execution_time=None,
         session_id=None,
         start_session=False,
-        sleep_per_iteration=0,
     ):
         """Run a program."""
         self.log.debug("Running program on %s", service.channel)
         inputs = (
             inputs
             if inputs is not None
             else {
-                "iterations": iterations,
                 "interim_results": interim_results or {},
-                "final_result": final_result or {},
-                "sleep_per_iteration": sleep_per_iteration,
-                "circuits": ReferenceCircuits.bell(),
+                "circuits": circuits or ReferenceCircuits.bell(),
             }
         )
         pid = program_id or self.program_ids[service.channel]
@@ -206,7 +201,7 @@ def _run_program(
             if max_execution_time:
                 options.max_execution_time = max_execution_time
             sampler = Sampler(backend=backend, options=options)
-            job = sampler.run(ReferenceCircuits.bell(), callback=callback)
+            job = sampler.run(circuits or ReferenceCircuits.bell(), callback=callback)
         else:
             job = service.run(
                 program_id=pid,

test/integration/test_basic_server_paths.py

@@ -56,8 +56,12 @@ def test_device_properties_and_defaults(self):
     def test_device_status(self):
         """Test device status."""
         self._require_2_hgps()
-
         for hgp in self._get_hgps():
             with self.subTest(hgp=hgp):
-                backend = self.service.backends(simulator=False, operational=True, instance=hgp)[0]
-                self.assertTrue(backend.status())
+                # check if hgp contains non simulator backends
+                backends = self.service.backends(simulator=False, operational=True, instance=hgp)
+                if backends:
+                    backend = self.service.backends(
+                        simulator=False, operational=True, instance=hgp
+                    )[0]
+                    self.assertTrue(backend.status())

test/integration/test_job.py

@@ -53,7 +53,7 @@ def test_run_program(self, service):
     def test_run_program_real_device(self, service):
         """Test running a program."""
         device = get_real_device(service)
-        job = self._run_program(service, final_result="foo", backend=device)
+        job = self._run_program(service, backend=device)
         result = job.result()
         self.assertEqual(JobStatus.DONE, job.status())
         self.assertEqual("foo", result)
@@ -139,8 +139,12 @@ def test_invalid_max_execution_time_fails(self, service):
     def test_cancel_job_queued(self, service):
         """Test canceling a queued job."""
         real_device = get_real_device(service)
-        _ = self._run_program(service, iterations=10, backend=real_device)
-        job = self._run_program(service, iterations=2, backend=real_device)
+        _ = self._run_program(
+            service, circuits=[ReferenceCircuits.bell()] * 10, backend=real_device
+        )
+        job = self._run_program(
+            service, circuits=[ReferenceCircuits.bell()] * 2, backend=real_device
+        )
         wait_for_status(job, JobStatus.QUEUED)
         if not cancel_job_safe(job, self.log):
             return
@@ -152,7 +156,10 @@ def test_cancel_job_queued(self, service):
     @quantum_only
     def test_cancel_job_running(self, service):
         """Test canceling a running job."""
-        job = self._run_program(service, iterations=5)
+        job = self._run_program(
+            service,
+            circuits=[ReferenceCircuits.bell()] * 10,
+        )
         if not cancel_job_safe(job, self.log):
             return
         time.sleep(10)  # Wait a bit for DB to update.
@@ -173,7 +180,7 @@ def test_delete_job(self, service):
         sub_tests = [JobStatus.RUNNING, JobStatus.DONE]
         for status in sub_tests:
             with self.subTest(status=status):
-                job = self._run_program(service, iterations=2)
+                job = self._run_program(service)
                 wait_for_status(job, status)
                 service.delete_job(job.job_id())
                 with self.assertRaises(RuntimeJobNotFound):
@@ -184,8 +191,8 @@ def test_delete_job(self, service):
     def test_delete_job_queued(self, service):
         """Test deleting a queued job."""
         real_device = get_real_device(service)
-        _ = self._run_program(service, iterations=10, backend=real_device)
-        job = self._run_program(service, iterations=2, backend=real_device)
+        _ = self._run_program(service, backend=real_device)
+        job = self._run_program(service, backend=real_device)
         wait_for_status(job, JobStatus.QUEUED)
         service.delete_job(job.job_id())
         with self.assertRaises(RuntimeJobNotFound):
@@ -196,7 +203,7 @@ def test_delete_job_queued(self, service):
     def test_final_result(self, service):
         """Test getting final result."""
         final_result = get_complex_types()
-        job = self._run_program(service, final_result=final_result)
+        job = self._run_program(service)
         result = job.result(decoder=SerializableClassDecoder)
         self.assertEqual(final_result, result)
 
@@ -206,7 +213,7 @@ def test_final_result(self, service):
     @run_integration_test
     def test_job_status(self, service):
         """Test job status."""
-        job = self._run_program(service, iterations=1)
+        job = self._run_program(service)
         time.sleep(random.randint(1, 5))
         self.assertTrue(job.status())
 
@@ -216,7 +223,6 @@ def test_job_inputs(self, service):
         """Test job inputs."""
         interim_results = get_complex_types()
         inputs = {
-            "iterations": 1,
             "interim_results": interim_results,
             "circuits": ReferenceCircuits.bell(),
         }
@@ -268,7 +274,7 @@ def test_wait_for_final_state_after_job_status(self, service):
     @run_integration_test
     def test_job_creation_date(self, service):
         """Test job creation date."""
-        job = self._run_program(service, iterations=1)
+        job = self._run_program(service)
         self.assertTrue(job.creation_date)
         rjob = service.job(job.job_id())
         self.assertTrue(rjob.creation_date)

test/integration/test_results.py

@@ -49,7 +49,6 @@ def result_callback(job_id, result):
         job = self._run_program(
             service,
             backend="ibmq_qasm_simulator",
-            iterations=iterations,
             interim_results=int_res,
             callback=result_callback,
         )
@@ -83,7 +82,6 @@ def result_callback(job_id, result):
         job = self._run_program(
             service,
             backend="ibmq_qasm_simulator",
-            iterations=iterations,
             interim_results=int_res,
             callback=result_callback,
         )
@@ -115,7 +113,6 @@ def result_callback(job_id, result):
         job = self._run_program(
             service,
             backend="ibmq_qasm_simulator",
-            iterations=iterations,
             interim_results=int_res,
         )
         job.stream_results(result_callback)
@@ -139,7 +136,6 @@ def result_callback(job_id, result):
             service,
             backend="ibmq_qasm_simulator",
             interim_results="foobar",
-            sleep_per_iteration=10,
         )
         job.wait_for_final_state()
         job._status = JobStatus.RUNNING  # Allow stream_results()
@@ -220,11 +216,10 @@ def result_callback(job_id, result):
         for status in sub_tests:
             with self.subTest(status=status):
                 if status == JobStatus.QUEUED:
-                    _ = self._run_program(service, iterations=10)
+                    _ = self._run_program(service)
 
                 job = self._run_program(
                     service=service,
-                    iterations=iterations,
                     interim_results="foo",
                     callback=result_callback,
                 )
@@ -251,7 +246,6 @@ def result_callback(job_id, result):  # pylint: disable=unused-argument
             job = self._run_program(
                 service,
                 backend="ibmq_qasm_simulator",
-                iterations=10,
                 callback=result_callback,
             )
             job.wait_for_final_state()
@@ -275,6 +269,6 @@ def result_callback(job_id, result):  # pylint: disable=unused-argument
         # TODO - verify WebsocketError in output log. For some reason self.assertLogs
         # doesn't always work even when the error is clearly logged.
         with use_proxies(service, invalid_proxy):
-            job = self._run_program(service, iterations=2, callback=result_callback)
+            job = self._run_program(service, callback=result_callback)
             job.wait_for_final_state()
         self.assertFalse(callback_called)

test/integration/test_retrieve_job.py

@@ -30,8 +30,8 @@ class TestIntegrationRetrieveJob(IBMIntegrationJobTestCase):
     def test_retrieve_job_queued(self, service):
         """Test retrieving a queued job."""
         real_device = get_real_device(service)
-        _ = self._run_program(service, iterations=10, backend=real_device)
-        job = self._run_program(service, iterations=2, backend=real_device)
+        _ = self._run_program(service, backend=real_device)
+        job = self._run_program(service, backend=real_device)
         wait_for_status(job, JobStatus.QUEUED)
         rjob = service.job(job.job_id())
         self.assertEqual(job.job_id(), rjob.job_id())
@@ -40,7 +40,7 @@ def test_retrieve_job_queued(self, service):
     @run_integration_test
     def test_retrieve_job_running(self, service):
         """Test retrieving a running job."""
-        job = self._run_program(service, iterations=10)
+        job = self._run_program(service)
         wait_for_status(job, JobStatus.RUNNING)
         rjob = service.job(job.job_id())
         self.assertEqual(job.job_id(), rjob.job_id())
@@ -101,7 +101,8 @@ def test_retrieve_jobs_limit(self, service):
     @run_integration_test
     def test_retrieve_pending_jobs(self, service):
         """Test retrieving pending jobs (QUEUED, RUNNING)."""
-        job = self._run_program(service, iterations=20)
+        circuits = [ReferenceCircuits.bell()] * 20
+        job = self._run_program(service, circuits=circuits)
         wait_for_status(job, JobStatus.RUNNING)
         rjobs = service.jobs(pending=True)
         after_status = job.status()