Split test_integration_job (#92)

* split job integration tests

* fix lint

Co-authored-by: Rathish Cholarajan <rathishc24@gmail.com>

.github/workflows/cron-prod.yml

@@ -75,3 +75,33 @@ jobs:
           pip install -U -c constraints.txt -r requirements-dev.txt
       - name: Run Tests
         run: make test2
+  test3:
+    name: tests3-python${{ matrix.python-version }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        os: ["windows-latest", "ubuntu-latest"]
+    env:
+      QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }}
+      QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }}
+      QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }}
+      QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }}
+      QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }}
+      QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }}
+      LOG_LEVEL: DEBUG
+      STREAM_LOG: True
+      QISKIT_IN_PARALLEL: True
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -c constraints.txt -e .
+          pip install -U -c constraints.txt -r requirements-dev.txt
+      - name: Run Tests
+        run: make test3

.github/workflows/cron-staging.yml

@@ -77,3 +77,34 @@ jobs:
           pip install -U -c constraints.txt -r requirements-dev.txt
       - name: Run Tests
         run: make test2
+  test3:
+    name: tests3-python${{ matrix.python-version }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        os: ["windows-latest", "ubuntu-latest"]
+    env:
+      QISKIT_IBM_STAGING_API_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_API_TOKEN }}
+      QISKIT_IBM_STAGING_API_URL: ${{ secrets.QISKIT_IBM_STAGING_API_URL }}
+      QISKIT_IBM_STAGING_HGP: ${{ secrets.QISKIT_IBM_STAGING_HGP }}
+      QISKIT_IBM_STAGING_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_TOKEN }}
+      QISKIT_IBM_STAGING_CLOUD_URL: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_URL }}
+      QISKIT_IBM_STAGING_CLOUD_CRN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_CRN }}
+      QISKIT_IBM_USE_STAGING_CREDENTIALS: True
+      LOG_LEVEL: DEBUG
+      STREAM_LOG: True
+      QISKIT_IN_PARALLEL: True
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -c constraints.txt -e .
+          pip install -U -c constraints.txt -r requirements-dev.txt
+      - name: Run Tests
+        run: make test3

Makefile

@@ -11,7 +11,7 @@
 # that they have been altered from the originals.
 
 
-.PHONY: lint style test mypy test1 test2 test3 runtime_integration
+.PHONY: lint style test mypy test1 test2 test3
 
 lint:
 	pylint -rn qiskit_ibm_runtime test
@@ -32,8 +32,8 @@ test1:
 test2:
 	python -m unittest -v test/test_integration_job.py
 
-runtime_integration:
-	python -m unittest -v test/ibm/runtime/test_runtime_integration.py
+test3:
+	python -m unittest -v test/test_integration_retrieve_job.py	test/test_integration_interim_results.py
 
 black:
 	black qiskit_ibm_runtime setup.py test

test/ibm_test_case.py

@@ -13,16 +13,22 @@
 """Custom TestCase for IBM Provider."""
 
 import os
+import copy
 import logging
 import inspect
-from unittest import TestCase
+import unittest
+from contextlib import suppress
+from collections import defaultdict
 
 from qiskit_ibm_runtime import QISKIT_IBM_RUNTIME_LOGGER_NAME
+from qiskit_ibm_runtime.exceptions import IBMNotAuthorizedError
 
 from .utils.utils import setup_test_logging
+from .utils.decorators import requires_cloud_legacy_services
+from .utils.templates import RUNTIME_PROGRAM, RUNTIME_PROGRAM_METADATA, PROGRAM_PREFIX
 
 
-class IBMTestCase(TestCase):
+class IBMTestCase(unittest.TestCase):
     """Custom TestCase for use with the Qiskit IBM Runtime."""
 
     @classmethod
@@ -55,3 +61,137 @@ def _set_logging_level(cls, logger: logging.Logger) -> None:
         ):
             logger.addHandler(logging.StreamHandler())
             logger.propagate = False
+
+
+class IBMIntegrationTestCase(IBMTestCase):
+    """Custom integration test case for use with the Qiskit IBM Runtime."""
+
+    @classmethod
+    @requires_cloud_legacy_services
+    def setUpClass(cls, services):
+        """Initial class level setup."""
+        # pylint: disable=arguments-differ
+        super().setUpClass()
+        cls.services = services
+
+    def setUp(self) -> None:
+        """Test level setup."""
+        super().setUp()
+        self.to_delete = defaultdict(list)
+        self.to_cancel = defaultdict(list)
+
+    def tearDown(self) -> None:
+        """Test level teardown."""
+        super().tearDown()
+        # Delete programs
+        for service in self.services:
+            for prog in self.to_delete[service.auth]:
+                with suppress(Exception):
+                    service.delete_program(prog)
+
+        # Cancel and delete jobs.
+        for service in self.services:
+            for job in self.to_cancel[service.auth]:
+                with suppress(Exception):
+                    job.cancel()
+                with suppress(Exception):
+                    service.delete_job(job.job_id)
+
+    def _upload_program(
+        self,
+        service,
+        name=None,
+        max_execution_time=300,
+        data=None,
+        is_public: bool = False,
+    ):
+        """Upload a new program."""
+        name = name or PROGRAM_PREFIX
+        data = data or RUNTIME_PROGRAM
+        metadata = copy.deepcopy(RUNTIME_PROGRAM_METADATA)
+        metadata["name"] = name
+        metadata["max_execution_time"] = max_execution_time
+        metadata["is_public"] = is_public
+        program_id = service.upload_program(data=data, metadata=metadata)
+        self.to_delete[service.auth].append(program_id)
+        return program_id
+
+
+class IBMIntegrationJobTestCase(IBMIntegrationTestCase):
+    """Custom integration test case for job-related tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Initial class level setup."""
+        # pylint: disable=arguments-differ
+        # pylint: disable=no-value-for-parameter
+        super().setUpClass()
+        cls._create_default_program()
+        cls._find_sim_backends()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        """Class level teardown."""
+        super().tearDownClass()
+        # Delete default program.
+        with suppress(Exception):
+            for service in cls.services:
+                service.delete_program(cls.program_ids[service.auth])
+                cls.log.debug(
+                    "Deleted %s program %s", service.auth, cls.program_ids[service.auth]
+                )
+
+    @classmethod
+    def _create_default_program(cls):
+        """Create a default program."""
+        metadata = copy.deepcopy(RUNTIME_PROGRAM_METADATA)
+        metadata["name"] = PROGRAM_PREFIX
+        cls.program_ids = {}
+        cls.sim_backends = {}
+        for service in cls.services:
+            try:
+                prog_id = service.upload_program(
+                    data=RUNTIME_PROGRAM, metadata=metadata
+                )
+                cls.log.debug("Uploaded %s program %s", service.auth, prog_id)
+                cls.program_ids[service.auth] = prog_id
+            except IBMNotAuthorizedError:
+                raise unittest.SkipTest("No upload access.")
+
+    @classmethod
+    def _find_sim_backends(cls):
+        """Find a simulator backend for each service."""
+        for service in cls.services:
+            cls.sim_backends[service.auth] = service.backends(simulator=True)[0].name()
+
+    def _run_program(
+        self,
+        service,
+        program_id=None,
+        iterations=1,
+        inputs=None,
+        interim_results=None,
+        final_result=None,
+        callback=None,
+        backend=None,
+    ):
+        """Run a program."""
+        self.log.debug("Running program on %s", service.auth)
+        inputs = (
+            inputs
+            if inputs is not None
+            else {
+                "iterations": iterations,
+                "interim_results": interim_results or {},
+                "final_result": final_result or {},
+            }
+        )
+        pid = program_id or self.program_ids[service.auth]
+        backend_name = backend or self.sim_backends[service.auth]
+        options = {"backend_name": backend_name}
+        job = service.run(
+            program_id=pid, inputs=inputs, options=options, callback=callback
+        )
+        self.log.info("Runtime job %s submitted.", job.job_id)
+        self.to_cancel[service.auth].append(job)
+        return job

test/test_integration_backend.py

@@ -14,25 +14,16 @@
 
 from unittest import SkipTest
 
-from .ibm_test_case import IBMTestCase
+from .ibm_test_case import IBMIntegrationTestCase
 from .utils.decorators import (
-    requires_cloud_legacy_services,
     run_cloud_legacy_real,
     requires_cloud_legacy_devices,
 )
 
 
-class TestIntegrationBackend(IBMTestCase):
+class TestIntegrationBackend(IBMIntegrationTestCase):
     """Integration tests for backend functions."""
 
-    @classmethod
-    @requires_cloud_legacy_services
-    def setUpClass(cls, services):
-        """Initial class level setup."""
-        # pylint: disable=arguments-differ
-        super().setUpClass()
-        cls.services = services
-
     @run_cloud_legacy_real
     def test_backends(self, service):
         """Test getting all backends."""
@@ -53,14 +44,15 @@ def test_get_backend(self, service):
         self.assertTrue(backend)
 
 
-class TestIBMBackend(IBMTestCase):
+class TestIBMBackend(IBMIntegrationTestCase):
     """Test ibm_backend module."""
 
     @classmethod
     @requires_cloud_legacy_devices
     def setUpClass(cls, devices):
         """Initial class level setup."""
         # pylint: disable=arguments-differ
+        # pylint: disable=no-value-for-parameter
         super().setUpClass()
         cls.devices = devices