MRG: Merge branch 'rc-kueue' into factor-out-event-attributes

skipci

.gitignore

@@ -98,7 +98,7 @@ ENV/
 # See: https://github.com/google-github-actions/auth/issues/123
 google_credentials.json
 gha-creds-*.json
-gcp-creds-*.json
+gcp-cred*
 terraform/gcp-credentials.json
 .terraform
 

.pre-commit-config.yaml

@@ -31,11 +31,6 @@ repos:
     hooks:
       - id: prettier
 
-  - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.1.1
-    hooks:
-      - id: pydocstyle
-
   - repo: https://github.com/thclark/pre-commit-sphinx
     rev: 0.0.3
     hooks:

octue/cli.py

@@ -212,7 +212,7 @@ def local(input_values, input_manifest, attributes, service_config):
         input_values = json.loads(input_values, cls=OctueJSONDecoder)
 
     if input_manifest:
-        input_manifest = Manifest.deserialise(input_manifest, from_string=True)
+        input_manifest = json.loads(input_manifest, cls=OctueJSONDecoder)
 
     service_configuration, app_configuration = load_service_and_app_configuration(service_config)
 

octue/cloud/pub_sub/service.py

@@ -291,7 +291,7 @@ def ask(
 
         :param str service_id: the ID of the child to ask the question to
         :param any|None input_values: any input values for the question
-        :param octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question
+        :param dict|octue.resources.manifest.Manifest|None input_manifest: an input manifest of any datasets needed for the question
         :param list(dict)|None children: a list of children for the child to use instead of its default children (if it uses children). These should be in the same format as in an app's app configuration file and have the same keys.
         :param bool subscribe_to_logs: if `True`, subscribe to the child's logs and handle them with the local log handlers
         :param bool allow_local_files: if `True`, allow the input manifest to contain references to local files - this should only be set to `True` if the child will be able to access these local files

octue/definitions.py

@@ -18,5 +18,5 @@
 # TODO this should probably be defined in twined
 RUN_STRANDS = ("input_values", "input_manifest", "credentials", "children")
 
-GOOGLE_COMPUTE_PROVIDERS = {"GOOGLE_CLOUD_FUNCTION", "GOOGLE_KUEUE"}
+GOOGLE_COMPUTE_PROVIDERS = {"GOOGLE_CLOUD_FUNCTION"}
 LOCAL_SDK_VERSION = importlib.metadata.version("octue")

pyproject.toml

@@ -67,16 +67,24 @@ ruff = "^0.6.9"
 
 [tool.ruff]
 line-length = 120
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes.
-lint.select = ["E", "F"]
+
+[tool.ruff.lint]
+# Enable pydocstyle (`D`), pycodestyle (`E`), and Pyflakes (`F`) codes.
+select = ["D", "E", "F"]
 # Ignore E501 line-too-long - see https://docs.astral.sh/ruff/faq/#is-the-ruff-linter-compatible-with-black for why
-lint.ignore = ["F405", "E501", "E203", "E731", "N818"]
+ignore = [
+    "D100", "D101", "D102", "D103", "D104", "D105", "D107", "D203", "D205", "D213", "D400", "D415",
+    "E501", "E203", "E731",
+    "F405",
+    "N818",
+]
 
 [tool.ruff.lint.isort]
 known-first-party = ["octue", "app", "fractal", "test", "tests", "twined"]
 section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]
 force-sort-within-sections = true
 
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

tests/cloud/deployment/test_cloud_run_deployment.py → tests/cloud/deployment/test_kueue_deployment.py

@@ -9,15 +9,14 @@
 from octue.resources import Child
 import twined.exceptions
 
-EXAMPLE_SERVICE_SRUID = "octue/example-service:0.5.0"
+EXAMPLE_SERVICE_SRUID = "octue/example-service-kueue:0.1.0"
 
 
 @unittest.skipUnless(
-    condition=os.getenv("RUN_CLOUD_RUN_DEPLOYMENT_TEST", "0").lower() == "1",
-    reason="'RUN_CLOUD_RUN_DEPLOYMENT_TEST' environment variable is False or not present.",
+    condition=os.getenv("RUN_DEPLOYMENT_TEST", "0").lower() == "1",
+    reason="'RUN_DEPLOYMENT_TEST' environment variable is False or not present.",
 )
-class TestCloudRunDeployment(TestCase):
-    # This is the service ID of the example service deployed to Google Cloud Run.
+class TestKueueDeployment(TestCase):
     child = Child(
         id=EXAMPLE_SERVICE_SRUID,
         backend={"name": "GCPPubSubBackend", "project_name": os.environ["TEST_PROJECT_NAME"]},
@@ -29,8 +28,8 @@ def test_forwards_exceptions_to_parent(self):
             self.child.ask(input_values={"invalid_input_data": "hello"})
 
     def test_synchronous_question(self):
-        """Test that the Google Cloud Run example deployment works, providing a service that can be asked questions and
-        send responses.
+        """Test that the Kueue example deployment works, providing a service that can be asked questions and send
+        responses.
         """
         answer, _ = self.child.ask(input_values={"n_iterations": 3})
 
@@ -47,9 +46,9 @@ def test_asynchronous_question(self):
         self.assertIsNone(answer)
 
         # Wait for question to complete.
-        time.sleep(15)
+        time.sleep(60)
 
-        events = get_events(table_id="octue_sdk_python_test_dataset.service-events", question_uuid=question_uuid)
+        events = get_events(table_id="octue_twined.service-events", question_uuid=question_uuid)
 
         self.assertTrue(
             is_event_valid(

tests/test_cli.py

@@ -12,7 +12,7 @@
 from octue.cloud.emulators._pub_sub import MockService
 from octue.cloud.emulators.service import ServicePatcher
 from octue.configuration import AppConfiguration, ServiceConfiguration
-from octue.resources import Dataset
+from octue.resources import Dataset, Manifest
 from octue.utils.patches import MultiPatcher
 from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME, TESTS_DIR
 from tests.base import BaseTestCase
@@ -230,7 +230,7 @@ def test_with_input_manifest(self):
 
         # Check question event.
         question = mock_answer_question_kwargs["question"]
-        self.assertEqual(question["event"]["input_manifest"].id, input_manifest.id)
+        self.assertEqual(Manifest.deserialise(question["event"]["input_manifest"]).id, input_manifest.id)
 
         # Check question attributes.
         self.assertTrue(question["attributes"]["recipient"].startswith("testing/test-app"))
@@ -272,7 +272,7 @@ def test_with_input_values_and_manifest(self):
         # Check question event.
         question = mock_answer_question_kwargs["question"]
         self.assertEqual(question["event"]["input_values"], input_values)
-        self.assertEqual(question["event"]["input_manifest"].id, input_manifest.id)
+        self.assertEqual(Manifest.deserialise(question["event"]["input_manifest"]).id, input_manifest.id)
 
         # Check question attributes.
         self.assertTrue(question["attributes"]["recipient"].startswith("testing/test-app"))

tests/test_log_handlers.py

@@ -17,11 +17,11 @@
 
 
 class TestLogging(BaseTestCase):
-    def test_log_record_attributes_without_timestamp_used_if_compute_provider_is_google_kueue(self):
+    def test_log_record_attributes_without_timestamp_used_if_compute_provider_is_google_cloud_function(self):
         """Test that the formatter without a timestamp is used for logging if the `COMPUTE_PROVIDER` environment
-        variable is present and equal to "GOOGLE_KUEUE", and `USE_OCTUE_LOG_HANDLER` is equal to "1".
+        variable is present and equal to "GOOGLE_CLOUD_FUNCTION", and `USE_OCTUE_LOG_HANDLER` is equal to "1".
         """
-        with mock.patch.dict(os.environ, USE_OCTUE_LOG_HANDLER="1", COMPUTE_PROVIDER="GOOGLE_KUEUE"):
+        with mock.patch.dict(os.environ, USE_OCTUE_LOG_HANDLER="1", COMPUTE_PROVIDER="GOOGLE_CLOUD_FUNCTION"):
             with mock.patch("octue.log_handlers.create_octue_formatter") as create_octue_formatter:
                 importlib.reload(sys.modules["octue"])
 
@@ -32,9 +32,9 @@ def test_log_record_attributes_without_timestamp_used_if_compute_provider_is_goo
             include_thread_name=False,
         )
 
-    def test_log_record_attributes_with_timestamp_used_if_compute_provider_is_not_google_kueue(self):
+    def test_log_record_attributes_with_timestamp_used_if_compute_provider_is_not_google_cloud_function(self):
         """Test that the formatter without a timestamp is used for logging if the `COMPUTE_PROVIDER` environment
-        variable is present and not equal to "GOOGLE_KUEUE", and `USE_OCTUE_LOG_HANDLER` is equal to "1".
+        variable is present and not equal to "GOOGLE_CLOUD_FUNCTION", and `USE_OCTUE_LOG_HANDLER` is equal to "1".
         """
         with mock.patch.dict(os.environ, USE_OCTUE_LOG_HANDLER="1", COMPUTE_PROVIDER="BLAH"):
             with mock.patch("octue.log_handlers.create_octue_formatter") as create_octue_formatter: