[Framework] Handle ReadTimeout errors (#983)

# Description

What - Handle ReadTimeout Errors

Why - Requests to port or 3party apps may take a while, on the other
hand we want to do out best effort to complete the resync

How - Increased the httpx client timeout and not raise exceptions during
batch upsert entities

## Type of change

Please leave one option from the following and delete the rest:

- [X] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

<h4> All tests should be run against the port production
environment(using a testing org). </h4>

### Core testing checklist

- [x] Integration able to create all default resources from scratch
- [x] Resync finishes successfully
- [x] Resync able to create entities
- [x] Resync able to update entities
- [x] Resync able to detect and delete entities
- [x] Scheduled resync able to abort existing resync and start a new one
- [x] Tested with at least 2 integrations from scratch
- [x] Tested with Kafka and Polling event listeners
- [x] Tested deletion of entities that don't pass the selector


### Integration testing checklist

- [ ] Integration able to create all default resources from scratch
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Resync finishes successfully
- [ ] If new resource kind is added or updated in the integration, add
example raw data, mapping and expected result to the `examples` folder
in the integration directory.
- [ ] If resource kind is updated, run the integration with the example
data and check if the expected result is achieved
- [ ] If new resource kind is added or updated, validate that
live-events for that resource are working as expected
- [ ] Docs PR link [here](#)

### Preflight checklist

- [ ] Handled rate limiting
- [ ] Handled pagination
- [ ] Implemented the code in async
- [ ] Support Multi account

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

Provide links to the API documentation used for this integration.

CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.10.8 (2024-09-04)
+
+### Bug Fixes
+
+- Avoid raising exception when receiving ReadTimeout on batch upsert entities
+- Increased both internal port client and third party client timeout to handle long requests
+
+
 ## 0.10.7 (2024-08-28)
 
 ### Improvements

port_ocean/clients/port/mixins/entities.py

@@ -101,7 +101,7 @@ async def batch_upsert_entities(
                 )
                 for entity in entities
             ),
-            return_exceptions=should_raise,
+            return_exceptions=True,
         )
         entity_results = [
             entity for entity in modified_entities_results if isinstance(entity, Entity)

port_ocean/clients/port/utils.py

@@ -18,7 +18,7 @@
 # The max_keepalive_connections can't be too high, as it will cause the application to run out of available connections.
 PORT_HTTP_MAX_CONNECTIONS_LIMIT = 200
 PORT_HTTP_MAX_KEEP_ALIVE_CONNECTIONS = 50
-PORT_HTTP_TIMEOUT = 10.0
+PORT_HTTP_TIMEOUT = 60.0
 
 PORT_HTTPX_TIMEOUT = httpx.Timeout(PORT_HTTP_TIMEOUT)
 PORT_HTTPX_LIMITS = httpx.Limits(

port_ocean/config/settings.py

@@ -67,7 +67,7 @@ class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
     allow_environment_variables_jq_access: bool = True
     initialize_port_resources: bool = True
     scheduled_resync_interval: int | None = None
-    client_timeout: int = 30
+    client_timeout: int = 60
     send_raw_data_examples: bool = True
     port: PortSettings
     event_listener: EventListenerSettingsType = Field(

port_ocean/helpers/retry.py

@@ -290,6 +290,11 @@ async def _retry_operation_async(
                 if remaining_attempts < 1:
                     self._log_error(request, error)
                     raise
+            except httpx.ReadTimeout as e:
+                error = e
+                if remaining_attempts < 1:
+                    self._log_error(request, error)
+                    raise
             except httpx.TimeoutException as e:
                 error = e
                 if remaining_attempts < 1:

port_ocean/tests/clients/port/mixins/test_entities.py

@@ -0,0 +1,53 @@
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+
+from port_ocean.clients.port.mixins.entities import EntityClientMixin
+from port_ocean.core.models import Entity
+from httpx import ReadTimeout
+
+
+errored_entity_identifier: str = "a"
+expected_result_entities = [
+    Entity(identifier="b", blueprint="b"),
+    Entity(identifier="c", blueprint="c"),
+]
+all_entities = [
+    Entity(identifier=errored_entity_identifier, blueprint="a")
+] + expected_result_entities
+
+
+async def mock_upsert_entity(entity: Entity, *args: Any, **kwargs: Any) -> Entity:
+    if entity.identifier == errored_entity_identifier:
+        raise ReadTimeout("")
+    else:
+        return entity
+
+
+@pytest.fixture
+async def entity_client(monkeypatch: Any) -> EntityClientMixin:
+    # Arrange
+    entity_client = EntityClientMixin(auth=MagicMock(), client=MagicMock())
+    monkeypatch.setattr(entity_client, "upsert_entity", mock_upsert_entity)
+
+    return entity_client
+
+
+async def test_batch_upsert_entities_read_timeout_should_raise_false(
+    entity_client: EntityClientMixin,
+) -> None:
+    result_entities = await entity_client.batch_upsert_entities(
+        entities=all_entities, request_options=MagicMock(), should_raise=False
+    )
+
+    assert result_entities == expected_result_entities
+
+
+async def test_batch_upsert_entities_read_timeout_should_raise_true(
+    entity_client: EntityClientMixin,
+) -> None:
+    with pytest.raises(ReadTimeout):
+        await entity_client.batch_upsert_entities(
+            entities=all_entities, request_options=MagicMock(), should_raise=True
+        )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.10.7"
+version = "0.10.8"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"