feat: Endpoints refreshed with new Request status model and Protobuf models added

.pre-commit-config.yaml

@@ -30,5 +30,9 @@ repos:
           - attrs
           - httpx
           - types-PyJWT
+          - grpcio-tools
+          - types-Pygments
+          - types-protobuf
+exclude: "^src/neptune_api/proto/"
 default_language_version:
     python: python3

CHANGELOG.md

@@ -5,8 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.1.0] [Unreleased]
+## [0.3.0] [Unreleased]
 
 ### Added
+- Endpoints refreshed with new Request status model and Protobuf models added ([#9](https://github.com/neptune-ai/neptune-api/pull/9))
 
-- ?
+## [0.2.0] - 2024-06-25
+
+### Added
+- Added OAuth JWT authorization support ([#8](https://github.com/neptune-ai/neptune-api/pull/8))
+
+## [0.1.0] - 2024-06-19
+
+### Added
+- Added initial package setup and OpenAPI client ([#1](https://github.com/neptune-ai/neptune-api/pull/1))
+
+[unreleased]: https://github.com/neptune-ai/neptune-api/compare/0.2.0...HEAD
+[0.2.0]: https://github.com/neptune-ai/neptune-api/compare/0.1.0...0.2.0
+[0.1.0]: https://github.com/neptune-ai/neptune-api/commits/0.1.0

dev_requirements.txt

@@ -10,3 +10,8 @@ pytest
 pytest-mock
 pytest-timeout
 freezegun
+grpcio-tools==1.60.0
+mypy-protobuf
+protoletariat
+types-Pygments
+icecream

openapi-generator-config.yaml

@@ -2,9 +2,9 @@ project_name_override: neptune-api
 package_name_override: neptune_api
 package_version_override: 0.1.0
 post_hooks:
-   - "ruff check . --fix"
-   - "ruff format ."
-   - "isort ."
+   - "ruff check neptune_api/ --fix"
+   - "ruff format neptune_api/"
+   - "isort neptune_api/"
    - "echo >> neptune_api/py.typed"
 content_type_overrides:
-  application/x-protobuf: application/json
+  application/x-protobuf: application/octet-stream

pyproject.toml

@@ -14,6 +14,7 @@ httpx = ">=0.15.4,<0.25.0"
 attrs = ">=21.3.0"
 python-dateutil = "^2.8.0"
 PyJWT = "^2.0.0"
+protobuf = "^4.0.0"
 
 [tool.poetry]
 name = "neptune-api"
@@ -72,9 +73,11 @@ exclude = '''
 profile = "black"
 line_length = 120
 force_grid_wrap = 2
+extend_skip = ["src/neptune_api/proto"]
 
 [tool.ruff]
 line-length = 120
+exclude = ["src/neptune_api/proto"]
 
 [tool.ruff.lint]
 select = ["F", "UP"]

scripts/preserve_files.txt

@@ -1,2 +1,5 @@
 neptune_api/credentials.py
 neptune_api/auth_helpers.py
+neptune_api/proto/
+neptune_api/api/data_ingestion/check_request_status.py
+neptune_api/api/data_ingestion/submit_operation.py

scripts/update.sh

@@ -3,14 +3,24 @@
 # Show every command and exit on error
 set -ex
 
+# Clean tmp directories
+rm -rf neptune_api tmp || true
+
 # Preserve package structure
 mv src/neptune_api neptune_api
 
-# Preserve specific files
+## Preserve specific files
 mkdir -p tmp
-cat scripts/preserve_files.txt | while read file; do
-    mkdir -p tmp/$(dirname $file) | true
-    mv $file tmp/$file
+cat scripts/preserve_files.txt | while read entry; do
+    if [ -d "$entry" ]; then
+        # If entry is a directory, copy it recursively
+        mkdir -p tmp/$entry
+        cp -r $entry/* tmp/$entry/
+    elif [ -f "$entry" ]; then
+        # If entry is a file, move it
+        mkdir -p tmp/$(dirname $entry)
+        mv $entry tmp/$entry
+    fi
 done
 
 # To generate in the same directory
@@ -27,10 +37,9 @@ openapi-python-client update \
 cd $INITIAL_DIRECTORY
 
 # Restore specific files
-cat scripts/preserve_files.txt | while read file; do
-    mkdir -p $(dirname $file) | true
-    mv tmp/$file $file
-done
+cp -R tmp/* .
+
+# Clean tmp directories
 rm -rf tmp
 
 # Restore package structure

src/neptune_api/api/data_ingestion/check_request_status.py

@@ -13,7 +13,7 @@
     AuthenticatedClient,
     Client,
 )
-from ...models.ingest_response import IngestResponse
+from ...proto.neptune_pb.ingest.v1.request_status_pb2 import RequestStatus
 from ...types import (
     UNSET,
     Response,
@@ -42,11 +42,10 @@ def _get_kwargs(
     return _kwargs
 
 
-def _parse_response(
-    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Optional[IngestResponse]:
+def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[RequestStatus]:
     if response.status_code == HTTPStatus.OK:
-        response_200 = IngestResponse.from_dict(response.json())
+        response_200 = RequestStatus()
+        response_200.ParseFromString(response.content)
 
         return response_200
     if client.raise_on_unexpected_status:
@@ -55,9 +54,7 @@ def _parse_response(
         return None
 
 
-def _build_response(
-    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Response[IngestResponse]:
+def _build_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Response[RequestStatus]:
     return Response(
         status_code=HTTPStatus(response.status_code),
         content=response.content,
@@ -71,7 +68,7 @@ def sync_detailed(
     client: AuthenticatedClient,
     project_identifier: str,
     request_id: str,
-) -> Response[IngestResponse]:
+) -> Response[RequestStatus]:
     """Checks operation processing status
 
     Args:
@@ -83,7 +80,7 @@ def sync_detailed(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[IngestResponse]
+        Response[RequestStatus]
     """
 
     kwargs = _get_kwargs(
@@ -103,7 +100,7 @@ def sync(
     client: AuthenticatedClient,
     project_identifier: str,
     request_id: str,
-) -> Optional[IngestResponse]:
+) -> Optional[RequestStatus]:
     """Checks operation processing status
 
     Args:
@@ -115,7 +112,7 @@ def sync(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        IngestResponse
+        RequestStatus
     """
 
     return sync_detailed(
@@ -130,7 +127,7 @@ async def asyncio_detailed(
     client: AuthenticatedClient,
     project_identifier: str,
     request_id: str,
-) -> Response[IngestResponse]:
+) -> Response[RequestStatus]:
     """Checks operation processing status
 
     Args:
@@ -142,7 +139,7 @@ async def asyncio_detailed(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[IngestResponse]
+        Response[RequestStatus]
     """
 
     kwargs = _get_kwargs(
@@ -160,7 +157,7 @@ async def asyncio(
     client: AuthenticatedClient,
     project_identifier: str,
     request_id: str,
-) -> Optional[IngestResponse]:
+) -> Optional[RequestStatus]:
     """Checks operation processing status
 
     Args:
@@ -172,7 +169,7 @@ async def asyncio(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        IngestResponse
+        RequestStatus
     """
 
     return (

src/neptune_api/api/data_ingestion/submit_operation.py

@@ -13,8 +13,8 @@
     AuthenticatedClient,
     Client,
 )
-from ...models.request_id import RequestId
-from ...models.run_operation import RunOperation
+from ...proto.neptune_pb.ingest.v1.pub.client_pb2 import RequestId
+from ...proto.neptune_pb.ingest.v1.pub.ingest_pb2 import RunOperation
 from ...types import Response
 
 
@@ -24,14 +24,8 @@ def _get_kwargs(
 ) -> Dict[str, Any]:
     headers: Dict[str, Any] = {}
 
-    _kwargs: Dict[str, Any] = {
-        "method": "post",
-        "url": "/api/client/v1/ingest",
-    }
+    _kwargs: Dict[str, Any] = {"method": "post", "url": "/api/client/v1/ingest", "content": body.SerializeToString()}
 
-    _body = body.to_dict()
-
-    _kwargs["json"] = _body
     headers["Content-Type"] = "application/x-protobuf"
 
     _kwargs["headers"] = headers
@@ -40,7 +34,8 @@ def _get_kwargs(
 
 def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[RequestId]:
     if response.status_code == HTTPStatus.OK:
-        response_200 = RequestId.from_dict(response.json())
+        response_200 = RequestId()
+        response_200.ParseFromString(response.content)
 
         return response_200
     if client.raise_on_unexpected_status:

src/neptune_api/models/__init__.py

@@ -3,21 +3,21 @@
 from .client_config import ClientConfig
 from .client_versions_config_dto import ClientVersionsConfigDTO
 from .error import Error
-from .ingest_response import IngestResponse
 from .neptune_oauth_token import NeptuneOauthToken
 from .project_dto import ProjectDTO
 from .request_id import RequestId
+from .request_status import RequestStatus
 from .run_operation import RunOperation
 from .security_dto import SecurityDTO
 
 __all__ = (
     "ClientConfig",
     "ClientVersionsConfigDTO",
     "Error",
-    "IngestResponse",
     "NeptuneOauthToken",
     "ProjectDTO",
     "RequestId",
+    "RequestStatus",
     "RunOperation",
     "SecurityDTO",
 )

src/neptune_api/models/ingest_response.py → src/neptune_api/models/request_status.py

@@ -9,11 +9,11 @@
 from attrs import define as _attrs_define
 from attrs import field as _attrs_field
 
-T = TypeVar("T", bound="IngestResponse")
+T = TypeVar("T", bound="RequestStatus")
 
 
 @_attrs_define
-class IngestResponse:
+class RequestStatus:
     """ """
 
     additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)
@@ -27,10 +27,10 @@ def to_dict(self) -> Dict[str, Any]:
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         d = src_dict.copy()
-        ingest_response = cls()
+        request_status = cls()
 
-        ingest_response.additional_properties = d
-        return ingest_response
+        request_status.additional_properties = d
+        return request_status
 
     @property
     def additional_keys(self) -> List[str]:

src/neptune_api/proto/__init__.pyi

@@ -0,0 +1,6 @@
+from . import (
+    google_rpc,
+    neptune_pb,
+)
+from . import google_rpc
+from . import neptune_pb

src/neptune_api/proto/google_rpc/__init__.pyi

@@ -0,0 +1 @@
+from . import code_pb2

src/neptune_api/proto/google_rpc/code_pb2.py

@@ -0,0 +1,15 @@
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_sym_db = _symbol_database.Default()
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15google_rpc/code.proto\x12\ngoogle_rpc*\xb7\x02\n\x04Code\x12\x06\n\x02OK\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11DEADLINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0eALREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13FAILED_PRECONDITION\x10\t\x12\x0b\n\x07ABORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0fBX\n\x0ecom.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPCb\x06proto3')
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google_rpc.code_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+    _globals['DESCRIPTOR']._options = None
+    _globals['DESCRIPTOR']._serialized_options = b'\n\x0ecom.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPC'
+    _globals['_CODE']._serialized_start = 38
+    _globals['_CODE']._serialized_end = 349