diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5933a4..e241cb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,5 +30,9 @@ repos: - attrs - httpx - types-PyJWT + - grpcio-tools + - types-Pygments + - types-protobuf +exclude: "^src/neptune_api/proto/" default_language_version: python: python3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2283564..ac95da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.0] [Unreleased] +## [0.3.0] [Unreleased] ### Added +- Endpoints refreshed with new Request status model and Protobuf models added ([#9](https://github.com/neptune-ai/neptune-api/pull/9)) -- ? +## [0.2.0] - 2024-06-25 + +### Added +- Added OAuth JWT authorization support ([#8](https://github.com/neptune-ai/neptune-api/pull/8)) + +## [0.1.0] - 2024-06-19 + +### Added +- Added initial package setup and OpenAPI client ([#1](https://github.com/neptune-ai/neptune-api/pull/1)) + +[unreleased]: https://github.com/neptune-ai/neptune-api/compare/0.2.0...HEAD +[0.2.0]: https://github.com/neptune-ai/neptune-api/compare/0.1.0...0.2.0 +[0.1.0]: https://github.com/neptune-ai/neptune-api/commits/0.1.0 diff --git a/dev_requirements.txt b/dev_requirements.txt index a1a3b9a..62a53c2 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -10,3 +10,8 @@ pytest pytest-mock pytest-timeout freezegun +grpcio-tools==1.60.0 +mypy-protobuf +protoletariat +types-Pygments +icecream diff --git a/openapi-generator-config.yaml b/openapi-generator-config.yaml index bcf312d..b4d5102 100644 --- a/openapi-generator-config.yaml +++ b/openapi-generator-config.yaml @@ -2,9 +2,9 @@ project_name_override: neptune-api package_name_override: neptune_api package_version_override: 0.1.0 post_hooks: - - "ruff check . --fix" - - "ruff format ." - - "isort ." + - "ruff check neptune_api/ --fix" + - "ruff format neptune_api/" + - "isort neptune_api/" - "echo >> neptune_api/py.typed" content_type_overrides: - application/x-protobuf: application/json + application/x-protobuf: application/octet-stream diff --git a/pyproject.toml b/pyproject.toml index 0fcb72b..9655e73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ httpx = ">=0.15.4,<0.25.0" attrs = ">=21.3.0" python-dateutil = "^2.8.0" PyJWT = "^2.0.0" +protobuf = "^4.0.0" [tool.poetry] name = "neptune-api" @@ -72,9 +73,11 @@ exclude = ''' profile = "black" line_length = 120 force_grid_wrap = 2 +extend_skip = ["src/neptune_api/proto"] [tool.ruff] line-length = 120 +exclude = ["src/neptune_api/proto"] [tool.ruff.lint] select = ["F", "UP"] diff --git a/scripts/preserve_files.txt b/scripts/preserve_files.txt index dbcb09c..93449ff 100644 --- a/scripts/preserve_files.txt +++ b/scripts/preserve_files.txt @@ -1,2 +1,5 @@ neptune_api/credentials.py neptune_api/auth_helpers.py +neptune_api/proto/ +neptune_api/api/data_ingestion/check_request_status.py +neptune_api/api/data_ingestion/submit_operation.py diff --git a/scripts/update.sh b/scripts/update.sh index b8a8fba..4c9e654 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -3,14 +3,24 @@ # Show every command and exit on error set -ex +# Clean tmp directories +rm -rf neptune_api tmp || true + # Preserve package structure mv src/neptune_api neptune_api -# Preserve specific files +## Preserve specific files mkdir -p tmp -cat scripts/preserve_files.txt | while read file; do - mkdir -p tmp/$(dirname $file) | true - mv $file tmp/$file +cat scripts/preserve_files.txt | while read entry; do + if [ -d "$entry" ]; then + # If entry is a directory, copy it recursively + mkdir -p tmp/$entry + cp -r $entry/* tmp/$entry/ + elif [ -f "$entry" ]; then + # If entry is a file, move it + mkdir -p tmp/$(dirname $entry) + mv $entry tmp/$entry + fi done # To generate in the same directory @@ -27,10 +37,9 @@ openapi-python-client update \ cd $INITIAL_DIRECTORY # Restore specific files -cat scripts/preserve_files.txt | while read file; do - mkdir -p $(dirname $file) | true - mv tmp/$file $file -done +cp -R tmp/* . + +# Clean tmp directories rm -rf tmp # Restore package structure diff --git a/src/neptune_api/api/data_ingestion/check_request_status.py b/src/neptune_api/api/data_ingestion/check_request_status.py index 34247d7..d13b122 100644 --- a/src/neptune_api/api/data_ingestion/check_request_status.py +++ b/src/neptune_api/api/data_ingestion/check_request_status.py @@ -8,13 +8,13 @@ import httpx -from ... import errors -from ...client import ( +from neptune_api import errors +from neptune_api.client import ( AuthenticatedClient, Client, ) -from ...models.ingest_response import IngestResponse -from ...types import ( +from neptune_api.proto.neptune_pb.ingest.v1.request_status_pb2 import RequestStatus +from neptune_api.types import ( UNSET, Response, ) @@ -42,11 +42,10 @@ def _get_kwargs( return _kwargs -def _parse_response( - *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[IngestResponse]: +def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[RequestStatus]: if response.status_code == HTTPStatus.OK: - response_200 = IngestResponse.from_dict(response.json()) + response_200 = RequestStatus() + response_200.ParseFromString(response.content) return response_200 if client.raise_on_unexpected_status: @@ -55,9 +54,7 @@ def _parse_response( return None -def _build_response( - *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[IngestResponse]: +def _build_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Response[RequestStatus]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -71,7 +68,7 @@ def sync_detailed( client: AuthenticatedClient, project_identifier: str, request_id: str, -) -> Response[IngestResponse]: +) -> Response[RequestStatus]: """Checks operation processing status Args: @@ -83,7 +80,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[IngestResponse] + Response[RequestStatus] """ kwargs = _get_kwargs( @@ -103,7 +100,7 @@ def sync( client: AuthenticatedClient, project_identifier: str, request_id: str, -) -> Optional[IngestResponse]: +) -> Optional[RequestStatus]: """Checks operation processing status Args: @@ -115,7 +112,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - IngestResponse + RequestStatus """ return sync_detailed( @@ -130,7 +127,7 @@ async def asyncio_detailed( client: AuthenticatedClient, project_identifier: str, request_id: str, -) -> Response[IngestResponse]: +) -> Response[RequestStatus]: """Checks operation processing status Args: @@ -142,7 +139,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[IngestResponse] + Response[RequestStatus] """ kwargs = _get_kwargs( @@ -160,7 +157,7 @@ async def asyncio( client: AuthenticatedClient, project_identifier: str, request_id: str, -) -> Optional[IngestResponse]: +) -> Optional[RequestStatus]: """Checks operation processing status Args: @@ -172,7 +169,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - IngestResponse + RequestStatus """ return ( diff --git a/src/neptune_api/api/data_ingestion/submit_operation.py b/src/neptune_api/api/data_ingestion/submit_operation.py index b8b148a..16254c3 100644 --- a/src/neptune_api/api/data_ingestion/submit_operation.py +++ b/src/neptune_api/api/data_ingestion/submit_operation.py @@ -8,14 +8,15 @@ import httpx +from neptune_api.proto.neptune_pb.ingest.v1.pub.client_pb2 import RequestId +from neptune_api.proto.neptune_pb.ingest.v1.pub.ingest_pb2 import RunOperation +from neptune_api.types import Response + from ... import errors from ...client import ( AuthenticatedClient, Client, ) -from ...models.request_id import RequestId -from ...models.run_operation import RunOperation -from ...types import Response def _get_kwargs( @@ -24,14 +25,8 @@ def _get_kwargs( ) -> Dict[str, Any]: headers: Dict[str, Any] = {} - _kwargs: Dict[str, Any] = { - "method": "post", - "url": "/api/client/v1/ingest", - } - - _body = body.to_dict() + _kwargs: Dict[str, Any] = {"method": "post", "url": "/api/client/v1/ingest", "content": body.SerializeToString()} - _kwargs["json"] = _body headers["Content-Type"] = "application/x-protobuf" _kwargs["headers"] = headers @@ -40,7 +35,8 @@ def _get_kwargs( def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[RequestId]: if response.status_code == HTTPStatus.OK: - response_200 = RequestId.from_dict(response.json()) + response_200 = RequestId() + response_200.ParseFromString(response.content) return response_200 if client.raise_on_unexpected_status: diff --git a/src/neptune_api/models/__init__.py b/src/neptune_api/models/__init__.py index e493417..5fdf2ec 100644 --- a/src/neptune_api/models/__init__.py +++ b/src/neptune_api/models/__init__.py @@ -3,10 +3,10 @@ from .client_config import ClientConfig from .client_versions_config_dto import ClientVersionsConfigDTO from .error import Error -from .ingest_response import IngestResponse from .neptune_oauth_token import NeptuneOauthToken from .project_dto import ProjectDTO from .request_id import RequestId +from .request_status import RequestStatus from .run_operation import RunOperation from .security_dto import SecurityDTO @@ -14,10 +14,10 @@ "ClientConfig", "ClientVersionsConfigDTO", "Error", - "IngestResponse", "NeptuneOauthToken", "ProjectDTO", "RequestId", + "RequestStatus", "RunOperation", "SecurityDTO", ) diff --git a/src/neptune_api/models/ingest_response.py b/src/neptune_api/models/request_status.py similarity index 85% rename from src/neptune_api/models/ingest_response.py rename to src/neptune_api/models/request_status.py index 090de71..d155719 100644 --- a/src/neptune_api/models/ingest_response.py +++ b/src/neptune_api/models/request_status.py @@ -9,11 +9,11 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -T = TypeVar("T", bound="IngestResponse") +T = TypeVar("T", bound="RequestStatus") @_attrs_define -class IngestResponse: +class RequestStatus: """ """ additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -27,10 +27,10 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: d = src_dict.copy() - ingest_response = cls() + request_status = cls() - ingest_response.additional_properties = d - return ingest_response + request_status.additional_properties = d + return request_status @property def additional_keys(self) -> List[str]: diff --git a/src/neptune_api/proto/__init__.py b/src/neptune_api/proto/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/__init__.pyi b/src/neptune_api/proto/__init__.pyi new file mode 100644 index 0000000..84e311b --- /dev/null +++ b/src/neptune_api/proto/__init__.pyi @@ -0,0 +1,6 @@ +from . import ( + google_rpc, + neptune_pb, +) +from . import google_rpc +from . import neptune_pb diff --git a/src/neptune_api/proto/google_rpc/__init__.py b/src/neptune_api/proto/google_rpc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/google_rpc/__init__.pyi b/src/neptune_api/proto/google_rpc/__init__.pyi new file mode 100644 index 0000000..1044a22 --- /dev/null +++ b/src/neptune_api/proto/google_rpc/__init__.pyi @@ -0,0 +1 @@ +from . import code_pb2 diff --git a/src/neptune_api/proto/google_rpc/code_pb2.py b/src/neptune_api/proto/google_rpc/code_pb2.py new file mode 100644 index 0000000..22d305f --- /dev/null +++ b/src/neptune_api/proto/google_rpc/code_pb2.py @@ -0,0 +1,15 @@ +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_sym_db = _symbol_database.Default() +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15google_rpc/code.proto\x12\ngoogle_rpc*\xb7\x02\n\x04Code\x12\x06\n\x02OK\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11DEADLINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0eALREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13FAILED_PRECONDITION\x10\t\x12\x0b\n\x07ABORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0fBX\n\x0ecom.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPCb\x06proto3') +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google_rpc.code_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\x0ecom.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPC' + _globals['_CODE']._serialized_start = 38 + _globals['_CODE']._serialized_end = 349 \ No newline at end of file diff --git a/src/neptune_api/proto/google_rpc/code_pb2.pyi b/src/neptune_api/proto/google_rpc/code_pb2.pyi new file mode 100644 index 0000000..1b0ff7d --- /dev/null +++ b/src/neptune_api/proto/google_rpc/code_pb2.pyi @@ -0,0 +1,113 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper +import sys +import typing +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _Code: + ValueType = typing.NewType('ValueType', builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _CodeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_Code.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + OK: _Code.ValueType + 'Not an error; returned on success.\n\n HTTP Mapping: 200 OK\n ' + CANCELLED: _Code.ValueType + 'The operation was cancelled, typically by the caller.\n\n HTTP Mapping: 499 Client Closed Request\n ' + UNKNOWN: _Code.ValueType + 'Unknown error. For example, this error may be returned when\n a `Status` value received from another address space belongs to\n an error space that is not known in this address space. Also\n errors raised by APIs that do not return enough error information\n may be converted to this error.\n\n HTTP Mapping: 500 Internal Server Error\n ' + INVALID_ARGUMENT: _Code.ValueType + 'The client specified an invalid argument. Note that this differs\n from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments\n that are problematic regardless of the state of the system\n (e.g., a malformed file name).\n\n HTTP Mapping: 400 Bad Request\n ' + DEADLINE_EXCEEDED: _Code.ValueType + 'The deadline expired before the operation could complete. For operations\n that change the state of the system, this error may be returned\n even if the operation has completed successfully. For example, a\n successful response from a server could have been delayed long\n enough for the deadline to expire.\n\n HTTP Mapping: 504 Gateway Timeout\n ' + NOT_FOUND: _Code.ValueType + 'Some requested entity (e.g., file or directory) was not found.\n\n Note to server developers: if a request is denied for an entire class\n of users, such as gradual feature rollout or undocumented allowlist,\n `NOT_FOUND` may be used. If a request is denied for some users within\n a class of users, such as user-based access control, `PERMISSION_DENIED`\n must be used.\n\n HTTP Mapping: 404 Not Found\n ' + ALREADY_EXISTS: _Code.ValueType + 'The entity that a client attempted to create (e.g., file or directory)\n already exists.\n\n HTTP Mapping: 409 Conflict\n ' + PERMISSION_DENIED: _Code.ValueType + 'The caller does not have permission to execute the specified\n operation. `PERMISSION_DENIED` must not be used for rejections\n caused by exhausting some resource (use `RESOURCE_EXHAUSTED`\n instead for those errors). `PERMISSION_DENIED` must not be\n used if the caller can not be identified (use `UNAUTHENTICATED`\n instead for those errors). This error code does not imply the\n request is valid or the requested entity exists or satisfies\n other pre-conditions.\n\n HTTP Mapping: 403 Forbidden\n ' + UNAUTHENTICATED: _Code.ValueType + 'The request does not have valid authentication credentials for the\n operation.\n\n HTTP Mapping: 401 Unauthorized\n ' + RESOURCE_EXHAUSTED: _Code.ValueType + 'Some resource has been exhausted, perhaps a per-user quota, or\n perhaps the entire file system is out of space.\n\n HTTP Mapping: 429 Too Many Requests\n ' + FAILED_PRECONDITION: _Code.ValueType + 'The operation was rejected because the system is not in a state\n required for the operation\'s execution. For example, the directory\n to be deleted is non-empty, an rmdir operation is applied to\n a non-directory, etc.\n\n Service implementors can use the following guidelines to decide\n between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`:\n (a) Use `UNAVAILABLE` if the client can retry just the failing call.\n (b) Use `ABORTED` if the client should retry at a higher level. For\n example, when a client-specified test-and-set fails, indicating the\n client should restart a read-modify-write sequence.\n (c) Use `FAILED_PRECONDITION` if the client should not retry until\n the system state has been explicitly fixed. For example, if an "rmdir"\n fails because the directory is non-empty, `FAILED_PRECONDITION`\n should be returned since the client should not retry unless\n the files are deleted from the directory.\n\n HTTP Mapping: 400 Bad Request\n ' + ABORTED: _Code.ValueType + 'The operation was aborted, typically due to a concurrency issue such as\n a sequencer check failure or transaction abort.\n\n See the guidelines above for deciding between `FAILED_PRECONDITION`,\n `ABORTED`, and `UNAVAILABLE`.\n\n HTTP Mapping: 409 Conflict\n ' + OUT_OF_RANGE: _Code.ValueType + 'The operation was attempted past the valid range. E.g., seeking or\n reading past end-of-file.\n\n Unlike `INVALID_ARGUMENT`, this error indicates a problem that may\n be fixed if the system state changes. For example, a 32-bit file\n system will generate `INVALID_ARGUMENT` if asked to read at an\n offset that is not in the range [0,2^32-1], but it will generate\n `OUT_OF_RANGE` if asked to read from an offset past the current\n file size.\n\n There is a fair bit of overlap between `FAILED_PRECONDITION` and\n `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific\n error) when it applies so that callers who are iterating through\n a space can easily look for an `OUT_OF_RANGE` error to detect when\n they are done.\n\n HTTP Mapping: 400 Bad Request\n ' + UNIMPLEMENTED: _Code.ValueType + 'The operation is not implemented or is not supported/enabled in this\n service.\n\n HTTP Mapping: 501 Not Implemented\n ' + INTERNAL: _Code.ValueType + 'Internal errors. This means that some invariants expected by the\n underlying system have been broken. This error code is reserved\n for serious errors.\n\n HTTP Mapping: 500 Internal Server Error\n ' + UNAVAILABLE: _Code.ValueType + 'The service is currently unavailable. This is most likely a\n transient condition, which can be corrected by retrying with\n a backoff. Note that it is not always safe to retry\n non-idempotent operations.\n\n See the guidelines above for deciding between `FAILED_PRECONDITION`,\n `ABORTED`, and `UNAVAILABLE`.\n\n HTTP Mapping: 503 Service Unavailable\n ' + DATA_LOSS: _Code.ValueType + 'Unrecoverable data loss or corruption.\n\n HTTP Mapping: 500 Internal Server Error\n ' + +class Code(_Code, metaclass=_CodeEnumTypeWrapper): + """The canonical error codes for gRPC APIs. + + + Sometimes multiple error codes may apply. Services should return + the most specific error code that applies. For example, prefer + `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply. + Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`. + """ +OK: Code.ValueType +'Not an error; returned on success.\n\nHTTP Mapping: 200 OK\n' +CANCELLED: Code.ValueType +'The operation was cancelled, typically by the caller.\n\nHTTP Mapping: 499 Client Closed Request\n' +UNKNOWN: Code.ValueType +'Unknown error. For example, this error may be returned when\na `Status` value received from another address space belongs to\nan error space that is not known in this address space. Also\nerrors raised by APIs that do not return enough error information\nmay be converted to this error.\n\nHTTP Mapping: 500 Internal Server Error\n' +INVALID_ARGUMENT: Code.ValueType +'The client specified an invalid argument. Note that this differs\nfrom `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments\nthat are problematic regardless of the state of the system\n(e.g., a malformed file name).\n\nHTTP Mapping: 400 Bad Request\n' +DEADLINE_EXCEEDED: Code.ValueType +'The deadline expired before the operation could complete. For operations\nthat change the state of the system, this error may be returned\neven if the operation has completed successfully. For example, a\nsuccessful response from a server could have been delayed long\nenough for the deadline to expire.\n\nHTTP Mapping: 504 Gateway Timeout\n' +NOT_FOUND: Code.ValueType +'Some requested entity (e.g., file or directory) was not found.\n\nNote to server developers: if a request is denied for an entire class\nof users, such as gradual feature rollout or undocumented allowlist,\n`NOT_FOUND` may be used. If a request is denied for some users within\na class of users, such as user-based access control, `PERMISSION_DENIED`\nmust be used.\n\nHTTP Mapping: 404 Not Found\n' +ALREADY_EXISTS: Code.ValueType +'The entity that a client attempted to create (e.g., file or directory)\nalready exists.\n\nHTTP Mapping: 409 Conflict\n' +PERMISSION_DENIED: Code.ValueType +'The caller does not have permission to execute the specified\noperation. `PERMISSION_DENIED` must not be used for rejections\ncaused by exhausting some resource (use `RESOURCE_EXHAUSTED`\ninstead for those errors). `PERMISSION_DENIED` must not be\nused if the caller can not be identified (use `UNAUTHENTICATED`\ninstead for those errors). This error code does not imply the\nrequest is valid or the requested entity exists or satisfies\nother pre-conditions.\n\nHTTP Mapping: 403 Forbidden\n' +UNAUTHENTICATED: Code.ValueType +'The request does not have valid authentication credentials for the\noperation.\n\nHTTP Mapping: 401 Unauthorized\n' +RESOURCE_EXHAUSTED: Code.ValueType +'Some resource has been exhausted, perhaps a per-user quota, or\nperhaps the entire file system is out of space.\n\nHTTP Mapping: 429 Too Many Requests\n' +FAILED_PRECONDITION: Code.ValueType +'The operation was rejected because the system is not in a state\nrequired for the operation\'s execution. For example, the directory\nto be deleted is non-empty, an rmdir operation is applied to\na non-directory, etc.\n\nService implementors can use the following guidelines to decide\nbetween `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`:\n (a) Use `UNAVAILABLE` if the client can retry just the failing call.\n (b) Use `ABORTED` if the client should retry at a higher level. For\n example, when a client-specified test-and-set fails, indicating the\n client should restart a read-modify-write sequence.\n (c) Use `FAILED_PRECONDITION` if the client should not retry until\n the system state has been explicitly fixed. For example, if an "rmdir"\n fails because the directory is non-empty, `FAILED_PRECONDITION`\n should be returned since the client should not retry unless\n the files are deleted from the directory.\n\nHTTP Mapping: 400 Bad Request\n' +ABORTED: Code.ValueType +'The operation was aborted, typically due to a concurrency issue such as\na sequencer check failure or transaction abort.\n\nSee the guidelines above for deciding between `FAILED_PRECONDITION`,\n`ABORTED`, and `UNAVAILABLE`.\n\nHTTP Mapping: 409 Conflict\n' +OUT_OF_RANGE: Code.ValueType +'The operation was attempted past the valid range. E.g., seeking or\nreading past end-of-file.\n\nUnlike `INVALID_ARGUMENT`, this error indicates a problem that may\nbe fixed if the system state changes. For example, a 32-bit file\nsystem will generate `INVALID_ARGUMENT` if asked to read at an\noffset that is not in the range [0,2^32-1], but it will generate\n`OUT_OF_RANGE` if asked to read from an offset past the current\nfile size.\n\nThere is a fair bit of overlap between `FAILED_PRECONDITION` and\n`OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific\nerror) when it applies so that callers who are iterating through\na space can easily look for an `OUT_OF_RANGE` error to detect when\nthey are done.\n\nHTTP Mapping: 400 Bad Request\n' +UNIMPLEMENTED: Code.ValueType +'The operation is not implemented or is not supported/enabled in this\nservice.\n\nHTTP Mapping: 501 Not Implemented\n' +INTERNAL: Code.ValueType +'Internal errors. This means that some invariants expected by the\nunderlying system have been broken. This error code is reserved\nfor serious errors.\n\nHTTP Mapping: 500 Internal Server Error\n' +UNAVAILABLE: Code.ValueType +'The service is currently unavailable. This is most likely a\ntransient condition, which can be corrected by retrying with\na backoff. Note that it is not always safe to retry\nnon-idempotent operations.\n\nSee the guidelines above for deciding between `FAILED_PRECONDITION`,\n`ABORTED`, and `UNAVAILABLE`.\n\nHTTP Mapping: 503 Service Unavailable\n' +DATA_LOSS: Code.ValueType +'Unrecoverable data loss or corruption.\n\nHTTP Mapping: 500 Internal Server Error\n' +global___Code = Code \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/__init__.py b/src/neptune_api/proto/neptune_pb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/neptune_pb/__init__.pyi b/src/neptune_api/proto/neptune_pb/__init__.pyi new file mode 100644 index 0000000..c444245 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/__init__.pyi @@ -0,0 +1 @@ +from . import ingest diff --git a/src/neptune_api/proto/neptune_pb/ingest/__init__.py b/src/neptune_api/proto/neptune_pb/ingest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/neptune_pb/ingest/__init__.pyi b/src/neptune_api/proto/neptune_pb/ingest/__init__.pyi new file mode 100644 index 0000000..326cdf4 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/__init__.pyi @@ -0,0 +1 @@ +from . import v1 diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/__init__.py b/src/neptune_api/proto/neptune_pb/ingest/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/__init__.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/__init__.pyi new file mode 100644 index 0000000..17f695f --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/__init__.pyi @@ -0,0 +1,3 @@ +from . import request_status_pb2 +from . import pub +from . import common_pb2 diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.py b/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.py new file mode 100644 index 0000000..2c7f3c4 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.py @@ -0,0 +1,48 @@ +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_sym_db = _symbol_database.Default() +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!neptune_pb/ingest/v1/common.proto\x12\x11neptune.ingest.v1\x1a\x1fgoogle/protobuf/timestamp.proto"$\n\x04Step\x12\r\n\x05whole\x18\x01 \x01(\x04\x12\r\n\x05micro\x18\x02 \x01(\x04"a\n\tForkPoint\x12\x16\n\x0eparent_project\x18\x01 \x01(\t\x12\x15\n\rparent_run_id\x18\x02 \x01(\t\x12%\n\x04step\x18\x04 \x01(\x0b2\x17.neptune.ingest.v1.Step"\x1b\n\tStringSet\x12\x0e\n\x06values\x18\x01 \x03(\t"\xbb\x01\n\x05Value\x12\x11\n\x07float64\x18\x01 \x01(\x01H\x00\x12\x0f\n\x05int64\x18\x03 \x01(\x03H\x00\x12\x0e\n\x04bool\x18\x05 \x01(\x08H\x00\x12\x10\n\x06string\x18\x06 \x01(\tH\x00\x12/\n\ttimestamp\x18\x08 \x01(\x0b2\x1a.google.protobuf.TimestampH\x00\x122\n\nstring_set\x18\x0c \x01(\x0b2\x1c.neptune.ingest.v1.StringSetH\x00B\x07\n\x05value"\xa2\x01\n\x0fModifyStringSet\x12>\n\x06values\x18\x01 \x03(\x0b2..neptune.ingest.v1.ModifyStringSet.ValuesEntry\x1aO\n\x0bValuesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0e2 .neptune.ingest.v1.SET_OPERATION:\x028\x01"I\n\tModifySet\x124\n\x06string\x18\x01 \x01(\x0b2".neptune.ingest.v1.ModifyStringSetH\x00B\x06\n\x04type"\x97\x02\n\x03Run\x12\x13\n\x06run_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rexperiment_id\x18\x05 \x01(\tH\x01\x88\x01\x01\x120\n\nfork_point\x18\x02 \x01(\x0b2\x1c.neptune.ingest.v1.ForkPoint\x12\x13\n\x06family\x18\x04 \x01(\tH\x02\x88\x01\x01\x126\n\rcreation_time\x18\x03 \x01(\x0b2\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x12\x17\n\nrequest_id\x18\x06 \x01(\tH\x04\x88\x01\x01B\t\n\x07_run_idB\x10\n\x0e_experiment_idB\t\n\x07_familyB\x10\n\x0e_creation_timeB\r\n\x0b_request_id"\xc3\x04\n\x11UpdateRunSnapshot\x12%\n\x04step\x18\x01 \x01(\x0b2\x17.neptune.ingest.v1.Step\x12-\n\ttimestamp\x18\x02 \x01(\x0b2\x1a.google.protobuf.Timestamp\x12@\n\x06assign\x18\x04 \x03(\x0b20.neptune.ingest.v1.UpdateRunSnapshot.AssignEntry\x12I\n\x0bmodify_sets\x18\x05 \x03(\x0b24.neptune.ingest.v1.UpdateRunSnapshot.ModifySetsEntry\x12@\n\x06append\x18\x08 \x03(\x0b20.neptune.ingest.v1.UpdateRunSnapshot.AppendEntry\x12\x17\n\nrequest_id\x18\t \x01(\tH\x00\x88\x01\x01\x1aG\n\x0bAssignEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b2\x18.neptune.ingest.v1.Value:\x028\x01\x1aO\n\x0fModifySetsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b2\x1c.neptune.ingest.v1.ModifySet:\x028\x01\x1aG\n\x0bAppendEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b2\x18.neptune.ingest.v1.Value:\x028\x01B\r\n\x0b_request_id*.\n\rSET_OPERATION\x12\x08\n\x04NOOP\x10\x00\x12\x07\n\x03ADD\x10\x01\x12\n\n\x06REMOVE\x10\x02B5\n$ml.neptune.leaderboard.api.ingest.v1B\x0bCommonProtoP\x01b\x06proto3') +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'neptune_pb.ingest.v1.common_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n$ml.neptune.leaderboard.api.ingest.v1B\x0bCommonProtoP\x01' + _globals['_MODIFYSTRINGSET_VALUESENTRY']._options = None + _globals['_MODIFYSTRINGSET_VALUESENTRY']._serialized_options = b'8\x01' + _globals['_UPDATERUNSNAPSHOT_ASSIGNENTRY']._options = None + _globals['_UPDATERUNSNAPSHOT_ASSIGNENTRY']._serialized_options = b'8\x01' + _globals['_UPDATERUNSNAPSHOT_MODIFYSETSENTRY']._options = None + _globals['_UPDATERUNSNAPSHOT_MODIFYSETSENTRY']._serialized_options = b'8\x01' + _globals['_UPDATERUNSNAPSHOT_APPENDENTRY']._options = None + _globals['_UPDATERUNSNAPSHOT_APPENDENTRY']._serialized_options = b'8\x01' + _globals['_SET_OPERATION']._serialized_start = 1549 + _globals['_SET_OPERATION']._serialized_end = 1595 + _globals['_STEP']._serialized_start = 89 + _globals['_STEP']._serialized_end = 125 + _globals['_FORKPOINT']._serialized_start = 127 + _globals['_FORKPOINT']._serialized_end = 224 + _globals['_STRINGSET']._serialized_start = 226 + _globals['_STRINGSET']._serialized_end = 253 + _globals['_VALUE']._serialized_start = 256 + _globals['_VALUE']._serialized_end = 443 + _globals['_MODIFYSTRINGSET']._serialized_start = 446 + _globals['_MODIFYSTRINGSET']._serialized_end = 608 + _globals['_MODIFYSTRINGSET_VALUESENTRY']._serialized_start = 529 + _globals['_MODIFYSTRINGSET_VALUESENTRY']._serialized_end = 608 + _globals['_MODIFYSET']._serialized_start = 610 + _globals['_MODIFYSET']._serialized_end = 683 + _globals['_RUN']._serialized_start = 686 + _globals['_RUN']._serialized_end = 965 + _globals['_UPDATERUNSNAPSHOT']._serialized_start = 968 + _globals['_UPDATERUNSNAPSHOT']._serialized_end = 1547 + _globals['_UPDATERUNSNAPSHOT_ASSIGNENTRY']._serialized_start = 1307 + _globals['_UPDATERUNSNAPSHOT_ASSIGNENTRY']._serialized_end = 1378 + _globals['_UPDATERUNSNAPSHOT_MODIFYSETSENTRY']._serialized_start = 1380 + _globals['_UPDATERUNSNAPSHOT_MODIFYSETSENTRY']._serialized_end = 1459 + _globals['_UPDATERUNSNAPSHOT_APPENDENTRY']._serialized_start = 1461 + _globals['_UPDATERUNSNAPSHOT_APPENDENTRY']._serialized_end = 1532 \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.pyi new file mode 100644 index 0000000..770394d --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/common_pb2.pyi @@ -0,0 +1,396 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.timestamp_pb2 +import sys +import typing +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _SET_OPERATION: + ValueType = typing.NewType('ValueType', builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _SET_OPERATIONEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_SET_OPERATION.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + NOOP: _SET_OPERATION.ValueType + ADD: _SET_OPERATION.ValueType + REMOVE: _SET_OPERATION.ValueType + +class SET_OPERATION(_SET_OPERATION, metaclass=_SET_OPERATIONEnumTypeWrapper): + """SET_OPERATION is used to describe the operation to be performed on a set.""" +NOOP: SET_OPERATION.ValueType +ADD: SET_OPERATION.ValueType +REMOVE: SET_OPERATION.ValueType +global___SET_OPERATION = SET_OPERATION + +@typing_extensions.final +class Step(google.protobuf.message.Message): + """Step is used to measure computational progress of the Run and it's used to stamp its state. + For example, to express Step `3.5`, use `Step{whole: 3, micro: 500_000`}. + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + WHOLE_FIELD_NUMBER: builtins.int + MICRO_FIELD_NUMBER: builtins.int + whole: builtins.int + 'Whole step index.' + micro: builtins.int + 'Fractional part of a step expressed as number of micro steps.\n Expression `0 <= micro < 1_000_000` must be true at all times.\n ' + + def __init__(self, *, whole: builtins.int=..., micro: builtins.int=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['micro', b'micro', 'whole', b'whole']) -> None: + ... +global___Step = Step + +@typing_extensions.final +class ForkPoint(google.protobuf.message.Message): + """ForkPoint is used to mark the parent and its last inherited state during Forking.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + PARENT_PROJECT_FIELD_NUMBER: builtins.int + PARENT_RUN_ID_FIELD_NUMBER: builtins.int + STEP_FIELD_NUMBER: builtins.int + parent_project: builtins.str + 'Optional. Parent project qualified name. If not set, it will default to the context project.' + parent_run_id: builtins.str + 'Required. The id of the parent run within the parent project.' + + @property + def step(self) -> global___Step: + """Fork Step, which is the last step that a new run will inherit from its parent. + If not set, it will default to the last seen step of the parent run by the server at the time of forking. + New run may start numbering steps from the next micro step after the fork step. + """ + + def __init__(self, *, parent_project: builtins.str=..., parent_run_id: builtins.str=..., step: global___Step | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['step', b'step']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['parent_project', b'parent_project', 'parent_run_id', b'parent_run_id', 'step', b'step']) -> None: + ... +global___ForkPoint = ForkPoint + +@typing_extensions.final +class StringSet(google.protobuf.message.Message): + """StringSet represents a set of strings. The order of strings is irrelevant and duplicates are ignored.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + VALUES_FIELD_NUMBER: builtins.int + + @property + def values(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + ... + + def __init__(self, *, values: collections.abc.Iterable[builtins.str] | None=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['values', b'values']) -> None: + ... +global___StringSet = StringSet + +@typing_extensions.final +class Value(google.protobuf.message.Message): + """Value is a union of all supported types that can be used to update a field. + Different types of operations support different subset of this field, so please refer to the documentation. + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + FLOAT64_FIELD_NUMBER: builtins.int + INT64_FIELD_NUMBER: builtins.int + BOOL_FIELD_NUMBER: builtins.int + STRING_FIELD_NUMBER: builtins.int + TIMESTAMP_FIELD_NUMBER: builtins.int + STRING_SET_FIELD_NUMBER: builtins.int + float64: builtins.float + int64: builtins.int + bool: builtins.bool + string: builtins.str + + @property + def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: + ... + + @property + def string_set(self) -> global___StringSet: + ... + + def __init__(self, *, float64: builtins.float=..., int64: builtins.int=..., bool: builtins.bool=..., string: builtins.str=..., timestamp: google.protobuf.timestamp_pb2.Timestamp | None=..., string_set: global___StringSet | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['bool', b'bool', 'float64', b'float64', 'int64', b'int64', 'string', b'string', 'string_set', b'string_set', 'timestamp', b'timestamp', 'value', b'value']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['bool', b'bool', 'float64', b'float64', 'int64', b'int64', 'string', b'string', 'string_set', b'string_set', 'timestamp', b'timestamp', 'value', b'value']) -> None: + ... + + def WhichOneof(self, oneof_group: typing_extensions.Literal['value', b'value']) -> typing_extensions.Literal['float64', 'int64', 'bool', 'string', 'timestamp', 'string_set'] | None: + ... +global___Value = Value + +@typing_extensions.final +class ModifyStringSet(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class ValuesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: global___SET_OPERATION.ValueType + + def __init__(self, *, key: builtins.str=..., value: global___SET_OPERATION.ValueType=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['key', b'key', 'value', b'value']) -> None: + ... + VALUES_FIELD_NUMBER: builtins.int + + @property + def values(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, global___SET_OPERATION.ValueType]: + ... + + def __init__(self, *, values: collections.abc.Mapping[builtins.str, global___SET_OPERATION.ValueType] | None=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['values', b'values']) -> None: + ... +global___ModifyStringSet = ModifyStringSet + +@typing_extensions.final +class ModifySet(google.protobuf.message.Message): + """Allows to update tag values in an incremental way.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + STRING_FIELD_NUMBER: builtins.int + + @property + def string(self) -> global___ModifyStringSet: + ... + + def __init__(self, *, string: global___ModifyStringSet | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['string', b'string', 'type', b'type']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['string', b'string', 'type', b'type']) -> None: + ... + + def WhichOneof(self, oneof_group: typing_extensions.Literal['type', b'type']) -> typing_extensions.Literal['string'] | None: + ... +global___ModifySet = ModifySet + +@typing_extensions.final +class Run(google.protobuf.message.Message): + """CreateRun can be used to create a new run. This can be done in two ways: + 1. Create a new run with no inherited state. You may specify a new run family that will be + inherited by future runs forking from this one, otherwise the new family will be selected by the server. + 2. Create a new run that inherits state from the parent run up to a specific step. You may specify + a new run family that will be inherited by future runs forking from this one. By default, the new run + will be in the same Family as the parent run. + CreateRun is idempotent, as long as `fork_point` and `family` parameters are the same. In case pf conflict, + the second operation will fail. + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + RUN_ID_FIELD_NUMBER: builtins.int + EXPERIMENT_ID_FIELD_NUMBER: builtins.int + FORK_POINT_FIELD_NUMBER: builtins.int + FAMILY_FIELD_NUMBER: builtins.int + CREATION_TIME_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int + run_id: builtins.str + 'Id of the run to be created. Optional if parent context has already specified run_id. If both are set, they\n must be equal, otherwise the operation will fail.\n ' + experiment_id: builtins.str + 'Experiment Id to assign to this Run. If Experiment Id is already assigned to another Run, specifying it in this\n field will move it from the previous Run, so that at most one Run in given project has a given Experiment Id.\n Note: Experiment Id is currently exposed as "sys/name" field in the Run metadata.\n ' + + @property + def fork_point(self) -> global___ForkPoint: + """Optional. ForkPoint is used to identify the exact point in the parent history from which the new run continues. + If not specified, the new run will start with no inherited state. + """ + family: builtins.str + 'Specifies Family for the new run. Run Family is used to group forking runs that share common ancestry.\n By default, the new forking run will be in the same family as the parent run.\n ' + + @property + def creation_time(self) -> google.protobuf.timestamp_pb2.Timestamp: + """User-specified creation time for the new run. This is especially useful for getting consistent relative timestamps + for series. + If not specified, server will use its current time instead when processing the request. + """ + request_id: builtins.str + 'Optional. The request ID generated by the Neptune client, used for tracking outcome of run creation.' + + def __init__(self, *, run_id: builtins.str | None=..., experiment_id: builtins.str | None=..., fork_point: global___ForkPoint | None=..., family: builtins.str | None=..., creation_time: google.protobuf.timestamp_pb2.Timestamp | None=..., request_id: builtins.str | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['_creation_time', b'_creation_time', '_experiment_id', b'_experiment_id', '_family', b'_family', '_request_id', b'_request_id', '_run_id', b'_run_id', 'creation_time', b'creation_time', 'experiment_id', b'experiment_id', 'family', b'family', 'fork_point', b'fork_point', 'request_id', b'request_id', 'run_id', b'run_id']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['_creation_time', b'_creation_time', '_experiment_id', b'_experiment_id', '_family', b'_family', '_request_id', b'_request_id', '_run_id', b'_run_id', 'creation_time', b'creation_time', 'experiment_id', b'experiment_id', 'family', b'family', 'fork_point', b'fork_point', 'request_id', b'request_id', 'run_id', b'run_id']) -> None: + ... + + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal['_creation_time', b'_creation_time']) -> typing_extensions.Literal['creation_time'] | None: + ... + + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal['_experiment_id', b'_experiment_id']) -> typing_extensions.Literal['experiment_id'] | None: + ... + + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal['_family', b'_family']) -> typing_extensions.Literal['family'] | None: + ... + + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal['_request_id', b'_request_id']) -> typing_extensions.Literal['request_id'] | None: + ... + + @typing.overload + def WhichOneof(self, oneof_group: typing_extensions.Literal['_run_id', b'_run_id']) -> typing_extensions.Literal['run_id'] | None: + ... +global___Run = Run + +@typing_extensions.final +class UpdateRunSnapshot(google.protobuf.message.Message): + """Stores Snapshot information about updated fields for a single step. Only passed fields will be updated.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class AssignEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + + @property + def value(self) -> global___Value: + ... + + def __init__(self, *, key: builtins.str=..., value: global___Value | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['value', b'value']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['key', b'key', 'value', b'value']) -> None: + ... + + @typing_extensions.final + class ModifySetsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + + @property + def value(self) -> global___ModifySet: + ... + + def __init__(self, *, key: builtins.str=..., value: global___ModifySet | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['value', b'value']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['key', b'key', 'value', b'value']) -> None: + ... + + @typing_extensions.final + class AppendEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + + @property + def value(self) -> global___Value: + ... + + def __init__(self, *, key: builtins.str=..., value: global___Value | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['value', b'value']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['key', b'key', 'value', b'value']) -> None: + ... + STEP_FIELD_NUMBER: builtins.int + TIMESTAMP_FIELD_NUMBER: builtins.int + ASSIGN_FIELD_NUMBER: builtins.int + MODIFY_SETS_FIELD_NUMBER: builtins.int + APPEND_FIELD_NUMBER: builtins.int + REQUEST_ID_FIELD_NUMBER: builtins.int + + @property + def step(self) -> global___Step: + """Optional. Step value within the run. If not set, it will default to next full step of the run + (highest step across step values). + """ + + @property + def timestamp(self) -> google.protobuf.timestamp_pb2.Timestamp: + """Timestamp field common to all included operations""" + + @property + def assign(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Value]: + """Assigns values for fields. Fields `field_path` is used as a key. Example: + ``` + {assign: { + "parameters/learning_rate": {float64: 0.01}, + "parameters/optimizer": {string: "sgd"}}} + ``` + Note: when using a StringSet value, this action will override the whole Set with the new values. + If you want to add or remove tags individually, use `modify_set.string` instead. + """ + + @property + def modify_sets(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___ModifySet]: + """Modify string sets with incremental changes. For example: + ``` + {modify_sets: { + "sys/tags": { + "string": { + values: { + "new_tag1": ADD, + "prev_tag1": REMOVE, + "new_tag2": ADD}}}} + ``` + """ + + @property + def append(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Value]: + """Appends values for Series fields. Fields `field_path` is used as a key. Example: + ``` + {assign: { + "metrics/recall": {float64: 0.6}, + "metrics/precision": {float64: 0.72}}} + ``` + Note: when using a StringSet value, this action will override the whole Set with the new values. + If you want to add or remove tags individually, use `modify_set.string` instead. + """ + request_id: builtins.str + 'Optional. The request ID generated by the Neptune client, used for tracking outcome of run update.' + + def __init__(self, *, step: global___Step | None=..., timestamp: google.protobuf.timestamp_pb2.Timestamp | None=..., assign: collections.abc.Mapping[builtins.str, global___Value] | None=..., modify_sets: collections.abc.Mapping[builtins.str, global___ModifySet] | None=..., append: collections.abc.Mapping[builtins.str, global___Value] | None=..., request_id: builtins.str | None=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['_request_id', b'_request_id', 'request_id', b'request_id', 'step', b'step', 'timestamp', b'timestamp']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['_request_id', b'_request_id', 'append', b'append', 'assign', b'assign', 'modify_sets', b'modify_sets', 'request_id', b'request_id', 'step', b'step', 'timestamp', b'timestamp']) -> None: + ... + + def WhichOneof(self, oneof_group: typing_extensions.Literal['_request_id', b'_request_id']) -> typing_extensions.Literal['request_id'] | None: + ... +global___UpdateRunSnapshot = UpdateRunSnapshot \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/__init__.py b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/__init__.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/__init__.pyi new file mode 100644 index 0000000..5895e3b --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/__init__.pyi @@ -0,0 +1,2 @@ +from . import client_pb2 +from . import ingest_pb2 diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.py b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.py new file mode 100644 index 0000000..a8cd302 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.py @@ -0,0 +1,15 @@ +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_sym_db = _symbol_database.Default() +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%neptune_pb/ingest/v1/pub/client.proto\x12\x15neptune.ingest.v1.pub"\x1a\n\tRequestId\x12\r\n\x05value\x18\x01 \x01(\tB3\n\x1cml.neptune.client.api.modelsB\x11ClientIngestProtoP\x01b\x06proto3') +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'neptune_pb.ingest.v1.pub.client_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\x1cml.neptune.client.api.modelsB\x11ClientIngestProtoP\x01' + _globals['_REQUESTID']._serialized_start = 64 + _globals['_REQUESTID']._serialized_end = 90 \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.pyi new file mode 100644 index 0000000..4d79be4 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/client_pb2.pyi @@ -0,0 +1,26 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.message +import sys +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class RequestId(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + VALUE_FIELD_NUMBER: builtins.int + value: builtins.str + + def __init__(self, *, value: builtins.str=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['value', b'value']) -> None: + ... +global___RequestId = RequestId \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.py b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.py new file mode 100644 index 0000000..c1010ad --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.py @@ -0,0 +1,16 @@ +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_sym_db = _symbol_database.Default() +from .....neptune_pb.ingest.v1 import common_pb2 as neptune__pb_dot_ingest_dot_v1_dot_common__pb2 +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%neptune_pb/ingest/v1/pub/ingest.proto\x12\x15neptune.ingest.v1.pub\x1a!neptune_pb/ingest/v1/common.proto"\xcf\x01\n\x0cRunOperation\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x1e\n\x16create_missing_project\x18\x03 \x01(\x08\x12(\n\x06create\x18\x06 \x01(\x0b2\x16.neptune.ingest.v1.RunH\x00\x126\n\x06update\x18\x07 \x01(\x0b2$.neptune.ingest.v1.UpdateRunSnapshotH\x00\x12\x0f\n\x07api_key\x18\x0c \x01(\x0cB\x0b\n\toperationB<\n(ml.neptune.leaderboard.api.ingest.v1.pubB\x0eIngestPubProtoP\x01b\x06proto3') +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'neptune_pb.ingest.v1.pub.ingest_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n(ml.neptune.leaderboard.api.ingest.v1.pubB\x0eIngestPubProtoP\x01' + _globals['_RUNOPERATION']._serialized_start = 100 + _globals['_RUNOPERATION']._serialized_end = 307 \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.pyi new file mode 100644 index 0000000..c0ca730 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/pub/ingest_pb2.pyi @@ -0,0 +1,56 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.message +from ..... import neptune_pb +import sys +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class RunOperation(google.protobuf.message.Message): + """RunOperation is a message body for the operation to be performed on the run.""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + PROJECT_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + CREATE_MISSING_PROJECT_FIELD_NUMBER: builtins.int + CREATE_FIELD_NUMBER: builtins.int + UPDATE_FIELD_NUMBER: builtins.int + API_KEY_FIELD_NUMBER: builtins.int + project: builtins.str + 'Qualified name of the context project. This name must include workspace, e.g.: "your-workspace/your-project".' + run_id: builtins.str + "Required. Subject run id of the operation. In case of `CreateRun` it's the id of the new run.\n User must ensure uniqueness of the id within the project.\n " + create_missing_project: builtins.bool + "Optional. Will create project if it doesn't yet exist. This operation is idempotent." + + @property + def create(self) -> neptune_pb.ingest.v1.common_pb2.Run: + """Creates a new run. See `CreateRun` for details.""" + + @property + def update(self) -> neptune_pb.ingest.v1.common_pb2.UpdateRunSnapshot: + """All included fields will be aligned to the same step. In case the step is not set, it will select the + successor of the highest last_step value among metrics currently being updated. + """ + api_key: builtins.bytes + 'API Key used for authorization of operations.\n See https://docs.neptune.ai/setup/setting_api_token/ for more information on how to obtain an API Key.\n ' + + def __init__(self, *, project: builtins.str=..., run_id: builtins.str=..., create_missing_project: builtins.bool=..., create: neptune_pb.ingest.v1.common_pb2.Run | None=..., update: neptune_pb.ingest.v1.common_pb2.UpdateRunSnapshot | None=..., api_key: builtins.bytes=...) -> None: + ... + + def HasField(self, field_name: typing_extensions.Literal['create', b'create', 'operation', b'operation', 'update', b'update']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing_extensions.Literal['api_key', b'api_key', 'create', b'create', 'create_missing_project', b'create_missing_project', 'operation', b'operation', 'project', b'project', 'run_id', b'run_id', 'update', b'update']) -> None: + ... + + def WhichOneof(self, oneof_group: typing_extensions.Literal['operation', b'operation']) -> typing_extensions.Literal['create', 'update'] | None: + ... +global___RunOperation = RunOperation \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.py b/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.py new file mode 100644 index 0000000..8d5d59d --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.py @@ -0,0 +1,18 @@ +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_sym_db = _symbol_database.Default() +from ....google_rpc import code_pb2 as google__rpc_dot_code__pb2 +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)neptune_pb/ingest/v1/request_status.proto\x12\x15neptune.ingest.v1.pub\x1a\x15google_rpc/code.proto"\x96\x01\n\rRequestStatus\x12G\n\rcode_by_count\x18\x01 \x03(\x0b20.neptune.ingest.v1.pub.RequestStatus.CodeByCount\x1a<\n\x0bCodeByCount\x12\x1e\n\x04code\x18\x01 \x01(\x0e2\x10.google_rpc.Code\x12\r\n\x05count\x18\x02 \x01(\x03BQ\n3ml.neptune.leaderboard.api.ingest.v1.request.statusB\x18IngestRequestStatusProtoP\x01b\x06proto3') +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'neptune_pb.ingest.v1.request_status_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'\n3ml.neptune.leaderboard.api.ingest.v1.request.statusB\x18IngestRequestStatusProtoP\x01' + _globals['_REQUESTSTATUS']._serialized_start = 92 + _globals['_REQUESTSTATUS']._serialized_end = 242 + _globals['_REQUESTSTATUS_CODEBYCOUNT']._serialized_start = 182 + _globals['_REQUESTSTATUS_CODEBYCOUNT']._serialized_end = 242 \ No newline at end of file diff --git a/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.pyi b/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.pyi new file mode 100644 index 0000000..1cebd18 --- /dev/null +++ b/src/neptune_api/proto/neptune_pb/ingest/v1/request_status_pb2.pyi @@ -0,0 +1,47 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +from .... import google_rpc +import sys +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class RequestStatus(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class CodeByCount(google.protobuf.message.Message): + """in case all operations are successful {"OK": 1}, otherwise errors are reported""" + DESCRIPTOR: google.protobuf.descriptor.Descriptor + CODE_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + code: google_rpc.code_pb2.Code.ValueType + count: builtins.int + + def __init__(self, *, code: google_rpc.code_pb2.Code.ValueType=..., count: builtins.int=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['code', b'code', 'count', b'count']) -> None: + ... + CODE_BY_COUNT_FIELD_NUMBER: builtins.int + + @property + def code_by_count(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___RequestStatus.CodeByCount]: + ... + + def __init__(self, *, code_by_count: collections.abc.Iterable[global___RequestStatus.CodeByCount] | None=...) -> None: + ... + + def ClearField(self, field_name: typing_extensions.Literal['code_by_count', b'code_by_count']) -> None: + ... +global___RequestStatus = RequestStatus \ No newline at end of file diff --git a/templates/types.py.jinja b/templates/types.py.jinja index cd291f3..391d937 100644 --- a/templates/types.py.jinja +++ b/templates/types.py.jinja @@ -89,8 +89,8 @@ class OAuthToken: @property def seconds_left(self) -> float: - return self._expiration_time - time.time() + return self._expiration_time - time.time() - MINIMAL_EXPIRATION_SECONDS @property def is_expired(self) -> bool: - return self.seconds_left <= MINIMAL_EXPIRATION_SECONDS + return self.seconds_left <= 0