diff --git a/.gitignore b/.gitignore index 94e4886..804fc04 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,7 @@ pyproject.toml.bak .pdm-python # ruff -.ruff_cache/* \ No newline at end of file +.ruff_cache/* + +# debugging scripts +funky_experiments/* \ No newline at end of file diff --git a/pdm.lock b/pdm.lock index 9586082..6fc6e7f 100644 --- a/pdm.lock +++ b/pdm.lock @@ -1743,7 +1743,7 @@ files = [ [[package]] name = "mypy" -version = "1.7.0" +version = "1.7.1" requires_python = ">=3.8" summary = "Optional static typing for Python" dependencies = [ @@ -1752,28 +1752,28 @@ dependencies = [ "typing-extensions>=4.1.0", ] files = [ - {file = "mypy-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5da84d7bf257fd8f66b4f759a904fd2c5a765f70d8b52dde62b521972a0a2357"}, - {file = "mypy-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a3637c03f4025f6405737570d6cbfa4f1400eb3c649317634d273687a09ffc2f"}, - {file = "mypy-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b633f188fc5ae1b6edca39dae566974d7ef4e9aaaae00bc36efe1f855e5173ac"}, - {file = "mypy-1.7.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6ed9a3997b90c6f891138e3f83fb8f475c74db4ccaa942a1c7bf99e83a989a1"}, - {file = "mypy-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:1fe46e96ae319df21359c8db77e1aecac8e5949da4773c0274c0ef3d8d1268a9"}, - {file = "mypy-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401"}, - {file = "mypy-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d"}, - {file = "mypy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1"}, - {file = "mypy-1.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46"}, - {file = "mypy-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee"}, - {file = "mypy-1.7.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:87c076c174e2c7ef8ab416c4e252d94c08cd4980a10967754f91571070bf5fbe"}, - {file = "mypy-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cb8d5f6d0fcd9e708bb190b224089e45902cacef6f6915481806b0c77f7786d"}, - {file = "mypy-1.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93e76c2256aa50d9c82a88e2f569232e9862c9982095f6d54e13509f01222fc"}, - {file = "mypy-1.7.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cddee95dea7990e2215576fae95f6b78a8c12f4c089d7e4367564704e99118d3"}, - {file = "mypy-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:d01921dbd691c4061a3e2ecdbfbfad029410c5c2b1ee88946bf45c62c6c91210"}, - {file = "mypy-1.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c1b06b4b109e342f7dccc9efda965fc3970a604db70f8560ddfdee7ef19afb05"}, - {file = "mypy-1.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bf7a2f0a6907f231d5e41adba1a82d7d88cf1f61a70335889412dec99feeb0f8"}, - {file = "mypy-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551d4a0cdcbd1d2cccdcc7cb516bb4ae888794929f5b040bb51aae1846062901"}, - {file = "mypy-1.7.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:55d28d7963bef00c330cb6461db80b0b72afe2f3c4e2963c99517cf06454e665"}, - {file = "mypy-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:870bd1ffc8a5862e593185a4c169804f2744112b4a7c55b93eb50f48e7a77010"}, - {file = "mypy-1.7.0-py3-none-any.whl", hash = "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96"}, - {file = "mypy-1.7.0.tar.gz", hash = "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc"}, + {file = "mypy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340"}, + {file = "mypy-1.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49"}, + {file = "mypy-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5"}, + {file = "mypy-1.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d"}, + {file = "mypy-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4b901927f16224d0d143b925ce9a4e6b3a758010673eeded9b748f250cf4e8f7"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2f7f6985d05a4e3ce8255396df363046c28bea790e40617654e91ed580ca7c51"}, + {file = "mypy-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:944bdc21ebd620eafefc090cdf83158393ec2b1391578359776c00de00e8907a"}, + {file = "mypy-1.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9c7ac372232c928fff0645d85f273a726970c014749b924ce5710d7d89763a28"}, + {file = "mypy-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:f6efc9bd72258f89a3816e3a98c09d36f079c223aa345c659622f056b760ab42"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6dbdec441c60699288adf051f51a5d512b0d818526d1dcfff5a41f8cd8b4aaf1"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4fc3d14ee80cd22367caaaf6e014494415bf440980a3045bf5045b525680ac33"}, + {file = "mypy-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c6e4464ed5f01dc44dc9821caf67b60a4e5c3b04278286a85c067010653a0eb"}, + {file = "mypy-1.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:d9b338c19fa2412f76e17525c1b4f2c687a55b156320acb588df79f2e6fa9fea"}, + {file = "mypy-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:204e0d6de5fd2317394a4eff62065614c4892d5a4d1a7ee55b765d7a3d9e3f82"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:75c4d2a6effd015786c87774e04331b6da863fc3fc4e8adfc3b40aa55ab516fe"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2643d145af5292ee956aa0a83c2ce1038a3bdb26e033dadeb2f7066fb0c9abce"}, + {file = "mypy-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75aa828610b67462ffe3057d4d8a4112105ed211596b750b53cbfe182f44777a"}, + {file = "mypy-1.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ee5d62d28b854eb61889cde4e1dbc10fbaa5560cb39780c3995f6737f7e82120"}, + {file = "mypy-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:72cf32ce7dd3562373f78bd751f73c96cfb441de147cc2448a92c1a308bd0ca6"}, + {file = "mypy-1.7.1-py3-none-any.whl", hash = "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea"}, + {file = "mypy-1.7.1.tar.gz", hash = "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2"}, ] [[package]] @@ -2778,27 +2778,27 @@ files = [ [[package]] name = "ruff" -version = "0.1.5" +version = "0.1.6" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." files = [ - {file = "ruff-0.1.5-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:32d47fc69261c21a4c48916f16ca272bf2f273eb635d91c65d5cd548bf1f3d96"}, - {file = "ruff-0.1.5-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:171276c1df6c07fa0597fb946139ced1c2978f4f0b8254f201281729981f3c17"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ef33cd0bb7316ca65649fc748acc1406dfa4da96a3d0cde6d52f2e866c7b39"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2c205827b3f8c13b4a432e9585750b93fd907986fe1aec62b2a02cf4401eee6"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb408e3a2ad8f6881d0f2e7ad70cddb3ed9f200eb3517a91a245bbe27101d379"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f20dc5e5905ddb407060ca27267c7174f532375c08076d1a953cf7bb016f5a24"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aafb9d2b671ed934998e881e2c0f5845a4295e84e719359c71c39a5363cccc91"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4894dddb476597a0ba4473d72a23151b8b3b0b5f958f2cf4d3f1c572cdb7af7"}, - {file = "ruff-0.1.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a00a7ec893f665ed60008c70fe9eeb58d210e6b4d83ec6654a9904871f982a2a"}, - {file = "ruff-0.1.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a8c11206b47f283cbda399a654fd0178d7a389e631f19f51da15cbe631480c5b"}, - {file = "ruff-0.1.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fa29e67b3284b9a79b1a85ee66e293a94ac6b7bb068b307a8a373c3d343aa8ec"}, - {file = "ruff-0.1.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9b97fd6da44d6cceb188147b68db69a5741fbc736465b5cea3928fdac0bc1aeb"}, - {file = "ruff-0.1.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:721f4b9d3b4161df8dc9f09aa8562e39d14e55a4dbaa451a8e55bdc9590e20f4"}, - {file = "ruff-0.1.5-py3-none-win32.whl", hash = "sha256:f80c73bba6bc69e4fdc73b3991db0b546ce641bdcd5b07210b8ad6f64c79f1ab"}, - {file = "ruff-0.1.5-py3-none-win_amd64.whl", hash = "sha256:c21fe20ee7d76206d290a76271c1af7a5096bc4c73ab9383ed2ad35f852a0087"}, - {file = "ruff-0.1.5-py3-none-win_arm64.whl", hash = "sha256:82bfcb9927e88c1ed50f49ac6c9728dab3ea451212693fe40d08d314663e412f"}, - {file = "ruff-0.1.5.tar.gz", hash = "sha256:5cbec0ef2ae1748fb194f420fb03fb2c25c3258c86129af7172ff8f198f125ab"}, + {file = "ruff-0.1.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703"}, + {file = "ruff-0.1.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35"}, + {file = "ruff-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745"}, + {file = "ruff-0.1.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff"}, + {file = "ruff-0.1.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc"}, + {file = "ruff-0.1.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543"}, + {file = "ruff-0.1.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462"}, + {file = "ruff-0.1.6-py3-none-win32.whl", hash = "sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a"}, + {file = "ruff-0.1.6-py3-none-win_amd64.whl", hash = "sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33"}, + {file = "ruff-0.1.6-py3-none-win_arm64.whl", hash = "sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc"}, + {file = "ruff-0.1.6.tar.gz", hash = "sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184"}, ] [[package]] @@ -3035,12 +3035,12 @@ files = [ [[package]] name = "types-setuptools" -version = "68.2.0.1" +version = "68.2.0.2" requires_python = ">=3.7" summary = "Typing stubs for setuptools" files = [ - {file = "types-setuptools-68.2.0.1.tar.gz", hash = "sha256:8f31e8201e7969789e0eb23463b53ebe5f67d92417df4b648a6ea3c357ca4f51"}, - {file = "types_setuptools-68.2.0.1-py3-none-any.whl", hash = "sha256:e9c649559743e9f98c924bec91eae97f3ba208a70686182c3658fd7e81778d37"}, + {file = "types-setuptools-68.2.0.2.tar.gz", hash = "sha256:09efc380ad5c7f78e30bca1546f706469568cf26084cfab73ecf83dea1d28446"}, + {file = "types_setuptools-68.2.0.2-py3-none-any.whl", hash = "sha256:d5b5ff568ea2474eb573dcb783def7dadfd9b1ff638bb653b3c7051ce5aeb6d1"}, ] [[package]] diff --git a/src/bee_py/modules/bytes.py b/src/bee_py/modules/bytes.py index edd78f0..ebbcb73 100644 --- a/src/bee_py/modules/bytes.py +++ b/src/bee_py/modules/bytes.py @@ -30,7 +30,7 @@ def upload( """ headers = { - "content-type": "application/octet-stream", + "Content-Type": "application/octet-stream", **extract_upload_headers(postage_batch_id, options), } diff --git a/src/bee_py/modules/bzz.py b/src/bee_py/modules/bzz.py new file mode 100644 index 0000000..a6353d4 --- /dev/null +++ b/src/bee_py/modules/bzz.py @@ -0,0 +1,218 @@ +from typing import Optional, Union + +from bee_py.types.type import ( # Reference, UploadHeaders, Data, + BatchId, + BeeRequestOptions, + Collection, + CollectionUploadHeaders, + CollectionUploadOptions, + FileData, + FileHeaders, + FileUploadHeaders, + FileUploadOptions, + Reference, + ReferenceOrENS, + UploadResult, +) +from bee_py.utils.bytes import wrap_bytes_with_helpers +from bee_py.utils.collection import assert_collection +from bee_py.utils.headers import extract_upload_headers, read_file_headers +from bee_py.utils.http import http +from bee_py.utils.logging import logger +from bee_py.utils.tar import make_tar +from bee_py.utils.type import make_tag_uid + +BZZ_ENDPOINT = "bzz" + + +def extract_file_upload_headers( + postage_batch_id: BatchId, options: Optional[FileUploadOptions] = None +) -> FileUploadHeaders: + headers = extract_upload_headers(postage_batch_id, options) + + if options and options.size: + headers.content_length = str(options.size) + + if options and options.content_type: + headers.content_type = options.content_type + + return headers + + +def upload_file( + request_options: BeeRequestOptions, + data: Union[str, bytes], + postage_batch_id: BatchId, + name: Optional[str] = None, + options: Optional[FileUploadOptions] = None, +): + """ + Uploads a single file to the Bee node. + + Args: + request_options (BeeRequestOptions): Ky Options for making requests. + data (str | bytes | Readable | ArrayBuffer): File data. + postage_batch_id (BatchId): Postage Batch ID to be used for the upload. + name (str | None): Optional name that will be attached to the uploaded file. + options (FileUploadOptions | None): Optional file upload options, such as content length and content type. + + Returns: + UploadResult: The result of the upload operation. + """ + + if options.content_type: + options = options or {} + options.content_type = "application/octet-stream" + + headers = extract_file_upload_headers(postage_batch_id, options) + + config = { + "url": BZZ_ENDPOINT, + "method": "POST", + "data": data, + "headers": headers, + "params": {"name": name}, + } + response = http(request_options, config) + + if response.status_code != 201: # noqa: PLR2004 + logger.info(response.json()) + logger.error(response.raise_for_status()) + + upload_response = response.json() + reference = Reference(upload_response["reference"]) + tag_uid = None + + if "swarm-tag" in response.headers: + tag_uid = make_tag_uid(response.headers["swarm-tag"]) + + return UploadResult(reference=reference, tag_uid=tag_uid) + + +def download_file(request_options: BeeRequestOptions, _hash: ReferenceOrENS, path: str = "") -> FileData: + """ + Downloads a single file from a Bee node as a buffer. + + Args: + request_options (BeeRequestOptions): Ky Options for making requests. + _hash (ReferenceOrEns): Bee file or collection _hash. + path (str): Optional path to a single file within a collection. + + Returns: + FileData: Downloaded file data. + """ + + config = {"url": f"{BZZ_ENDPOINT}/{_hash}/{path}", "method": "GET"} + response = http(request_options, config) + + if response.status_code != 200: # noqa: PLR2004 + logger.info(response.json()) + logger.error(response.raise_for_status()) + + file_headers = FileHeaders.parse_obj(read_file_headers(response.headers)) + file_data = wrap_bytes_with_helpers(response.content).text() + + return FileData(headers=file_headers, data=file_data) + + +def download_file_readable(request_options: BeeRequestOptions, _hash: ReferenceOrENS, path: str = "") -> FileData: + """ + Downloads a single file from a Bee node as a readable stream. + + Args: + request_options (BeeRequestOptions): Ky Options for making requests. + _hash (ReferenceOrEns): Bee file or collection hash. + path (str): Optional path to a single file within a collection. + + Returns: + FileData[ReadableStream[Uint8Array]]: Downloaded file data. + """ + + config = {"url": f"{BZZ_ENDPOINT}/{_hash}/{path}", "method": "GET"} + response = http(request_options, config) + + if response.status_code != 200: # noqa: PLR2004 + logger.info(response.json()) + logger.error(response.raise_for_status()) + + file_headers = read_file_headers(response.headers) + file_data = response.data + + return FileData(file_headers, file_data) + + +def extract_collection_upload_headers( + postage_batch_id: BatchId, options: Optional[CollectionUploadOptions] = None +) -> CollectionUploadHeaders: + """ + Extracts headers for collection upload requests. + + Args: + postage_batch_id (BatchId): Postage Batch ID to be used for the upload. + options (CollectionUploadOptions | None): Optional collection upload options, + such as index document and error document. + + Returns: + CollectionUploadHeaders: Extracted collection upload headers. + """ + + headers = extract_upload_headers(postage_batch_id, options) + + if options and options.index_document: + headers.swarm_index_document = options.index_document + + if options and options.error_document: + headers.swarm_error_document = options.error_document + + return headers + + +def upload_collection( + request_options: BeeRequestOptions, + collection: Collection, + postage_batch_id: BatchId, + options: Optional[CollectionUploadOptions] = None, +) -> UploadResult: + """ + Uploads a collection of data to the Bee node. + + Args: + request_options (BeeRequestOptions): Ky Options for making requests. + collection (Collection[Uint8Array]): Collection of data to upload. + postage_batch_id (BatchId): Postage Batch ID to be used for the upload. + options (CollectionUploadOptions | None): Optional collection upload options, + such as index document and error document. + + Returns: + UploadResult: The result of the upload operation. + """ + + assert_collection(collection) + tar_data = make_tar(collection) + + headers = { + "Content-Type": "application/x-tar", + "swarm-collection": "true", + **extract_collection_upload_headers(postage_batch_id, options), + } + + config = { + "url": BZZ_ENDPOINT, + "method": "POST", + "data": tar_data, + "headers": headers, + } + response = http(request_options, config) + + if response.status_code != 201: # noqa: PLR2004 + logger.info(response.json()) + logger.error(response.raise_for_status()) + + upload_response = response.json() + reference = Reference(upload_response["reference"]) + tag_uid = None + + if "swarm-tag" in response.headers: + tag_uid = make_tag_uid(response.headers["swarm-tag"]) + + return UploadResult(reference=reference, tag_uid=tag_uid) diff --git a/src/bee_py/modules/soc.py b/src/bee_py/modules/soc.py index 6bdc381..2851375 100644 --- a/src/bee_py/modules/soc.py +++ b/src/bee_py/modules/soc.py @@ -43,7 +43,7 @@ def upload( "url": f"{SOC_ENDPOINT}/{owner}/{identifier}", "data": data, "headers": { - "content-type": "application/octet-stream", + "Content-Type": "application/octet-stream", **extract_upload_headers(postage_batch_id, options), }, "params": {"sig": signature}, diff --git a/src/bee_py/types/type.py b/src/bee_py/types/type.py index 5dea935..6e4190d 100644 --- a/src/bee_py/types/type.py +++ b/src/bee_py/types/type.py @@ -1,7 +1,7 @@ import json from abc import abstractmethod from enum import Enum -from typing import Annotated, Any, Callable, Generic, NewType, Optional, TypeVar, Union +from typing import Annotated, Any, Generic, NewType, Optional, TypeVar, Union from ape.managers.accounts import AccountAPI from ape.types import AddressType @@ -16,7 +16,7 @@ Type = TypeVar("Type") Name = TypeVar("Name") Length = TypeVar("Length", bound=int) -T = TypeVar("T", bound=Callable) +T: TypeAlias = str BeeRequestOptions = dict[str, Optional[Union[str, int, dict[str, str], PreparedRequest, Response]]] @@ -171,7 +171,7 @@ def length(self) -> Length: return self.__length -class PrefixedHexString(Generic[T]): +class PrefixedHexString: """ Type for HexString with prefix. @@ -277,18 +277,13 @@ def __init__( self.deferred = deferred -class FileHeaders: +class FileHeaders(BaseModel): """Represents the headers for a file.""" name: Optional[str] tag_uid: Optional[int] content_type: Optional[str] - def __init__(self, name: Optional[str] = None, tag_uid: Optional[int] = None, content_type: Optional[str] = None): - self.name = name - self.tagUid = tag_uid - self.contentType = content_type - class OverLayAddress: value: str @@ -725,3 +720,44 @@ class UploadOptions(BaseModel): encrypt: Optional[bool] = False tag: Optional[int] = None deferred: Optional[bool] = True + + +class FileUploadOptions(UploadOptions): + size: Optional[int] = None + content_type: Optional[str] = Field(None, alias="contentType") + + +class CollectionEntry(BaseModel): + data: T + path: str + + +class Collection(BaseModel): + entries: list[CollectionEntry] + + +class CollectionUploadOptions(UploadOptions): + index_document: Optional[str] = Field(None, alias="indexDocument") + error_document: Optional[str] = Field(None, alias="errorDocument") + + +class FileData(FileHeaders, BaseModel): + headers: FileHeaders + data: T + + +class UploadHeaders(BaseModel): + swarm_pin: Optional[str] = None + swarm_encrypt: Optional[str] = None + swarm_tag: Optional[str] = None + swarm_postage_batch_id: Optional[str] = None + + +class FileUploadHeaders(UploadHeaders): + content_length: Optional[str] = None + content_type: Optional[str] = None + + +class CollectionUploadHeaders(UploadHeaders): + swarm_index_document: Optional[str] = None + swarm_error_document: Optional[str] = None diff --git a/src/bee_py/utils/collection.py b/src/bee_py/utils/collection.py new file mode 100644 index 0000000..8d7a9de --- /dev/null +++ b/src/bee_py/utils/collection.py @@ -0,0 +1,19 @@ +# from bee_py.types.type import Collection +# from bee_py.utils.error import BeeArgumentError +from typing import Any + + +def is_collection(data: Any): + if not isinstance(data, list): + return False + + return all( + isinstance(entry, dict) and "data" in entry and "path" in entry and isinstance(entry["data"], bytes) + for entry in data + ) + + +def assert_collection(data: Any): + if not is_collection(data): + msg = "invalid collection" + raise ValueError(msg) diff --git a/src/bee_py/utils/headers.py b/src/bee_py/utils/headers.py index 3a6f3a1..91dfa64 100644 --- a/src/bee_py/utils/headers.py +++ b/src/bee_py/utils/headers.py @@ -1,11 +1,11 @@ import re -from typing import Optional, Union +from typing import Optional from bee_py.types.type import BatchId, FileHeaders, UploadOptions from bee_py.utils.error import BeeError -def read_content_disposition_filename(header: Union[str, None]) -> str: +def read_content_disposition_filename(header: Optional[str]) -> str: """Reads the filename from the content-disposition header. See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition @@ -24,15 +24,15 @@ def read_content_disposition_filename(header: Union[str, None]) -> str: # Regex was found here # https://stackoverflow.com/questions/23054475/javascript-regex-for-extracting-filename-from-content-disposition-header - disposition_match = re.match(r"filename[^;\n]*=(UTF-\d['\"]*)?((['\"])(.*?[.])\4|[^;\n]*)", header, re.I) + disposition_match = re.search(r"filename[^;\n]*=(UTF-\d['\"]*)?((['\"])(.*?[.])\4|[^;\n]*)", header, re.I) if disposition_match and len(disposition_match.groups()) > 0: - return disposition_match.group(1) + return disposition_match.group(0).split("=")[-1].strip('"') msg = "invalid content-disposition header" raise BeeError(msg) -def read_tag_uid(header: Union[str, None]) -> Union[int, None]: +def read_tag_uid(header: Optional[str] = None) -> Optional[int]: """Reads the tag UID from the header. Args: @@ -61,11 +61,11 @@ def read_file_headers(headers: dict[str, str]) -> FileHeaders: The file headers. """ - name = read_content_disposition_filename(headers.get("content-disposition")) + name = read_content_disposition_filename(headers.get("Content-Disposition")) tag_uid = read_tag_uid(headers.get("swarm-tag-uid")) - content_type = headers.get("content-type") + content_type = headers.get("Content-Type") - return FileHeaders(name, tag_uid, content_type) + return FileHeaders(name=name, tag_uid=tag_uid, content_type=content_type) def extract_upload_headers(postage_batch_id: BatchId, options: Optional[UploadOptions] = None) -> dict[str, str]: diff --git a/src/bee_py/utils/tar.py b/src/bee_py/utils/tar.py index fbe5799..5ef5c75 100644 --- a/src/bee_py/utils/tar.py +++ b/src/bee_py/utils/tar.py @@ -1,6 +1,9 @@ +import io import tarfile from typing import Protocol +from bee_py.types.type import Collection + class StringLike(Protocol): length: int @@ -19,7 +22,7 @@ def fix_unicode_path(path: str) -> StringLike: return StringLike(length=len(codes), char_code_at=lambda index: codes[index]) -def make_tar(data: list[tuple[str, bytes]]) -> bytes: +def make_tar(data: Collection) -> bytes: """Creates a tar archive from the given data. Args: @@ -31,12 +34,8 @@ def make_tar(data: list[tuple[str, bytes]]) -> bytes: A bytes object containing the tar archive. """ - tar = tarfile.open("w:gz") - - for _, entry in enumerate(data): - path, data = entry - tar.add(fix_unicode_path(path), data) - - tar.close() - - return tar.getvalue() + tar_io = io.BytesIO() + with tarfile.open(fileobj=tar_io, mode="w") as tar: + for entry in data: + tar.addfile(tarfile.TarInfo(name=entry["path"]), io.BytesIO(entry["data"])) + return tar_io.getvalue() diff --git a/tests/conftest.py b/tests/conftest.py index 0a75547..9706b59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import pytest from bee_py.modules.debug.connectivity import get_node_addresses +from bee_py.modules.debug.stamps import create_postage_batch, get_postage_batch from bee_py.types.type import BatchId @@ -60,20 +61,68 @@ def bee_debug_ky_options(bee_peer_debug_url) -> dict: @pytest.fixture -def get_postage_batch(request, url: str = "bee_debug_url") -> BatchId: +def read_local_bee_stamp() -> str: + with open(BEE_DATA_FILE) as f: + stamp = json.loads(f) + if stamp["BEE_POSTAGE"]: + return stamp["BEE_POSTAGE"] + return False + + +@pytest.fixture +def read_local_bee_peer_stamp() -> str: + with open(BEE_DATA_FILE) as f: + stamp = json.loads(f) + if stamp["BEE_PEER_POSTAGE"]: + return stamp["BEE_PEER_POSTAGE"] + return False + + +@pytest.fixture +def get_debug_postage(printer, read_local_bee_stamp, bee_debug_ky_options) -> BatchId: stamp: BatchId - if url == "bee_debug_url": - stamp = request.getfixturevalue("read_bee_postage")["BEE_POSTAGE"] - elif url == "bee_peer_debug_url": - stamp = request.getfixturevalue("read_bee_postage")["BEE_PEER_POSTAGE"] - else: - msg = f"Unknown url: {url}" + printer("[*]Getting Debug Postage....") + + if read_local_bee_stamp: + printer(read_local_bee_stamp) + return read_local_bee_stamp + + stamp = create_postage_batch(bee_debug_ky_options, 100, 20) + + if not stamp: + msg = "There is no valid postage stamp" raise ValueError(msg) + printer("[*]Waiting for postage to be usable....") + while True: + usable = get_postage_batch(bee_debug_ky_options, stamp).usable + if usable: + break + printer(f"[*]Valid Postage found: {stamp}") + return stamp + + +@pytest.fixture +def get_peer_debug_postage(printer, read_local_bee_peer_stamp, bee_peer_debug_ky_options) -> BatchId: + stamp: BatchId + + if read_local_bee_peer_stamp: + return read_local_bee_peer_stamp + + printer("[*]Getting Debug Postage....") + stamp = create_postage_batch(bee_peer_debug_ky_options, 100, 20) + if not stamp: - msg = f"There is no postage stamp configured for URL: {url}" + msg = "There is no valid postage stamp" raise ValueError(msg) + + printer("[*]Waiting for postage to be usable....") + while True: + usable = get_postage_batch(bee_peer_debug_ky_options, stamp).usable + if usable: + break + printer(f"[*]Valid Postage found: {stamp}") return stamp @@ -97,3 +146,6 @@ def peer_overlay(bee_peer_debug_ky_options) -> str: node_addresses = get_node_addresses(bee_peer_debug_ky_options) return node_addresses.overlay + + +BIG_FILE_TIMEOUT = 100_000 diff --git a/tests/data/bee_data.json b/tests/data/bee_data.json index 3009afc..8764cc2 100644 --- a/tests/data/bee_data.json +++ b/tests/data/bee_data.json @@ -1,4 +1,4 @@ { - "BEE_POSTAGE": "fd9d9fbd6d1a65db4b40e1c410aeeadd1db227e51fc5af6da01e65eb1de2dd61", - "BEE_PEER_POSTAGE": "9271b024d7ccdec82ee696c53c4be7f63818dc404aa7df14300cf10742254d85" + "BEE_POSTAGE": "061dfc1fc656410f1901e06f5b413039a6923e2747ca7d8cfe23391cd7b86c20", + "BEE_PEER_POSTAGE": "" } \ No newline at end of file diff --git a/tests/integration/modules/test_bytes.py b/tests/integration/modules/test_bytes.py index 65eb4ff..a40ec15 100644 --- a/tests/integration/modules/test_bytes.py +++ b/tests/integration/modules/test_bytes.py @@ -1,7 +1,7 @@ import pytest import requests -from bee_py.modules.bytes import download, download_readable, upload +from bee_py.modules.bytes import download, upload def test_store_and_retrieve_data(bee_ky_options, get_debug_postage): diff --git a/tests/integration/modules/test_bzz.py b/tests/integration/modules/test_bzz.py new file mode 100644 index 0000000..59c1e2b --- /dev/null +++ b/tests/integration/modules/test_bzz.py @@ -0,0 +1,13 @@ +from bee_py.modules.bzz import download_file, extract_file_upload_headers, upload_collection +from bee_py.modules.tag import create_tag + + +def test_store_and_retrieve_collection_with_single_file(bee_ky_options): + directory_structure = [{"path": "0", "data": bytes([0])}] + + get_debug_postage = "061dfc1fc656410f1901e06f5b413039a6923e2747ca7d8cfe23391cd7b86c20" + result = upload_collection(bee_ky_options, directory_structure, get_debug_postage) + file = download_file(bee_ky_options, result.reference, directory_structure[0]["path"]) + + assert file.headers.name == directory_structure[0]["path"] + assert file.data.encode() == directory_structure[0]["data"]