From 64d39cb92057dfe9127a129a630e4e60e66ca96d Mon Sep 17 00:00:00 2001 From: Erick Corona Date: Thu, 6 Jun 2024 06:39:40 -0600 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Source=20Shopify:=20add=20resilienc?= =?UTF-8?q?y=20on=20some=20transient=20errors=20using=20the=20HttpClient?= =?UTF-8?q?=20(#38084)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Oleksandr Bazarnov Co-authored-by: maxi297 Co-authored-by: Maxime Carbonneau-Leclerc <3360483+maxi297@users.noreply.github.com> --- .../connectors/source-shopify/metadata.yaml | 2 +- .../connectors/source-shopify/poetry.lock | 340 +++++++++++++++++- .../connectors/source-shopify/pyproject.toml | 5 +- .../source_shopify/http_request.py | 49 +++ .../source-shopify/source_shopify/scopes.py | 21 +- .../shopify_graphql/bulk/exceptions.py | 9 +- .../shopify_graphql/bulk/job.py | 168 ++++----- .../shopify_graphql/bulk/retry.py | 11 + .../source_shopify/streams/base_streams.py | 50 +-- .../source-shopify/source_shopify/utils.py | 2 +- .../source-shopify/unit_tests/conftest.py | 7 + .../unit_tests/graphql_bulk/test_job.py | 72 ++-- .../unit_tests/integration/__init__.py | 0 .../unit_tests/integration/api/__init__.py | 0 .../integration/api/authentication.py | 25 ++ .../unit_tests/integration/api/bulk.py | 177 +++++++++ .../integration/test_bulk_stream.py | 185 ++++++++++ .../resource/http/response/shop.json | 58 +++ docs/integrations/sources/shopify.md | 1 + 19 files changed, 976 insertions(+), 206 deletions(-) create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/integration/__init__.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/__init__.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/resource/http/response/shop.json diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml index 6e3f4c8b84ab..220f5b685686 100644 --- a/airbyte-integrations/connectors/source-shopify/metadata.yaml +++ b/airbyte-integrations/connectors/source-shopify/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 - dockerImageTag: 2.2.2 + dockerImageTag: 2.2.3 dockerRepository: airbyte/source-shopify documentationUrl: https://docs.airbyte.com/integrations/sources/shopify githubIssueLabel: source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/poetry.lock b/airbyte-integrations/connectors/source-shopify/poetry.lock index df34541cacf0..d226410969f8 100644 --- a/airbyte-integrations/connectors/source-shopify/poetry.lock +++ b/airbyte-integrations/connectors/source-shopify/poetry.lock @@ -1,20 +1,21 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.81.4" +version = "0.90.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.81.4-py3-none-any.whl", hash = "sha256:4ed193da4e8be4867e1d8983172d10afb3c3b10f3e10ec618431deec1f2af4cb"}, - {file = "airbyte_cdk-0.81.4.tar.gz", hash = "sha256:5c63d8c792edf5f24d0ad804b34b3ebcc056ecede6cb4f87ebf9ac07aa987f24"}, + {file = "airbyte_cdk-0.90.0-py3-none-any.whl", hash = "sha256:bd0aa5843cdc4901f2e482f0e86695ca4e6db83b65c5017799255dd20535cf56"}, + {file = "airbyte_cdk-0.90.0.tar.gz", hash = "sha256:25cefc010718bada5cce3f87e7ae93068630732c0d34ce5145f8ddf7457d4d3c"}, ] [package.dependencies] -airbyte-protocol-models = "*" +airbyte-protocol-models = ">=0.9.0,<1.0" backoff = "*" cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" Deprecated = ">=1.2,<1.3" dpath = ">=2.0.1,<2.1.0" genson = "1.2.2" @@ -22,10 +23,13 @@ isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyjwt = ">=2.8.0,<3.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" +pytz = "2024.1" PyYAML = ">=6.0.1,<7.0.0" requests = "*" requests_cache = "*" @@ -34,7 +38,7 @@ wcmatch = "8.4" [package.extras] file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] -vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" @@ -138,6 +142,70 @@ files = [ {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -248,6 +316,60 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "42.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, + {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, + {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, + {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, + {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, + {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, + {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "deprecated" version = "1.2.14" @@ -290,6 +412,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "freezegun" +version = "1.5.1" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.5.1-py3-none-any.whl", hash = "sha256:bf111d7138a8abe55ab48a71755673dbaa4ab87f4cff5634a4442dfec34c15f1"}, + {file = "freezegun-1.5.1.tar.gz", hash = "sha256:b29dedfcda6d5e8e083ce71b2b542753ad48cfec44037b3fc79702e2980a89e9"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "genson" version = "1.2.2" @@ -384,6 +520,31 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "2.4" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, +] + [[package]] name = "jsonref" version = "0.2" @@ -416,6 +577,44 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.63" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.63-py3-none-any.whl", hash = "sha256:7810afdf5e3f3b472fc581a29371fb96cd843dde2149e048d1b9610325159d1e"}, + {file = "langsmith-0.1.63.tar.gz", hash = "sha256:a609405b52f6f54df442a142cbf19ab38662d54e532f96028b4c546434d4afdf"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = ">=1,<3" +requests = ">=2,<3" + [[package]] name = "markupsafe" version = "2.1.5" @@ -485,15 +684,70 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "orjson" +version = "3.10.3" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.3-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9fb6c3f9f5490a3eb4ddd46fc1b6eadb0d6fc16fb3f07320149c3286a1409dd8"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:252124b198662eee80428f1af8c63f7ff077c88723fe206a25df8dc57a57b1fa"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f3e87733823089a338ef9bbf363ef4de45e5c599a9bf50a7a9b82e86d0228da"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8334c0d87103bb9fbbe59b78129f1f40d1d1e8355bbed2ca71853af15fa4ed3"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1952c03439e4dce23482ac846e7961f9d4ec62086eb98ae76d97bd41d72644d7"}, + {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0403ed9c706dcd2809f1600ed18f4aae50be263bd7112e54b50e2c2bc3ebd6d"}, + {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:382e52aa4270a037d41f325e7d1dfa395b7de0c367800b6f337d8157367bf3a7"}, + {file = "orjson-3.10.3-cp310-none-win32.whl", hash = "sha256:be2aab54313752c04f2cbaab4515291ef5af8c2256ce22abc007f89f42f49109"}, + {file = "orjson-3.10.3-cp310-none-win_amd64.whl", hash = "sha256:416b195f78ae461601893f482287cee1e3059ec49b4f99479aedf22a20b1098b"}, + {file = "orjson-3.10.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:73100d9abbbe730331f2242c1fc0bcb46a3ea3b4ae3348847e5a141265479700"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a12eee96e3ab828dbfcb4d5a0023aa971b27143a1d35dc214c176fdfb29b3"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:520de5e2ef0b4ae546bea25129d6c7c74edb43fc6cf5213f511a927f2b28148b"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccaa0a401fc02e8828a5bedfd80f8cd389d24f65e5ca3954d72c6582495b4bcf"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7bc9e8bc11bac40f905640acd41cbeaa87209e7e1f57ade386da658092dc16"}, + {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3582b34b70543a1ed6944aca75e219e1192661a63da4d039d088a09c67543b08"}, + {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c23dfa91481de880890d17aa7b91d586a4746a4c2aa9a145bebdbaf233768d5"}, + {file = "orjson-3.10.3-cp311-none-win32.whl", hash = "sha256:1770e2a0eae728b050705206d84eda8b074b65ee835e7f85c919f5705b006c9b"}, + {file = "orjson-3.10.3-cp311-none-win_amd64.whl", hash = "sha256:93433b3c1f852660eb5abdc1f4dd0ced2be031ba30900433223b28ee0140cde5"}, + {file = "orjson-3.10.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a39aa73e53bec8d410875683bfa3a8edf61e5a1c7bb4014f65f81d36467ea098"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0943a96b3fa09bee1afdfccc2cb236c9c64715afa375b2af296c73d91c23eab2"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e852baafceff8da3c9defae29414cc8513a1586ad93e45f27b89a639c68e8176"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18566beb5acd76f3769c1d1a7ec06cdb81edc4d55d2765fb677e3eaa10fa99e0"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bd2218d5a3aa43060efe649ec564ebedec8ce6ae0a43654b81376216d5ebd42"}, + {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cf20465e74c6e17a104ecf01bf8cd3b7b252565b4ccee4548f18b012ff2f8069"}, + {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba7f67aa7f983c4345eeda16054a4677289011a478ca947cd69c0a86ea45e534"}, + {file = "orjson-3.10.3-cp312-none-win32.whl", hash = "sha256:17e0713fc159abc261eea0f4feda611d32eabc35708b74bef6ad44f6c78d5ea0"}, + {file = "orjson-3.10.3-cp312-none-win_amd64.whl", hash = "sha256:4c895383b1ec42b017dd2c75ae8a5b862fc489006afde06f14afbdd0309b2af0"}, + {file = "orjson-3.10.3-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:be2719e5041e9fb76c8c2c06b9600fe8e8584e6980061ff88dcbc2691a16d20d"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0175a5798bdc878956099f5c54b9837cb62cfbf5d0b86ba6d77e43861bcec2"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:978be58a68ade24f1af7758626806e13cff7748a677faf95fbb298359aa1e20d"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16bda83b5c61586f6f788333d3cf3ed19015e3b9019188c56983b5a299210eb5"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ad1f26bea425041e0a1adad34630c4825a9e3adec49079b1fb6ac8d36f8b754"}, + {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9e253498bee561fe85d6325ba55ff2ff08fb5e7184cd6a4d7754133bd19c9195"}, + {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a62f9968bab8a676a164263e485f30a0b748255ee2f4ae49a0224be95f4532b"}, + {file = "orjson-3.10.3-cp38-none-win32.whl", hash = "sha256:8d0b84403d287d4bfa9bf7d1dc298d5c1c5d9f444f3737929a66f2fe4fb8f134"}, + {file = "orjson-3.10.3-cp38-none-win_amd64.whl", hash = "sha256:8bc7a4df90da5d535e18157220d7915780d07198b54f4de0110eca6b6c11e290"}, + {file = "orjson-3.10.3-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9059d15c30e675a58fdcd6f95465c1522b8426e092de9fff20edebfdc15e1cb0"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d40c7f7938c9c2b934b297412c067936d0b54e4b8ab916fd1a9eb8f54c02294"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4a654ec1de8fdaae1d80d55cee65893cb06494e124681ab335218be6a0691e7"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:831c6ef73f9aa53c5f40ae8f949ff7681b38eaddb6904aab89dca4d85099cb78"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99b880d7e34542db89f48d14ddecbd26f06838b12427d5a25d71baceb5ba119d"}, + {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e5e176c994ce4bd434d7aafb9ecc893c15f347d3d2bbd8e7ce0b63071c52e25"}, + {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b69a58a37dab856491bf2d3bbf259775fdce262b727f96aafbda359cb1d114d8"}, + {file = "orjson-3.10.3-cp39-none-win32.whl", hash = "sha256:b8d4d1a6868cde356f1402c8faeb50d62cee765a1f7ffcfd6de732ab0581e063"}, + {file = "orjson-3.10.3-cp39-none-win_amd64.whl", hash = "sha256:5102f50c5fc46d94f2033fe00d392588564378260d64377aec702f21a7a22912"}, + {file = "orjson-3.10.3.tar.gz", hash = "sha256:2b166507acae7ba2f7c315dcf185a9111ad5e992ac81f2d507aac39193c2c818"}, +] + [[package]] name = "packaging" -version = "24.0" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -561,6 +815,17 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "1.10.15" @@ -613,6 +878,23 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pyrate-limiter" version = "3.1.1" @@ -722,6 +1004,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "pytzdata" version = "2020.1" @@ -795,13 +1088,13 @@ files = [ [[package]] name = "requests" -version = "2.32.1" +version = "2.32.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.1-py3-none-any.whl", hash = "sha256:21ac9465cdf8c1650fe1ecde8a71669a93d4e6f147550483a2967d08396a56a5"}, - {file = "requests-2.32.1.tar.gz", hash = "sha256:eb97e87e64c79e64e5b8ac75cee9dd1f97f49e289b083ee6be96268930725685"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -905,6 +1198,21 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tenacity" +version = "8.3.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, + {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" version = "2.0.1" @@ -918,13 +1226,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, ] [[package]] @@ -1054,4 +1362,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "a88ba9d29c8cc1a7dd520d152b96c4b43d36bbecafb1a276ef9965650ccc7b2b" +content-hash = "702ede56bf9c39032986e5709136f2c66da780844d77c54b154001d7244a16c0" diff --git a/airbyte-integrations/connectors/source-shopify/pyproject.toml b/airbyte-integrations/connectors/source-shopify/pyproject.toml index 45aeed8dcbf8..8cc085ed3180 100644 --- a/airbyte-integrations/connectors/source-shopify/pyproject.toml +++ b/airbyte-integrations/connectors/source-shopify/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.2.2" +version = "2.2.3" name = "source-shopify" description = "Source CDK implementation for Shopify." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_shopify" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "0.81.4" +airbyte-cdk = "0.90.0" sgqlc = "==16.3" graphql-query = "^1.1.1" @@ -28,3 +28,4 @@ source-shopify = "source_shopify.run:run" requests-mock = "^1.11.0" pytest-mock = "^3.12.0" pytest = "^8.0.0" +freezegun = "^1.4.0" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py new file mode 100644 index 000000000000..a3e9c7318987 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py @@ -0,0 +1,49 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from typing import Optional, Union + +import requests +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction +from airbyte_protocol.models import FailureType +from requests import exceptions + +RESPONSE_CONSUMPTION_EXCEPTIONS = ( + exceptions.ChunkedEncodingError, + exceptions.JSONDecodeError, +) + +TRANSIENT_EXCEPTIONS = ( + exceptions.ConnectTimeout, + exceptions.ConnectionError, + exceptions.HTTPError, + exceptions.ReadTimeout, + # This error was added as part of the migration from REST to bulk (https://github.com/airbytehq/airbyte/commit/f5094041bebb80cd6602a98829c19a7515276ed3) but it is unclear in which case it occurs and why it is transient + exceptions.SSLError, +) + RESPONSE_CONSUMPTION_EXCEPTIONS + +_NO_ERROR_RESOLUTION = ErrorResolution(ResponseAction.SUCCESS, None, None) + + +class ShopifyErrorHandler(ErrorHandler): + def __init__(self, stream_name: str = "") -> None: + self._stream_name = stream_name + + def interpret_response(self, response: Optional[Union[requests.Response, Exception]]) -> ErrorResolution: + if isinstance(response, TRANSIENT_EXCEPTIONS): + return ErrorResolution( + ResponseAction.RETRY, + FailureType.transient_error, + f"Error of type {type(response)} is considered transient. Try again later. (full error message is {response})", + ) + elif isinstance(response, requests.Response): + if response.ok: + return _NO_ERROR_RESOLUTION + + if response.status_code == 429 or response.status_code >= 500: + return ErrorResolution( + ResponseAction.RETRY, + FailureType.transient_error, + f"Status code `{response.status_code}` is considered transient. Try again later. (full error message is {response.content})", + ) + + return _NO_ERROR_RESOLUTION # Not all the error handling is defined here so it assumes the previous code will handle the error if there is one diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py index 805acc1f8d68..00fd6b6d7c2f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py @@ -7,8 +7,10 @@ from typing import Any, Iterable, List, Mapping, Optional import requests +from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import ConnectionError, InvalidURL, JSONDecodeError, SSLError +from .http_request import ShopifyErrorHandler from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyConnectionError, ShopifyWrongShopNameError SCOPES_MAPPING: Mapping[str, set[str]] = { @@ -83,35 +85,32 @@ class ShopifyScopes: logger = logging.getLogger("airbyte") def __init__(self, config: Mapping[str, Any]) -> None: + self.permitted_streams: List[str] = list(ALWAYS_PERMITTED_STREAMS) + self.not_permitted_streams: List[set[str, str]] = [] + self._error_handler = ShopifyErrorHandler() + self._http_client = HttpClient("ShopifyScopes", self.logger, self._error_handler, session=requests.Session()) + self.user_scopes = self.get_user_scopes(config) # for each stream check the authenticated user has all scopes required self.get_streams_from_user_scopes() # log if there are streams missing scopes and should be omitted self.emit_missing_scopes() - # the list of validated streams - permitted_streams: List[str] = ALWAYS_PERMITTED_STREAMS - # the list of not permitted streams - not_permitted_streams: List[set[str, str]] = [] # template for the log message missing_scope_message: str = ( "The stream `{stream}` could not be synced without the `{scope}` scope. Please check the `{scope}` is granted." ) - @staticmethod - def get_user_scopes(config) -> list[Any]: - session = requests.Session() + def get_user_scopes(self, config) -> list[Any]: url = f"https://{config['shop']}.myshopify.com/admin/oauth/access_scopes.json" headers = config["authenticator"].get_auth_header() try: - response = session.get(url, headers=headers).json() - access_scopes = [scope.get("handle") for scope in response.get("access_scopes")] + _, response = self._http_client.send_request("GET", url, headers=headers, request_kwargs={}) + access_scopes = [scope.get("handle") for scope in response.json().get("access_scopes")] except InvalidURL: raise ShopifyWrongShopNameError(url) except JSONDecodeError as json_error: raise ShopifyBadJsonError(json_error) - except (SSLError, ConnectionError) as con_error: - raise ShopifyConnectionError(con_error) if access_scopes: return access_scopes diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py index 1177d0fbdcf1..3dcc00d14e52 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py @@ -47,5 +47,12 @@ class BulkJobTimout(BaseBulkException): class BulkJobAccessDenied(BaseBulkException): """Raised when BULK Job has ACCESS_DENIED status""" + class BulkJobCreationFailedConcurrentError(BaseBulkException): + """Raised when an attempt to create a job as failed because of concurrency limits.""" + + failure_type: FailureType = FailureType.transient_error + class BulkJobConcurrentError(BaseBulkException): - """Raised when BULK Job could not be created, since the 1 Bulk job / shop quota is exceeded.""" + """Raised when failing the job after hitting too many BulkJobCreationFailedConcurrentError.""" + + failure_type: FailureType = FailureType.transient_error diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py index ad6fd10a82f9..2414dbe6481a 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py @@ -10,12 +10,14 @@ import pendulum as pdm import requests +from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import JSONDecodeError from source_shopify.utils import ApiTypeEnum from source_shopify.utils import ShopifyRateLimiter as limiter +from ...http_request import ShopifyErrorHandler from .exceptions import AirbyteTracedException, ShopifyBulkExceptions -from .query import ShopifyBulkTemplates +from .query import ShopifyBulkQuery, ShopifyBulkTemplates from .retry import bulk_retry_on_exception from .status import ShopifyBulkJobStatus from .tools import END_OF_FILE, BulkTools @@ -26,6 +28,7 @@ class ShopifyBulkManager: session: requests.Session base_url: str stream_name: str + query: ShopifyBulkQuery # default logger logger: Final[logging.Logger] = logging.getLogger("airbyte") @@ -34,8 +37,6 @@ class ShopifyBulkManager: _retrieve_chunk_size: Final[int] = 1024 * 1024 * 10 _job_max_retries: Final[int] = 6 _job_backoff_time: int = 5 - # saved latest request - _request: Optional[requests.Request] = None # running job logger constrain, every 100-ish message will be printed _log_job_msg_frequency: Final[int] = 100 @@ -51,7 +52,7 @@ class ShopifyBulkManager: # currents: _job_id, _job_state, _job_created_at, _job_self_canceled _job_id: Optional[str] = field(init=False, default=None) - _job_state: ShopifyBulkJobStatus = field(init=False, default=None) + _job_state: str = field(init=False, default=None) # this string is based on ShopifyBulkJobStatus # completed and saved Bulk Job result filename _job_result_filename: Optional[str] = field(init=False, default=None) # date-time when the Bulk Job was created on the server @@ -81,6 +82,9 @@ class ShopifyBulkManager: # 2 sec is set as default value to cover the case with the empty-fast-completed jobs _job_last_elapsed_time: float = field(init=False, default=2.0) + def __post_init__(self): + self._http_client = HttpClient(self.stream_name, self.logger, ShopifyErrorHandler(), session=self.session) + @property def _tools(self) -> BulkTools: return BulkTools() @@ -142,9 +146,6 @@ def _expand_job_size(self) -> None: def _reduce_job_size(self) -> None: self.job_size /= self._job_size_adjusted_reduce_factor - def _save_latest_request(self, response: requests.Response) -> None: - self._request = response.request - def _job_size_reduce_next(self) -> None: # revert the flag self._job_should_revert_slice = False @@ -180,14 +181,17 @@ def _job_canceled(self) -> bool: return self._job_state == ShopifyBulkJobStatus.CANCELED.value def _job_cancel(self) -> None: - # re-use of `self._session(*, **)` to make BULK Job cancel request - cancel_args = self._job_get_request_args(ShopifyBulkTemplates.cancel) - with self.session as cancel_job: - canceled_response = cancel_job.request(**cancel_args) - # mark the job was self-canceled - self._job_self_canceled = True - # check CANCELED Job health - self._job_healthcheck(canceled_response) + _, canceled_response = self._http_client.send_request( + http_method="POST", + url=self.base_url, + data=ShopifyBulkTemplates.cancel(self._job_id), + headers={"Content-Type": "application/graphql"}, + request_kwargs={}, + ) + # mark the job was self-canceled + self._job_self_canceled = True + # check CANCELED Job health + self._job_healthcheck(canceled_response) # sleep to ensure the cancelation sleep(self._job_check_interval) @@ -209,27 +213,19 @@ def _log_state(self, message: Optional[str] = None) -> None: else: self.logger.info(pattern) - def _job_get_request_args(self, template: ShopifyBulkTemplates) -> Mapping[str, Any]: - return { - "method": "POST", - "url": self.base_url, - "data": template(self._job_id), - "headers": {"Content-Type": "application/graphql"}, - } - def _job_get_result(self, response: Optional[requests.Response] = None) -> Optional[str]: parsed_response = response.json().get("data", {}).get("node", {}) if response else None job_result_url = parsed_response.get("url") if parsed_response and not self._job_self_canceled else None if job_result_url: # save to local file using chunks to avoid OOM filename = self._tools.filename_from_url(job_result_url) - with self.session.get(job_result_url, stream=True) as response: - response.raise_for_status() - with open(filename, "wb") as file: - for chunk in response.iter_content(chunk_size=self._retrieve_chunk_size): - file.write(chunk) - # add `` line to the bottom of the saved data for easy parsing - file.write(END_OF_FILE.encode()) + _, response = self._http_client.send_request(http_method="GET", url=job_result_url, request_kwargs={"stream": True}) + response.raise_for_status() + with open(filename, "wb") as file: + for chunk in response.iter_content(chunk_size=self._retrieve_chunk_size): + file.write(chunk) + # add `` line to the bottom of the saved data for easy parsing + file.write(END_OF_FILE.encode()) return filename def _job_update_state(self, response: Optional[requests.Response] = None) -> None: @@ -298,45 +294,27 @@ def _collect_bulk_errors(self, response: requests.Response) -> List[Optional[dic ) def _job_healthcheck(self, response: requests.Response) -> Optional[Exception]: - try: - # save the latest request to retry - self._save_latest_request(response) - - # get the errors, if occured - errors = self._collect_bulk_errors(response) + errors = self._collect_bulk_errors(response) - # when the concurrent job takes place, - # another job could not be created - # we typically need to wait and retry, but no longer than 10 min. - if self._has_running_concurrent_job(errors): - return self._job_retry_on_concurrency() - - # when the job was already created and the error appears in the middle - if self._job_state and errors: - self._on_job_with_errors(errors) - - # when the job was not created because of some errors - if not self._job_state and errors: - self._on_non_handable_job_error(errors) - - except (ShopifyBulkExceptions.BulkJobBadResponse, ShopifyBulkExceptions.BulkJobError) as e: - raise e - - def _job_send_state_request(self) -> requests.Response: - with self.session as job_state_request: - status_args = self._job_get_request_args(ShopifyBulkTemplates.status) - self._request = requests.Request(**status_args, auth=self.session.auth).prepare() - return job_state_request.send(self._request) + if self._job_state and errors: + self._on_job_with_errors(errors) def _job_track_running(self) -> None: - job_state_response = self._job_send_state_request() - self._job_healthcheck(job_state_response) - self._job_update_state(job_state_response) - self._job_state_to_fn_map.get(self._job_state)(response=job_state_response) + _, response = self._http_client.send_request( + http_method="POST", + url=self.base_url, + data=ShopifyBulkTemplates.status(self._job_id), + headers={"Content-Type": "application/graphql"}, + request_kwargs={}, + ) + self._job_healthcheck(response) + + self._job_update_state(response) + self._job_state_to_fn_map.get(self._job_state)(response=response) def _has_running_concurrent_job(self, errors: Optional[Iterable[Mapping[str, Any]]] = None) -> bool: """ - When concurent BULK Job is already running for the same SHOP we receive: + When concurrent BULK Job is already running for the same SHOP we receive: Error example: [ { @@ -346,63 +324,63 @@ def _has_running_concurrent_job(self, errors: Optional[Iterable[Mapping[str, Any ] """ - concurent_job_pattern = "A bulk query operation for this app and shop is already in progress" + concurrent_job_pattern = "A bulk query operation for this app and shop is already in progress" # the errors are handled in `job_job_check_for_errors` if errors: for error in errors: message = error.get("message", "") if isinstance(error, dict) else "" - if concurent_job_pattern in message: + if concurrent_job_pattern in message: return True - # reset the `_concurrent_attempt` counter, once there is no concurrent job error - self._concurrent_attempt = 0 return False def _has_reached_max_concurrency(self) -> bool: return self._concurrent_attempt == self._concurrent_max_retry - def _job_retry_request(self) -> Optional[requests.Response]: - with self.session.send(self._request) as retried_request: - return retried_request - - def _job_retry_concurrent(self) -> Optional[requests.Response]: - self._concurrent_attempt += 1 - self.logger.warning( - f"Stream: `{self.stream_name}`, the BULK concurrency limit has reached. Waiting {self._concurrent_interval} sec before retry, atttempt: {self._concurrent_attempt}.", - ) - sleep(self._concurrent_interval) - retried_response = self._job_retry_request() - return self.job_process_created(retried_response) - - def _job_retry_on_concurrency(self) -> Optional[requests.Response]: - if self._has_reached_max_concurrency(): - # indicate we're out of attempts to retry with job creation - message = f"The BULK Job couldn't be created at this time, since another job is running." - self.logger.error(message) - # raise AibyteTracebackException with `INCOMPLETE` status - raise ShopifyBulkExceptions.BulkJobConcurrentError(message) - else: - return self._job_retry_concurrent() - @bulk_retry_on_exception(logger) - def _job_check_state(self) -> Optional[str]: + def _job_check_state(self) -> None: while not self._job_completed(): if self._job_canceled(): break else: self._job_track_running() - # external method to be used within other components - @bulk_retry_on_exception(logger) - def job_process_created(self, response: requests.Response) -> None: + def create_job(self, stream_slice: Mapping[str, str], filter_field: str) -> None: + if stream_slice: + query = self.query.get(filter_field, stream_slice["start"], stream_slice["end"]) + else: + query = self.query.get() + + _, response = self._http_client.send_request( + http_method="POST", + url=self.base_url, + json={"query": ShopifyBulkTemplates.prepare(query)}, + request_kwargs={}, + ) + + errors = self._collect_bulk_errors(response) + if self._has_running_concurrent_job(errors): + # when the concurrent job takes place, another job could not be created + # we typically need to wait and retry, but no longer than 10 min. (see retry in `bulk_retry_on_exception`) + raise ShopifyBulkExceptions.BulkJobCreationFailedConcurrentError(f"Failed to create job for stream {self.stream_name}") + else: + # There were no concurrent error for this job so even if there were other errors, we can reset this + self._concurrent_attempt = 0 + + if errors: + self._on_non_handable_job_error(errors) + + self._job_process_created(response) + + def _job_process_created(self, response: requests.Response) -> None: """ The Bulk Job with CREATED status, should be processed, before we move forward with Job Status Checks. """ - self._job_healthcheck(response) bulk_response = response.json().get("data", {}).get("bulkOperationRunQuery", {}).get("bulkOperation", {}) if response else None if bulk_response and bulk_response.get("status") == ShopifyBulkJobStatus.CREATED.value: self._job_id = bulk_response.get("id") self._job_created_at = bulk_response.get("createdAt") + self._job_state = ShopifyBulkJobStatus.CREATED.value self.logger.info(f"Stream: `{self.stream_name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}") def job_size_normalize(self, start: datetime, end: datetime) -> datetime: diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py index 12d5d4d651cd..140d77e91ad5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py @@ -43,6 +43,17 @@ def wrapper(self, *args, **kwargs) -> Any: f"Stream `{stream_name}`: {ex}. Retrying {current_retries}/{max_retries} after {backoff_time} seconds." ) sleep(backoff_time) + except ShopifyBulkExceptions.BulkJobCreationFailedConcurrentError: + if self._concurrent_attempt == self._concurrent_max_retry: + message = f"The BULK Job couldn't be created at this time, since another job is running." + logger.error(message) + raise ShopifyBulkExceptions.BulkJobConcurrentError(message) + + self._concurrent_attempt += 1 + logger.warning( + f"Stream: `{self.stream_name}`, the BULK concurrency limit has reached. Waiting {self._concurrent_interval} sec before retry, attempt: {self._concurrent_attempt}.", + ) + sleep(self._concurrent_interval) return wrapper diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py index bc084f3c482b..0a10a1714e03 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py @@ -7,12 +7,14 @@ from abc import ABC, abstractmethod from datetime import datetime from functools import cached_property -from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Union +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl, urlparse import pendulum as pdm import requests +from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_protocol.models import SyncMode from requests.exceptions import RequestException from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager from source_shopify.shopify_graphql.bulk.query import ShopifyBulkQuery, ShopifyBulkTemplates @@ -631,7 +633,6 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream): filter_field = "updated_at" cursor_field = "updated_at" data_field = "graphql" - http_method = "POST" parent_stream_class: Optional[Union[ShopifyStream, IncrementalShopifyStream]] = None @@ -644,6 +645,7 @@ def __init__(self, config: Dict) -> None: session=self._session, base_url=f"{self.url_base}{self.path()}", stream_name=self.name, + query=self.query, ) # overide the default job slice size, if provided (it's auto-adjusted, later on) self.bulk_window_in_days = config.get("bulk_window_in_days") @@ -687,27 +689,6 @@ def availability_strategy(self) -> None: """NOT USED FOR BULK OPERATIONS TO SAVE THE RATE LIMITS AND TIME FOR THE SYNC.""" return None - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - """ - NOT USED FOR SHOPIFY BULK OPERARTIONS. - https://shopify.dev/docs/api/usage/bulk-operations/queries#write-a-bulk-operation - """ - return {} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - NOT USED FOR SHOPIFY BULK OPERATIONS. - https://shopify.dev/docs/api/usage/bulk-operations/queries#write-a-bulk-operation - """ - return None - - def request_body_json(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Mapping[str, Any]: - """ - Override for _send_request CDK method to send HTTP request to Shopify BULK Operatoions. - https://shopify.dev/docs/api/usage/bulk-operations/queries#bulk-query-overview - """ - return {"query": ShopifyBulkTemplates.prepare(stream_slice.get("query"))} - def get_updated_state( self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] ) -> MutableMapping[str, Any]: @@ -761,21 +742,23 @@ def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwar self.job_manager.job_size_normalize(start, end) slice_end = self.job_manager.get_adjusted_job_start(start) self.emit_slice_message(start, slice_end) - yield {"query": self.query.get(self.filter_field, start.to_rfc3339_string(), slice_end.to_rfc3339_string())} + yield {"start": start.to_rfc3339_string(), "end": slice_end.to_rfc3339_string()} # increment the end of the slice or reduce the next slice start = self.job_manager.get_adjusted_job_end(start, slice_end) else: # for the streams that don't support filtering - yield {"query": self.query.get()} + yield {} - def process_bulk_results( + def read_records( self, - response: requests.Response, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, stream_state: Optional[Mapping[str, Any]] = None, - ) -> Optional[Iterable[Mapping[str, Any]]]: - # process the CREATED Job prior to other actions - self.job_manager.job_process_created(response) - # get results fetched from COMPLETED BULK Job + ) -> Iterable[StreamData]: + self.job_manager.create_job(stream_slice, self.filter_field) + stream_state = stream_state_cache.cached_state.get(self.name, {self.cursor_field: self.default_state_comparison_value}) + filename = self.job_manager.job_check_for_completion() # the `filename` could be `None`, meaning there are no data available for the slice period. if filename: @@ -785,8 +768,3 @@ def process_bulk_results( self.record_producer.read_file(filename) ) yield from self.filter_records_newer_than_state(stream_state, records) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # get the cached substream state, to avoid state collisions for Incremental Syncs - stream_state = stream_state_cache.cached_state.get(self.name, {self.cursor_field: self.default_state_comparison_value}) - yield from self.process_bulk_results(response, stream_state) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index 4a34ad1fab31..69a27c8e5af8 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -43,7 +43,7 @@ class ShopifyBadJsonError(AirbyteTracedException): def __init__(self, message, **kwargs) -> None: self.message = f"Reason: Bad JSON Response from the Shopify server. Details: {message}." - super().__init__(internal_message=self.message, failure_type=FailureType.config_error, **kwargs) + super().__init__(internal_message=self.message, failure_type=FailureType.transient_error, **kwargs) class ShopifyConnectionError(AirbyteTracedException): diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index a627ced549fd..79bada364f95 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -13,6 +13,13 @@ os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" + +@pytest.fixture(autouse=True) +def time_sleep_mock(mocker): + time_mock = mocker.patch("time.sleep", lambda x: None) + yield time_mock + + def records_per_slice(parent_records: List[Mapping[str, Any]], state_checkpoint_interval) -> List[int]: num_batches = len(parent_records) // state_checkpoint_interval if len(parent_records) % state_checkpoint_interval != 0: diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py index c59e7cd4e50e..8ce58ef3eae8 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py @@ -5,6 +5,7 @@ import pytest import requests +from airbyte_protocol.models import SyncMode from source_shopify.shopify_graphql.bulk.exceptions import ShopifyBulkExceptions from source_shopify.shopify_graphql.bulk.status import ShopifyBulkJobStatus from source_shopify.streams.streams import ( @@ -22,6 +23,9 @@ TransactionsGraphql, ) +_ANY_SLICE = {} +_ANY_FILTER_FIELD = "any_filter_field" + def test_get_errors_from_response_invalid_response(auth_config) -> None: expected = "Couldn't check the `response` for `errors`" @@ -38,7 +42,7 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: stream = MetafieldOrders(auth_config) stream.job_manager._concurrent_interval = 0 # mocking responses - requests_mock.get( + requests_mock.post( stream.job_manager.base_url, [ # concurrent request is running (3 - retries) @@ -49,8 +53,7 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: {"json": request.getfixturevalue("bulk_successful_response")}, ]) - test_response = requests.get(stream.job_manager.base_url) - stream.job_manager._job_healthcheck(test_response) + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) # call count should be 4 (3 retries, 1 - succeeded) assert requests_mock.call_count == 4 @@ -58,38 +61,34 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: @pytest.mark.parametrize( "bulk_job_response, concurrent_max_retry, error_type, expected", [ - # method should return this response fixture, once retried. - ("bulk_successful_completed_response", 2, None, "gid://shopify/BulkOperation/4046733967549"), # method should raise AirbyteTracebackException, because the concurrent BULK Job is in progress ( - "bulk_error_with_concurrent_job", - 1, - ShopifyBulkExceptions.BulkJobConcurrentError, + "bulk_error_with_concurrent_job", + 1, + ShopifyBulkExceptions.BulkJobConcurrentError, "The BULK Job couldn't be created at this time, since another job is running", ), ], ids=[ - "regular concurrent request", - "max atttempt reached", + "max attempt reached", ] ) def test_job_retry_on_concurrency(request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected) -> None: stream = MetafieldOrders(auth_config) - # patching concurent settings + # patching concurrent settings stream.job_manager._concurrent_max_retry = concurrent_max_retry stream.job_manager._concurrent_interval = 1 - requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) - stream.job_manager._request = requests.get(stream.job_manager.base_url).request - + requests_mock.post(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + if error_type: with pytest.raises(error_type) as error: - stream.job_manager._job_retry_on_concurrency() + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) assert expected in repr(error.value) and requests_mock.call_count == 2 else: # simulate the real job_id from created job stream.job_manager._job_id = expected - stream.job_manager._job_retry_on_concurrency() + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) assert requests_mock.call_count == 2 @@ -105,7 +104,7 @@ def test_job_process_created(request, requests_mock, bulk_job_response, auth_con requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) test_response = requests.get(stream.job_manager.base_url) # process the job with id (typically CREATED one) - stream.job_manager.job_process_created(test_response) + stream.job_manager._job_process_created(test_response) assert stream.job_manager._job_id == expected @@ -165,15 +164,6 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, 1, ), # Should be retried - ( - "bulk_successful_response_with_errors", - True, - ShopifyBulkExceptions.BulkJobError, - 2, - "Could not validate the status of the BULK Job", - 3, - ), - # Should be retried ( None, False, @@ -185,11 +175,10 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, ], ids=[ "BulkJobNonHandableError", - "BulkJobError", "BulkJobBadResponse", ], ) -def test_retry_on_job_exception(mocker, request, requests_mock, job_response, auth_config, job_state, error_type, max_retry, call_count_expected, expected_msg) -> None: +def test_retry_on_job_creation_exception(request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg) -> None: stream = MetafieldOrders(auth_config) stream.job_manager._job_backoff_time = 0 stream.job_manager._job_max_retries = max_retry @@ -207,7 +196,7 @@ def test_retry_on_job_exception(mocker, request, requests_mock, job_response, au # testing raised exception and backoff with pytest.raises(error_type) as error: - stream.job_manager._job_check_state() + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) # we expect different call_count, because we set the different max_retries assert expected_msg in repr(error.value) and requests_mock.call_count == call_count_expected @@ -304,12 +293,10 @@ def test_bulk_stream_parse_response( test_result_url = bulk_job_completed_response.get("data").get("node").get("url") # mocking the result url with jsonl content requests_mock.post(stream.job_manager.base_url, json=bulk_job_completed_response) - # getting mock response - test_bulk_response: requests.Response = requests.post(stream.job_manager.base_url) # mocking nested api call to get data from result url requests_mock.get(test_result_url, text=request.getfixturevalue(json_content_example)) # parsing result from completed job - test_records = list(stream.parse_response(test_bulk_response)) + test_records = list(stream.read_records(SyncMode.full_refresh, stream_slice={})) expected_result = request.getfixturevalue(expected) if isinstance(expected_result, dict): assert test_records == [expected_result] @@ -318,13 +305,13 @@ def test_bulk_stream_parse_response( @pytest.mark.parametrize( - "stream, stream_state, with_start_date, expected", + "stream, stream_state, with_start_date, expected_start", [ - (DiscountCodes, {}, True, "updated_at:>='2023-01-01T00:00:00+00:00'"), + (DiscountCodes, {}, True, "2023-01-01T00:00:00+00:00"), # here the config migration is applied and the value should be "2020-01-01" - (DiscountCodes, {}, False, "updated_at:>='2020-01-01T00:00:00+00:00'"), - (DiscountCodes, {"updated_at": "2022-01-01T00:00:00Z"}, True, "updated_at:>='2022-01-01T00:00:00+00:00'"), - (DiscountCodes, {"updated_at": "2021-01-01T00:00:00Z"}, False, "updated_at:>='2021-01-01T00:00:00+00:00'"), + (DiscountCodes, {}, False, "2020-01-01T00:00:00+00:00"), + (DiscountCodes, {"updated_at": "2022-01-01T00:00:00Z"}, True, "2022-01-01T00:00:00+00:00"), + (DiscountCodes, {"updated_at": "2021-01-01T00:00:00Z"}, False, "2021-01-01T00:00:00+00:00"), ], ids=[ "No State, but Start Date", @@ -338,7 +325,7 @@ def test_stream_slices( stream, stream_state, with_start_date, - expected, + expected_start, ) -> None: # simulating `None` for `start_date` and `config migration` if not with_start_date: @@ -347,8 +334,7 @@ def test_stream_slices( stream = stream(auth_config) stream.job_manager.job_size = 1000 test_result = list(stream.stream_slices(stream_state=stream_state)) - test_query_from_slice = test_result[0].get("query") - assert expected in test_query_from_slice + assert test_result[0].get("start") == expected_start @pytest.mark.parametrize( @@ -377,8 +363,6 @@ def test_expand_stream_slices_job_size( test_result_url = bulk_job_completed_response.get("data").get("node").get("url") # mocking the result url with jsonl content requests_mock.post(stream.job_manager.base_url, json=bulk_job_completed_response) - # getting mock response - test_bulk_response: requests.Response = requests.post(stream.job_manager.base_url) # mocking nested api call to get data from result url requests_mock.get(test_result_url, text=request.getfixturevalue(json_content_example)) @@ -389,6 +373,8 @@ def test_expand_stream_slices_job_size( if last_job_elapsed_time: stream.job_manager._job_last_elapsed_time = last_job_elapsed_time # parsing result from completed job - list(stream.parse_response(test_bulk_response)) + + first_slice = next(stream.stream_slices()) + list(stream.read_records(SyncMode.incremental, stream_slice=first_slice)) # check the next slice assert stream.job_manager.job_size == adjusted_slice_size diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/__init__.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py new file mode 100644 index 000000000000..df16077abc14 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS +from airbyte_cdk.test.mock_http.response_builder import find_template +from source_shopify.scopes import SCOPES_MAPPING +from source_shopify.streams.base_streams import ShopifyStream + +_ALL_SCOPES = [scope for stream_scopes in SCOPES_MAPPING.values() for scope in stream_scopes] + + +def set_up_shop(http_mocker: HttpMocker, shop_name: str) -> None: + http_mocker.get( + HttpRequest(f"https://{shop_name}.myshopify.com/admin/api/{ShopifyStream.api_version}/shop.json", query_params=ANY_QUERY_PARAMS), + HttpResponse(json.dumps(find_template("shop", __file__)), status_code=200), + ) + + +def grant_all_scopes(http_mocker: HttpMocker, shop_name: str) -> None: + http_mocker.get( + HttpRequest(f"https://{shop_name}.myshopify.com/admin/oauth/access_scopes.json"), + HttpResponse(json.dumps({"access_scopes": [{"handle": scope} for scope in _ALL_SCOPES]}), status_code=200), + ) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py new file mode 100644 index 000000000000..548e8c6e3e14 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py @@ -0,0 +1,177 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime +from random import randint + +from airbyte_cdk.test.mock_http import HttpRequest, HttpResponse +from source_shopify.shopify_graphql.bulk.query import ShopifyBulkTemplates +from source_shopify.streams.base_streams import ShopifyStream + + +def _create_job_url(shop_name: str) -> str: + return f"https://{shop_name}.myshopify.com/admin/api/{ShopifyStream.api_version}/graphql.json" + + +def create_job_creation_body(lower_boundary: datetime, upper_boundary: datetime): + query = """ { + orders( + query: "updated_at:>='%LOWER_BOUNDARY_TOKEN%' AND updated_at:<='%UPPER_BOUNDARY_TOKEN%'" + sortKey: UPDATED_AT + ) { + edges { + node { + __typename + id + metafields { + edges { + node { + __typename + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } +}""" + query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace("%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat()) + prepared_query = ShopifyBulkTemplates.prepare(query) + return json.dumps({"query": prepared_query}) + + +def create_job_creation_request(shop_name: str, lower_boundary: datetime, upper_boundary: datetime) -> HttpRequest: + return HttpRequest( + url=_create_job_url(shop_name), + body=create_job_creation_body(lower_boundary, upper_boundary) + ) + + +def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: + return HttpRequest( + url=_create_job_url(shop_name), + body=f"""query {{ + node(id: "{job_id}") {{ + ... on BulkOperation {{ + id + status + errorCode + createdAt + objectCount + fileSize + url + partialDataUrl + }} + }} + }}""" + ) + + +class JobCreationResponseBuilder: + def __init__(self) -> None: + self._template = { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": { + "id": "gid://shopify/BulkOperation/0", + "status": "CREATED", + "createdAt": "2024-05-05T02:00:00Z" + }, + "userErrors": [] + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 2000.0, + "currentlyAvailable": 1990, + "restoreRate": 100.0 + } + } + } + } + + def with_bulk_operation_id(self, bulk_operation_id: str) -> "JobCreationResponseBuilder": + self._template["data"]["bulkOperationRunQuery"]["bulkOperation"]["id"] = bulk_operation_id + return self + + def build(self) -> HttpResponse: + return HttpResponse(json.dumps(self._template), status_code=200) + + +class JobStatusResponseBuilder: + def __init__(self) -> None: + self._template = { + "data": { + "node": {}, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 2000.0, + "currentlyAvailable": 1999, + "restoreRate": 100.0 + } + } + } + } + } + + def with_running_status(self, bulk_operation_id: str) -> "JobStatusResponseBuilder": + self._template["data"]["node"] = { + "id": bulk_operation_id, + "status": "RUNNING", + "errorCode": None, + "createdAt": "2024-05-28T18:57:54Z", + "objectCount": "10", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + return self + + def with_completed_status(self, bulk_operation_id: str, job_result_url: str) -> "JobStatusResponseBuilder": + self._template["data"]["node"] = { + "id": bulk_operation_id, + "status": "COMPLETED", + "errorCode": None, + "createdAt": "2024-05-05T00:45:48Z", + "objectCount": "4", + "fileSize": "774", + "url": job_result_url, + "partialDataUrl": None + } + return self + + def build(self) -> HttpResponse: + return HttpResponse(json.dumps(self._template), status_code=200) + + +class MetafieldOrdersJobResponseBuilder: + def __init__(self) -> None: + self._records = [] + + def _any_record(self) -> str: + an_id = str(randint(1000000000000, 9999999999999)) + a_parent_id = str(randint(1000000000000, 9999999999999)) + return f"""{{"__typename":"Order","id":"gid:\/\/shopify\/Order\/{a_parent_id}"}} +{{"__typename":"Metafield","id":"gid:\/\/shopify\/Metafield\/{an_id}","namespace":"my_fields","value":"asdfasdf","key":"purchase_order","description":null,"createdAt":"2023-04-13T12:09:50Z","updatedAt":"2024-05-05T01:09:50Z","type":"single_line_text_field","__parentId":"gid:\/\/shopify\/Order\/{a_parent_id}"}} +""" + + + def with_record(self) -> "MetafieldOrdersJobResponseBuilder": + self._records.append(self._any_record()) + return self + + def build(self) -> HttpResponse: + return HttpResponse("".join(self._records), status_code=200) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py new file mode 100644 index 000000000000..399998756877 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py @@ -0,0 +1,185 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import json +from datetime import datetime, timedelta +from typing import Any, Dict +from unittest import TestCase + +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse + +_AN_ERROR_RESPONSE = HttpResponse(json.dumps({"errors": ["an error"]})) +from airbyte_protocol.models import SyncMode +from freezegun import freeze_time +from requests.exceptions import ConnectionError +from source_shopify import SourceShopify +from unit_tests.integration.api.authentication import grant_all_scopes, set_up_shop +from unit_tests.integration.api.bulk import ( + JobCreationResponseBuilder, + JobStatusResponseBuilder, + MetafieldOrdersJobResponseBuilder, + create_job_creation_body, + create_job_creation_request, + create_job_status_request, +) + +_BULK_OPERATION_ID = "gid://shopify/BulkOperation/4472588009661" +_BULK_STREAM = "metafield_orders" +_SHOP_NAME = "airbyte-integration-test" + +_JOB_START_DATE = datetime.fromisoformat("2024-05-05T00:00:00+00:00") +_JOB_END_DATE = _JOB_START_DATE + timedelta(hours=2, minutes=24) + +_URL_GRAPHQL = f"https://{_SHOP_NAME}.myshopify.com/admin/api/2024-04/graphql.json" +_JOB_RESULT_URL = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693949.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693949.jsonl&response-content-type=application%2Fjsonl" + + +def _get_config(start_date: datetime, bulk_window: int = 1) -> Dict[str, Any]: + return { + "start_date": start_date.strftime("%Y-%m-%d"), + "shop": _SHOP_NAME, + "credentials": { + "auth_method": "api_password", + "api_password": "api_password", + }, + "bulk_window_in_days": bulk_window + } + + +@freeze_time(_JOB_END_DATE) +class GraphQlBulkStreamTest(TestCase): + + def setUp(self) -> None: + self._http_mocker = HttpMocker() + self._http_mocker.__enter__() + + set_up_shop(self._http_mocker, _SHOP_NAME) + grant_all_scopes(self._http_mocker, _SHOP_NAME) + + def tearDown(self) -> None: + self._http_mocker.__exit__(None, None, None) + + def test_when_read_then_extract_records(self) -> None: + self._http_mocker.post( + create_job_creation_request(_SHOP_NAME, _JOB_START_DATE, _JOB_END_DATE), + JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), + ) + self._http_mocker.post( + create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), + JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), + ) + self._http_mocker.get( + HttpRequest(_JOB_RESULT_URL), + MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), + ) + + output = self._read(_get_config(_JOB_START_DATE)) + + assert output.errors == [] + assert len(output.records) == 2 + + def test_given_errors_on_job_creation_when_read_then_do_not_retry(self) -> None: + """ + The purpose of this test is to document the current behavior as I'm not sure we have an example of such errors on the job creation + """ + job_creation_request = create_job_creation_request(_SHOP_NAME, _JOB_START_DATE, _JOB_END_DATE) + self._http_mocker.post(job_creation_request, _AN_ERROR_RESPONSE) + + self._read(_get_config(_JOB_START_DATE)) + + self._http_mocker.assert_number_of_calls(job_creation_request, 1) + + def test_given_response_is_not_json_on_job_creation_when_read_then_retry(self) -> None: + job_creation_request = create_job_creation_request(_SHOP_NAME, _JOB_START_DATE, _JOB_END_DATE) + self._http_mocker.post( + job_creation_request, + [ + HttpResponse("This is not json"), + JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), # This will never get called (see assertion below) + ] + ) + + self._http_mocker.post( + create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), + JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), + ) + self._http_mocker.get( + HttpRequest(_JOB_RESULT_URL), + MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), + ) + + output = self._read(_get_config(_JOB_START_DATE)) + + assert output.errors == [] + assert len(output.records) == 2 + + def test_given_connection_error_on_job_creation_when_read_then_retry_job_creation(self) -> None: + inner_mocker = self._http_mocker.__getattribute__("_mocker") + inner_mocker.register_uri( # TODO the testing library should have the ability to generate ConnectionError. As this might not be trivial, we will wait for another case before implementing + "POST", + _URL_GRAPHQL, + [{"exc": ConnectionError("ConnectionError")}, {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}], + additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE) + ) + self._http_mocker.post( + create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), + JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), + ) + self._http_mocker.get( + HttpRequest(_JOB_RESULT_URL), + MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), + ) + + output = self._read(_get_config(_JOB_START_DATE)) + + assert output.errors == [] + + def test_given_retryable_error_on_first_get_job_status_when_read_then_retry(self) -> None: + self._http_mocker.post( + create_job_creation_request(_SHOP_NAME, _JOB_START_DATE, _JOB_END_DATE), + JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), + ) + self._http_mocker.post( + create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), + [ + _AN_ERROR_RESPONSE, + JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), + ] + ) + self._http_mocker.get( + HttpRequest(_JOB_RESULT_URL), + MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), + ) + + output = self._read(_get_config(_JOB_START_DATE)) + + assert output.errors == [] + assert len(output.records) == 2 + + def test_given_retryable_error_on_get_job_status_when_read_then_retry(self) -> None: + self._http_mocker.post( + create_job_creation_request(_SHOP_NAME, _JOB_START_DATE, _JOB_END_DATE), + JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), + ) + self._http_mocker.post( + create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), + [ + JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID).build(), + HttpResponse(json.dumps({"errors": ["an error"]})), + JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), + ] + ) + self._http_mocker.get( + HttpRequest(_JOB_RESULT_URL), + MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), + ) + + output = self._read(_get_config(_JOB_START_DATE)) + + assert output.errors == [] + assert len(output.records) == 2 + + def _read(self, config): + catalog = CatalogBuilder().with_stream(_BULK_STREAM, SyncMode.full_refresh).build() + output = read(SourceShopify(), config, catalog) + return output diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/resource/http/response/shop.json b/airbyte-integrations/connectors/source-shopify/unit_tests/resource/http/response/shop.json new file mode 100644 index 000000000000..e4b4eb802a4f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/resource/http/response/shop.json @@ -0,0 +1,58 @@ +{ + "shop": { + "id": 58033176765, + "name": "airbyte integration test", + "email": "sherif@airbyte.io", + "domain": "a-shop.myshopify.com", + "province": "California", + "country": "US", + "address1": "350 29th Avenue", + "zip": "94121", + "city": "San Francisco", + "source": null, + "phone": "8023494963", + "latitude": 37.7827286, + "longitude": -122.4889911, + "primary_locale": "en", + "address2": "", + "created_at": "2021-06-22T18:00:23-07:00", + "updated_at": "2024-05-05T01:11:05-08:00", + "country_code": "US", + "country_name": "United States", + "currency": "USD", + "customer_email": "sherif@airbyte.io", + "timezone": "(GMT-08:00) America/Los_Angeles", + "iana_timezone": "America/Los_Angeles", + "shop_owner": "Airbyte Airbyte", + "money_format": "${{amount}}", + "money_with_currency_format": "${{amount}} USD", + "weight_unit": "kg", + "province_code": "CA", + "taxes_included": true, + "auto_configure_tax_inclusivity": null, + "tax_shipping": null, + "county_taxes": true, + "plan_display_name": "Developer Preview", + "plan_name": "partner_test", + "has_discounts": true, + "has_gift_cards": false, + "myshopify_domain": "a-shop.myshopify.com", + "google_apps_domain": null, + "google_apps_login_enabled": null, + "money_in_emails_format": "${{amount}}", + "money_with_currency_in_emails_format": "${{amount}} USD", + "eligible_for_payments": true, + "requires_extra_payments_agreement": false, + "password_enabled": true, + "has_storefront": true, + "finances": true, + "primary_location_id": 63590301885, + "checkout_api_supported": true, + "multi_location_enabled": true, + "setup_required": false, + "pre_launch_enabled": false, + "enabled_presentment_currencies": ["USD"], + "transactional_sms_disabled": false, + "marketing_sms_consent_enabled_at_checkout": false + } +} diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index af13681143d2..69602c9ab6d9 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -211,6 +211,7 @@ For all `Shopify GraphQL BULK` api requests these limitations are applied: https | Version | Date | Pull Request | Subject | | :------ |:-----------| :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.2.3 | 2024-06-06 | [38084](https://github.com/airbytehq/airbyte/pull/38084) | add resiliency on some transient errors using the HttpClient | | 2.2.2 | 2024-06-04 | [39019](https://github.com/airbytehq/airbyte/pull/39019) | [autopull] Upgrade base image to v1.2.1 | | 2.2.1 | 2024-05-30 | [38769](https://github.com/airbytehq/airbyte/pull/38769) | Have products stream return all the tags comma separated | | 2.2.0 | 2024-05-29 | [38746](https://github.com/airbytehq/airbyte/pull/38746) | Updated countries schema |