From f5c862d7c80192edb64eb025bf9c1807cd8a14cd Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Thu, 3 Aug 2023 01:57:52 +0200 Subject: [PATCH 1/5] Add incremental for issues stream --- .../connectors/source-sentry/Dockerfile | 2 +- .../source-sentry/acceptance-test-config.yml | 8 ---- .../integration_tests/configured_catalog.json | 6 +-- .../integration_tests/expected_records.jsonl | 2 + .../connectors/source-sentry/metadata.yaml | 2 +- .../source-sentry/source_sentry/manifest.yaml | 11 +++++- docs/integrations/sources/sentry.md | 37 ++++++++++--------- 7 files changed, 35 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 9b4d66a31b28..1aefd080b8f7 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index 798820ceba20..91c39b263e9d 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -5,14 +5,6 @@ acceptance_tests: timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: "events" - bypass_reason: "Data reset after some time" - - name: "issues" - bypass_reason: "Data reset after some time" ignored_fields: project_detail: - name: access diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json index 1faba770179b..943781880339 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json @@ -10,12 +10,12 @@ } }, { - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", + "sync_mode": "incremental", + "destination_sync_mode": "append", "stream": { "name": "issues", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] } }, { diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl index 935e3ecaee19..fe181c199a95 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl @@ -2,3 +2,5 @@ {"stream":"projects","data":{"id":"6712547","slug":"demo-integration","name":"demo-integration","platform":"javascript-react","dateCreated":"2022-09-02T15:01:28.946777Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view"],"firstEvent":"2022-09-02T15:36:50.870000Z","firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf833f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328146} {"stream":"projects","data":{"id":"5942472","slug":"airbyte-09","name":"airbyte-09","platform":"python","dateCreated":"2021-09-02T07:42:22.421223Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","releases"],"firstEvent":null,"firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#803fbf","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328148} {"stream": "releases", "data": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": [], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "emitted_at": 1689246658349} +{"stream": "issues", "data": {"id": "4365423845", "shareId": null, "shortId": "AIRBYTE-09-4", "title": "This is an example Python exception", "culprit": "raven.scripts.runner in main", "permalink": "https://airbyte-09.sentry.io/issues/4365423845/", "logger": null, "level": "error", "status": "unresolved", "statusDetails": {}, "substatus": "ongoing", "isPublic": false, "platform": "python", "project": {"id": "5942472", "name": "airbyte-09", "slug": "airbyte-09", "platform": "python"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "numComments": 0, "assignedTo": null, "isBookmarked": false, "isSubscribed": false, "subscriptionDetails": null, "hasSeen": true, "annotations": [], "issueType": "error", "issueCategory": "error", "isUnhandled": false, "count": "10", "userCount": 1, "firstSeen": "2023-08-02T23:22:34.982000Z", "lastSeen": "2023-08-02T23:31:20.165000Z"}, "emitted_at": 1691020096265} +{"stream": "events", "data": {"id": "1cce9233aeb04eba8bbdb7bb00f00592", "groupID": "4365423845", "eventID": "1cce9233aeb04eba8bbdb7bb00f00592", "projectID": "5942472", "size": 8151, "entries": [{"data": {"formatted": "This is an example Python exception"}, "type": "message"}, {"data": {"frames": [{"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "build_msg", "rawFunction": null, "symbol": null, "context": [[298, " frames = stack"], [299, ""], [300, " data.update({"], [301, " 'sentry.interfaces.Stacktrace': {"], [302, " 'frames': get_stack_info(frames,"], [303, " transformer=self.transform)"], [304, " },"], [305, " })"], [306, ""], [307, " if 'sentry.interfaces.Stacktrace' in data:"], [308, " if self.include_paths:"]], "lineNo": 303, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'culprit'": null, "'data'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'date'": "datetime.datetime(2013, 8, 13, 3, 8, 24, 880386)", "'event_id'": "'54a322436e1b47b88e239b78998ae742'", "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'frames'": "", "'handler'": "", "'k'": "'sentry.interfaces.Message'", "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'public_key'": null, "'result'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null, "'v'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "capture", "rawFunction": null, "symbol": null, "context": [[454, " if not self.is_enabled():"], [455, " return"], [456, ""], [457, " data = self.build_msg("], [458, " event_type, data, date, time_spent, extra, stack, tags=tags,"], [459, " **kwargs)"], [460, ""], [461, " self.send(**data)"], [462, ""], [463, " return (data.get('event_id'),)"], [464, ""]], "lineNo": 459, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'data'": null, "'date'": null, "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "captureMessage", "rawFunction": null, "symbol": null, "context": [[572, " \"\"\""], [573, " Creates an event from ``message``."], [574, ""], [575, " >>> client.captureMessage('My event just happened!')"], [576, " \"\"\""], [577, " return self.capture('raven.events.Message', message=message, **kwargs)"], [578, ""], [579, " def captureException(self, exc_info=None, **kwargs):"], [580, " \"\"\""], [581, " Creates an event from an exception."], [582, ""]], "lineNo": 577, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'kwargs'": {"'data'": null, "'extra'": {"'go_deeper'": ["[{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}]"], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'level'": 20, "'stack'": true, "'tags'": null}, "'message'": "'This is a test message generated using ``raven test``'", "'self'": ""}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "send_test_message", "rawFunction": null, "symbol": null, "context": [[72, " level=logging.INFO,"], [73, " stack=True,"], [74, " tags=options.get('tags', {}),"], [75, " extra={"], [76, " 'user': get_uid(),"], [77, " 'loadavg': get_loadavg(),"], [78, " },"], [79, " ))"], [80, ""], [81, " if client.state.did_fail():"], [82, " print('error!')"]], "lineNo": 77, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'client'": "", "'data'": null, "'k'": "'secret_key'", "'options'": {"'data'": null, "'tags'": null}}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "main", "rawFunction": null, "symbol": null, "context": [[107, " print(\"Using DSN configuration:\")"], [108, " print(\" \", dsn)"], [109, " print()"], [110, ""], [111, " client = Client(dsn, include_paths=['raven'])"], [112, " send_test_message(client, opts.__dict__)"]], "lineNo": 112, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'args'": ["'test'", "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'"], "'client'": "", "'dsn'": "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'", "'opts'": "", "'parser'": "", "'root'": ""}}], "framesOmitted": null, "registers": null, "hasSystemFrames": false}, "type": "stacktrace"}, {"data": {"apiTarget": null, "method": "GET", "url": "http://example.com/foo", "query": [["foo", "bar"]], "fragment": null, "data": {"hello": "world"}, "headers": [["Content-Type", "application/json"], ["Referer", "http://example.com"], ["User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"]], "cookies": [["foo", "bar"], ["biz", "baz"]], "env": {"ENV": "prod"}, "inferredContentType": "application/json"}, "type": "request"}], "dist": null, "message": "This is an example Python exception", "title": "This is an example Python exception", "location": null, "user": {"id": "1", "email": "sentry@example.com", "username": "sentry", "ip_address": "127.0.0.1", "name": "Sentry", "data": null}, "contexts": {"browser": {"name": "Chrome", "version": "28.0.1500", "type": "browser"}, "client_os": {"name": "Windows", "version": "8", "type": "os"}}, "sdk": null, "context": {"emptyList": [], "emptyMap": {}, "length": 10837790, "results": [1, 2, 3, 4, 5], "session": {"foo": "bar"}, "unauthorized": false, "url": "http://example.org/foo/bar/"}, "packages": {"my.package": "1.0.0"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "tags": [{"key": "browser", "value": "Chrome 28.0.1500"}, {"key": "browser.name", "value": "Chrome"}, {"key": "client_os", "value": "Windows 8"}, {"key": "client_os.name", "value": "Windows"}, {"key": "environment", "value": "prod"}, {"key": "level", "value": "error"}, {"key": "sample_event", "value": "yes"}, {"key": "server_name", "value": "web01.example.org"}, {"key": "url", "value": "http://example.com/foo"}, {"key": "user", "value": "id:1", "query": "user.id:\"1\""}], "platform": "python", "dateReceived": "2023-08-02T23:31:41.101814Z", "errors": [], "occurrence": null, "_meta": {"entries": {}, "message": null, "user": null, "contexts": null, "sdk": null, "context": null, "packages": null, "tags": {}}, "crashFile": null, "culprit": "raven.scripts.runner in main", "dateCreated": "2023-08-02T23:30:41Z", "fingerprints": ["3a2b45089d0211943e5a6645fb4cea3f"], "groupingConfig": {"id": "newstyle:2023-01-11", "enhancements": "eJybzDRxc15qeXFJZU6qlZGBkbGugaGuoeEEAHJMCAM"}}, "emitted_at": 1691020171602} diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index f94bd643908f..06745a4629c4 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 maxSecondsBetweenMessages: 64800 dockerRepository: airbyte/source-sentry githubIssueLabel: source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml index 4cd5508a78eb..9d0f9ce9db14 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml @@ -32,7 +32,6 @@ definitions: stop_condition: "{{ headers.link.next.results != 'true' }}" retriever: type: SimpleRetriever - streams: - type: DeclarativeStream $parameters: @@ -67,9 +66,17 @@ streams: path: "projects/{{config.organization}}/{{config.project}}/issues/" request_parameters: statsPeriod: "" - query: "" + query: "lastSeen:>{{stream_interval.start_time}}" paginator: $ref: "#/definitions/paginator" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: lastseen + datetime_format: '%Y-%m-%dT%H:%M:%S.%f' + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' - type: DeclarativeStream $parameters: name: "projects" diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index 2addcbb1ff8c..ba8847e29b04 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -45,21 +45,22 @@ The Sentry source connector supports the following [sync modes](https://docs.air ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------| -| 0.2.2 | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection | -| 0.2.1 | 2023-04-27 | [25602](https://github.com/airbytehq/airbyte/pull/25602) | Add validation of project and organization names during connector setup | -| 0.2.0 | 2023-04-03 | [23923](https://github.com/airbytehq/airbyte/pull/23923) | Add Releases stream | -| 0.1.12 | 2023-03-01 | [23619](https://github.com/airbytehq/airbyte/pull/23619) | Fix bug when `stream state` is `None` or any other bad value occurs | -| 0.1.11 | 2023-02-02 | [22303](https://github.com/airbytehq/airbyte/pull/22303) | Turn ON default AvailabilityStrategy | -| 0.1.10 | 2023-01-27 | [22041](https://github.com/airbytehq/airbyte/pull/22041) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.9 | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync | -| 0.1.8 | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync | -| 0.1.7 | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states | -| 0.1.6 | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK | -| 0.1.5 | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime | -| 0.1.4 | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime | -| 0.1.3 | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator | -| 0.1.2 | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework | -| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.2.3 | 2023-08-03 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add incremental for `issues` stream | +| 0.2.2 | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection | +| 0.2.1 | 2023-04-27 | [25602](https://github.com/airbytehq/airbyte/pull/25602) | Add validation of project and organization names during connector setup | +| 0.2.0 | 2023-04-03 | [23923](https://github.com/airbytehq/airbyte/pull/23923) | Add Releases stream | +| 0.1.12 | 2023-03-01 | [23619](https://github.com/airbytehq/airbyte/pull/23619) | Fix bug when `stream state` is `None` or any other bad value occurs | +| 0.1.11 | 2023-02-02 | [22303](https://github.com/airbytehq/airbyte/pull/22303) | Turn ON default AvailabilityStrategy | +| 0.1.10 | 2023-01-27 | [22041](https://github.com/airbytehq/airbyte/pull/22041) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.9 | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync | +| 0.1.8 | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync | +| 0.1.7 | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states | +| 0.1.6 | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK | +| 0.1.5 | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime | +| 0.1.4 | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime | +| 0.1.3 | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator | +| 0.1.2 | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework | +| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | From 784cdb4152cb1f5036d90ed44c866057769dfb58 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Thu, 3 Aug 2023 02:27:30 +0200 Subject: [PATCH 2/5] Update PR number --- docs/integrations/sources/sentry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index ba8847e29b04..d7dae9ffc58a 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -47,7 +47,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| -| 0.2.3 | 2023-08-03 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add incremental for `issues` stream | +| 0.2.3 | 2023-08-03 | [29023](https://github.com/airbytehq/airbyte/pull/29023) | Add incremental for `issues` stream | | 0.2.2 | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection | | 0.2.1 | 2023-04-27 | [25602](https://github.com/airbytehq/airbyte/pull/25602) | Add validation of project and organization names during connector setup | | 0.2.0 | 2023-04-03 | [23923](https://github.com/airbytehq/airbyte/pull/23923) | Add Releases stream | From 6aa611ff931e35dfd11e10c5edf88d1479f9f871 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Tue, 8 Aug 2023 21:46:02 +0200 Subject: [PATCH 3/5] Change python version instead yaml manifest --- .../source-sentry/source_sentry/manifest.yaml | 11 +----- .../source-sentry/source_sentry/streams.py | 37 +++++++++++++++---- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml index 9d0f9ce9db14..4cd5508a78eb 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml @@ -32,6 +32,7 @@ definitions: stop_condition: "{{ headers.link.next.results != 'true' }}" retriever: type: SimpleRetriever + streams: - type: DeclarativeStream $parameters: @@ -66,17 +67,9 @@ streams: path: "projects/{{config.organization}}/{{config.project}}/issues/" request_parameters: statsPeriod: "" - query: "lastSeen:>{{stream_interval.start_time}}" + query: "" paginator: $ref: "#/definitions/paginator" - incremental_sync: - type: DatetimeBasedCursor - cursor_field: lastseen - datetime_format: '%Y-%m-%dT%H:%M:%S.%f' - start_datetime: - type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' - type: DeclarativeStream $parameters: name: "projects" diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index fb6b39e964d5..168f844e5f7c 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -4,7 +4,7 @@ from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional +from typing import Any, Iterable, Mapping, MutableMapping, Optional, Dict import pendulum import requests @@ -171,16 +171,39 @@ def path( return f"projects/{self._organization}/{self._project}/issues/" def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, + self, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"statsPeriod": "", "query": ""}) - + filter_date = self._get_filter_date(stream_state) + params.update(self._build_query_params(filter_date)) return params + def _get_filter_date(self, stream_state: Optional[Mapping[str, Any]]) -> str: + """Retrieve the filter date from the stream state or use the start_date.""" + return stream_state.get(self.cursor_field) if stream_state else self.start_date + + def _build_query_params(self, filter_date: str) -> Dict[str, str]: + """Generate query parameters for the request.""" + filter_date_iso = pendulum.parse(filter_date).to_iso8601_string() + return {"statsPeriod": "", "query": f"lastSeen:>{filter_date_iso}"} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[MutableMapping]: + json_response = response.json() or [] + + for record in json_response: + cursor_value = self._get_cursor_value(record, stream_state) + self.state = {self.cursor_field: str(cursor_value)} + yield record + + def _get_cursor_value(self, record: Dict[str, Any], stream_state: Mapping[str, Any]) -> pendulum.datetime: + """Compute the maximum cursor value based on the record and stream state.""" + record_time = pendulum.parse(record[self.cursor_field]) + state_time = self.get_state_value(stream_state) + return max(record_time, state_time) + class Projects(SentryIncremental): """ From ebc5b7df36435ae00c02237374b8c98da054c798 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Tue, 8 Aug 2023 21:54:06 +0200 Subject: [PATCH 4/5] Fix code style and unt test --- .../connectors/source-sentry/source_sentry/streams.py | 10 +++++----- .../source-sentry/unit_tests/test_streams.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 168f844e5f7c..02dfaa64d131 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -4,7 +4,7 @@ from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Dict +from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional import pendulum import requests @@ -171,10 +171,10 @@ def path( return f"projects/{self._organization}/{self._project}/issues/" def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, + self, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state, stream_slice, next_page_token) filter_date = self._get_filter_date(stream_state) diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 21347f70fb5b..1ac42a5217e2 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -95,7 +95,7 @@ def test_events_request_params(): def test_issues_request_params(): stream = Issues(**INIT_ARGS) - expected = {"cursor": "next-page", "statsPeriod": "", "query": ""} + expected = {"cursor": "next-page", "statsPeriod": "", "query": "lastSeen:>1900-01-01T00:00:00Z"} assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected From 905f474ea2155bdddfd11a079e4cdab282f9d084 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Tue, 8 Aug 2023 22:26:16 +0200 Subject: [PATCH 5/5] Fix CAT for issues --- .../connectors/source-sentry/source_sentry/streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 02dfaa64d131..7a8512e08664 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -195,13 +195,13 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, for record in json_response: cursor_value = self._get_cursor_value(record, stream_state) - self.state = {self.cursor_field: str(cursor_value)} + self.state = {self.cursor_field: cursor_value} yield record def _get_cursor_value(self, record: Dict[str, Any], stream_state: Mapping[str, Any]) -> pendulum.datetime: """Compute the maximum cursor value based on the record and stream state.""" - record_time = pendulum.parse(record[self.cursor_field]) - state_time = self.get_state_value(stream_state) + record_time = record[self.cursor_field] + state_time = str(self.get_state_value(stream_state)) return max(record_time, state_time)