From 572cc97ab77330c554006e8a2aca050c9621bb8f Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Wed, 15 Jan 2025 15:58:57 -0500 Subject: [PATCH 1/4] Schemas for form, opportunity, and user --- tap_sunwave/schemas/form.json | 26 ++ tap_sunwave/schemas/opportunity.json | 644 +++++++++++++++++++++++++++ tap_sunwave/schemas/user.json | 86 ++-- 3 files changed, 718 insertions(+), 38 deletions(-) create mode 100644 tap_sunwave/schemas/form.json create mode 100644 tap_sunwave/schemas/opportunity.json diff --git a/tap_sunwave/schemas/form.json b/tap_sunwave/schemas/form.json new file mode 100644 index 0000000..fea4b44 --- /dev/null +++ b/tap_sunwave/schemas/form.json @@ -0,0 +1,26 @@ +{ + "type": [ + "object", + "null" + ], + "properties": { + "is_n_form": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "string", + "null" + ] + } + } +} \ No newline at end of file diff --git a/tap_sunwave/schemas/opportunity.json b/tap_sunwave/schemas/opportunity.json new file mode 100644 index 0000000..3677ac9 --- /dev/null +++ b/tap_sunwave/schemas/opportunity.json @@ -0,0 +1,644 @@ +{ + "type": [ + "object", + "null" + ], + "properties": { + "abandoned comment": { + "type": [ + "string", + "null" + ] + }, + "abandoned reason": { + "type": [ + "string", + "null" + ] + }, + "abandoned_by": { + "type": [ + "string", + "null" + ] + }, + "abandoned_date": { + "type": [ + "string", + "null" + ] + }, + "account_id": { + "type": [ + "string", + "null" + ] + }, + "account_legacy_id": { + "type": [ + "string", + "null" + ] + }, + "adm. representative": { + "type": [ + "string", + "null" + ] + }, + "adm. representative email": { + "type": [ + "string", + "null" + ] + }, + "admission_date": { + "type": [ + "string", + "null" + ] + }, + "admission_level_of_care": { + "type": [ + "string", + "null" + ] + }, + "admitted_by": { + "type": [ + "string", + "null" + ] + }, + "business development rep_": { + "type": [ + "string", + "null" + ] + }, + "call_id": { + "type": [ + "string", + "null" + ] + }, + "caller_address": { + "type": [ + "string", + "null" + ] + }, + "caller_city": { + "type": [ + "string", + "null" + ] + }, + "caller_email": { + "type": [ + "string", + "null" + ] + }, + "caller_firstname": { + "type": [ + "string", + "null" + ] + }, + "caller_lastname": { + "type": [ + "string", + "null" + ] + }, + "caller_phone_home": { + "type": [ + "string", + "null" + ] + }, + "caller_phone_mobile": { + "type": [ + "string", + "null" + ] + }, + "caller_phone_office": { + "type": [ + "string", + "null" + ] + }, + "caller_state": { + "type": [ + "string", + "null" + ] + }, + "caller_zipcode": { + "type": [ + "string", + "null" + ] + }, + "campaign id": { + "type": [ + "string", + "null" + ] + }, + "campaign_type": { + "type": [ + "string", + "null" + ] + }, + "communication type": { + "type": [ + "string", + "null" + ] + }, + "created_by": { + "type": [ + "string", + "null" + ] + }, + "created_on": { + "type": [ + "string", + "null" + ] + }, + "created_time": { + "type": [ + "string", + "null" + ] + }, + "discharge_date": { + "type": [ + "string", + "null" + ] + }, + "drug of choice_": { + "type": [ + "string", + "null" + ] + }, + "first contact": { + "type": [ + "string", + "null" + ] + }, + "first contact email": { + "type": [ + "string", + "null" + ] + }, + "gender": { + "type": [ + "string", + "null" + ] + }, + "google_ad_group": { + "type": [ + "string", + "null" + ] + }, + "google_campaign": { + "type": [ + "string", + "null" + ] + }, + "google_click_id": { + "type": [ + "string", + "null" + ] + }, + "google_keyword": { + "type": [ + "string", + "null" + ] + }, + "google_source": { + "type": [ + "string", + "null" + ] + }, + "has_been_scheduled": { + "type": [ + "string", + "null" + ] + }, + "insurance provider": { + "type": [ + "string", + "null" + ] + }, + "insurance_group_number": { + "type": [ + "string", + "null" + ] + }, + "level_of_care": { + "type": [ + "string", + "null" + ] + }, + "lost comment": { + "type": [ + "string", + "null" + ] + }, + "lost reason": { + "type": [ + "string", + "null" + ] + }, + "lost_by": { + "type": [ + "string", + "null" + ] + }, + "lost_date": { + "type": [ + "string", + "null" + ] + }, + "marital status": { + "type": [ + "string", + "null" + ] + }, + "member id": { + "type": [ + "string", + "null" + ] + }, + "nickname_": { + "type": [ + "string", + "null" + ] + }, + "opportunity_id": { + "type": [ + "string", + "null" + ] + }, + "opportunity_legacy_id": { + "type": [ + "string", + "null" + ] + }, + "opportunity_referral_source_contact_account_owner": { + "type": [ + "string", + "null" + ] + }, + "opportunity_referral_source_contact_secondary_account_owner": { + "type": [ + "string", + "null" + ] + }, + "opportunity_referral_source_secondary_account_owner": { + "type": [ + "string", + "null" + ] + }, + "outcome": { + "type": [ + "string", + "null" + ] + }, + "parent referral name": { + "type": [ + "string", + "null" + ] + }, + "patient dob": { + "type": [ + "string", + "null" + ] + }, + "patient name": { + "type": [ + "string", + "null" + ] + }, + "patient_address": { + "type": [ + "string", + "null" + ] + }, + "patient_city": { + "type": [ + "string", + "null" + ] + }, + "patient_email": { + "type": [ + "string", + "null" + ] + }, + "patient_phone_home": { + "type": [ + "string", + "null" + ] + }, + "patient_phone_mobile": { + "type": [ + "string", + "null" + ] + }, + "patient_phone_work": { + "type": [ + "string", + "null" + ] + }, + "patient_ssn": { + "type": [ + "string", + "null" + ] + }, + "patient_state": { + "type": [ + "string", + "null" + ] + }, + "patient_zip": { + "type": [ + "string", + "null" + ] + }, + "policy_type": { + "type": [ + "string", + "null" + ] + }, + "preadmission notes_": { + "type": [ + "string", + "null" + ] + }, + "primary_referral_source_contact_owner": { + "type": [ + "string", + "null" + ] + }, + "referral name": { + "type": [ + "string", + "null" + ] + }, + "referral source contact": { + "type": [ + "string", + "null" + ] + }, + "referral source opportunity owner": { + "type": [ + "string", + "null" + ] + }, + "referral type": { + "type": [ + "string", + "null" + ] + }, + "referral_destination": { + "type": [ + "string", + "null" + ] + }, + "referral_source_id": { + "type": [ + "string", + "null" + ] + }, + "referral_source_owner": { + "type": [ + "string", + "null" + ] + }, + "referral_source_owner_email": { + "type": [ + "string", + "null" + ] + }, + "relationship": { + "type": [ + "string", + "null" + ] + }, + "religion_": { + "type": [ + "string", + "null" + ] + }, + "scheduled admission time (hh:mm)": { + "type": [ + "string", + "null" + ] + }, + "scheduled on": { + "type": [ + "string", + "null" + ] + }, + "scheduled program": { + "type": [ + "string", + "null" + ] + }, + "scheduled_by": { + "type": [ + "string", + "null" + ] + }, + "scheduled_level_of_care": { + "type": [ + "string", + "null" + ] + }, + "scheduled_service_facility": { + "type": [ + "string", + "null" + ] + }, + "secondary term date_": { + "type": [ + "string", + "null" + ] + }, + "secondary_referral_source_contact_owner": { + "type": [ + "string", + "null" + ] + }, + "service_facility": { + "type": [ + "string", + "null" + ] + }, + "stage": { + "type": [ + "string", + "null" + ] + }, + "submit_claim_to": { + "type": [ + "string", + "null" + ] + }, + "subscriber_address": { + "type": [ + "string", + "null" + ] + }, + "subscriber_city": { + "type": [ + "string", + "null" + ] + }, + "subscriber_date_of_birth": { + "type": [ + "string", + "null" + ] + }, + "subscriber_first_name": { + "type": [ + "string", + "null" + ] + }, + "subscriber_last_name": { + "type": [ + "string", + "null" + ] + }, + "subscriber_ssn": { + "type": [ + "string", + "null" + ] + }, + "subscriber_state": { + "type": [ + "string", + "null" + ] + }, + "subscriber_zip": { + "type": [ + "string", + "null" + ] + }, + "target_admission_date": { + "type": [ + "string", + "null" + ] + }, + "target_level_of_care": { + "type": [ + "string", + "null" + ] + }, + "target_service_facility": { + "type": [ + "string", + "null" + ] + }, + "temporal_detox": { + "type": [ + "string", + "null" + ] + }, + "term date_": { + "type": [ + "string", + "null" + ] + }, + "transferred": { + "type": [ + "string", + "null" + ] + } + } +} \ No newline at end of file diff --git a/tap_sunwave/schemas/user.json b/tap_sunwave/schemas/user.json index cde87de..148346e 100644 --- a/tap_sunwave/schemas/user.json +++ b/tap_sunwave/schemas/user.json @@ -1,41 +1,51 @@ { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "User ID", - "example": "404" - }, - "first_name": { - "type": "string", - "description": "First Name", - "example": "Erlich" - }, - "last_name": { - "type": "string", - "description": "Last Name", - "example": "Bachman" - }, - "email": { - "type": "string", - "description": "Users email address", - "example": "admin@sunwave.com" - }, - "created_on": { - "type": "string", - "description": "Date user was created on", - "example": "2019-02-11 18:02:09.0" - }, - "created_by": { - "type": "string", - "description": "User that created this user", - "example": "admin@sunwave.com" - }, - "last_login": { - "type": "string", - "description": "Date user last logged into the system", - "example": "2023-02-23 11:27:07" - } + "$schema": "http://json-schema.org/schema#", + "type": [ + "object", + "null" + ], + "properties": { + "created_on": { + "type": [ + "string", + "null" + ] }, - "description": "User response object" + "last_login": { + "type": [ + "string", + "null" + ] + }, + "last_name": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "string", + "null" + ] + }, + "first_name": { + "type": [ + "string", + "null" + ] + }, + "created_by": { + "type": [ + "string", + "null" + ] + }, + "email": { + "type": [ + "string", + "null" + ] + } + } } \ No newline at end of file From d9755f504ce2a2082182f8a0aa042edd53c96dc8 Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Thu, 16 Jan 2025 15:29:25 -0500 Subject: [PATCH 2/4] Fix Authentication errors with Retries, migrate to static schemas as dynamic was missing fields --- tap_sunwave/client.py | 21 +- tap_sunwave/schemas/census.json | 641 ++++++++++++++++++ tap_sunwave/schemas/form.json | 1 + tap_sunwave/schemas/opportunity.json | 1 + tap_sunwave/schemas/opportunity_timeline.json | 237 +++++++ tap_sunwave/streams.py | 22 - 6 files changed, 894 insertions(+), 29 deletions(-) create mode 100644 tap_sunwave/schemas/census.json create mode 100644 tap_sunwave/schemas/opportunity_timeline.json diff --git a/tap_sunwave/client.py b/tap_sunwave/client.py index c155f02..1c67af6 100644 --- a/tap_sunwave/client.py +++ b/tap_sunwave/client.py @@ -27,6 +27,16 @@ class SunwaveStream(RESTStream): """Sunwave stream class.""" url_base = "https://emr.sunwavehealth.com/SunwaveEMR" + auth_errors = 0 + schema_filepath = SCHEMAS_DIR / f"{name}.json" + + + def _request(self, prepared_request: requests.PreparedRequest, context: Context | None) -> requests.Response: + """Auth wasn't getting ran for every retried request, so we need to do it here""" + # Manually run authenticator before sending the request + reauthed_request = self.authenticator.authenticate_request(prepared_request) + # Then call the parent's method with our newly authenticated request + return super()._request(reauthed_request, context) @cached_property def authenticator(self) -> Auth: @@ -71,17 +81,14 @@ def validate_response(self, response: requests.Response) -> None: # Check if response is a dict and has error if isinstance(json_response, dict) and json_response.get("error"): error_msg = self.response_error_message(response) - # TODO: figure out how to handle this better - if self.name == "opportunity_timeline": - log_msg = f"Skipping 'opportunity_timeline' record due to {error_msg}" - self.logger.info(log_msg) - else: - raise FatalAPIError(error_msg) + raise FatalAPIError(error_msg) except requests.exceptions.JSONDecodeError as e: # Their API returns a 200 status code when there's an error # We detect that by noticing the response isn't valid JSON - msg self.response_error_message(response) + msg = self.response_error_message(response) raise FatalAPIError(msg) + # Reset auth errors + self.auth_errors = 0 def _cleanup_schema(self, schema_fragment): if isinstance(schema_fragment, dict): diff --git a/tap_sunwave/schemas/census.json b/tap_sunwave/schemas/census.json new file mode 100644 index 0000000..7c478a8 --- /dev/null +++ b/tap_sunwave/schemas/census.json @@ -0,0 +1,641 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": [ + "object", + "null" + ], + "properties": { + "Account Id": { + "type": [ + "string", + "null" + ] + }, + "Account Legacy Id": { + "type": [ + "string", + "null" + ] + }, + "Admission Date": { + "type": [ + "string", + "null" + ] + }, + "Admission Id": { + "type": [ + "string", + "null" + ] + }, + "Admission Level Of Care": { + "type": [ + "string", + "null" + ] + }, + "Admission Service Facility": { + "type": [ + "string", + "null" + ] + }, + "Admission Time": { + "type": [ + "string", + "null" + ] + }, + "Admissions Rep": { + "type": [ + "string", + "null" + ] + }, + "Age": { + "type": [ + "string", + "null" + ] + }, + "Allergy Risk": { + "type": [ + "string", + "null" + ] + }, + "Assigned Therapist": { + "type": [ + "string", + "null" + ] + }, + "Assigned Therapist Email": { + "type": [ + "string", + "null" + ] + }, + "Authorization Code": { + "type": [ + "string", + "null" + ] + }, + "Bd Rep": { + "type": [ + "string", + "null" + ] + }, + "Bed Name": { + "type": [ + "string", + "null" + ] + }, + "Building Name": { + "type": [ + "string", + "null" + ] + }, + "Business Development Rep ": { + "type": [ + "string", + "null" + ] + }, + "Caller Phone Home": { + "type": [ + "string", + "null" + ] + }, + "Caller Phone Mobile": { + "type": [ + "string", + "null" + ] + }, + "Campaign Id": { + "type": [ + "string", + "null" + ] + }, + "Case Manager": { + "type": [ + "string", + "null" + ] + }, + "Collector": { + "type": [ + "string", + "null" + ] + }, + "Communication Type": { + "type": [ + "string", + "null" + ] + }, + "Culture": { + "type": [ + "string", + "null" + ] + }, + "Current Level Of Care": { + "type": [ + "string", + "null" + ] + }, + "Current Service Facility Name": { + "type": [ + "string", + "null" + ] + }, + "Diagnosis": { + "type": [ + "string", + "null" + ] + }, + "Dietary Risk": { + "type": [ + "string", + "null" + ] + }, + "Discharge Date": { + "type": [ + "string", + "null" + ] + }, + "Discharge Hour": { + "type": [ + "string", + "null" + ] + }, + "Discharge Type": { + "type": [ + "string", + "null" + ] + }, + "Do Not Readmit": { + "type": [ + "string", + "null" + ] + }, + "Do Not Readmit Before Consult": { + "type": [ + "string", + "null" + ] + }, + "Do Not Readmit Before Consult Comment": { + "type": [ + "string", + "null" + ] + }, + "Do Not Readmit Comment": { + "type": [ + "string", + "null" + ] + }, + "Drug Of Choice ": { + "type": [ + "string", + "null" + ] + }, + "End On": { + "type": [ + "string", + "null" + ] + }, + "Financial Class Name": { + "type": [ + "string", + "null" + ] + }, + "First Contact": { + "type": [ + "string", + "null" + ] + }, + "High Risk": { + "type": [ + "string", + "null" + ] + }, + "Insurance Group Number": { + "type": [ + "string", + "null" + ] + }, + "Insurance Name": { + "type": [ + "string", + "null" + ] + }, + "Last Level Of Care": { + "type": [ + "string", + "null" + ] + }, + "Last Service Facility": { + "type": [ + "string", + "null" + ] + }, + "Length Of Stay": { + "type": [ + "string", + "null" + ] + }, + "Level Of Care Id": { + "type": [ + "string", + "null" + ] + }, + "Levels Of Care List": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "start_on": { + "type": [ + "string", + "null" + ] + }, + "level_of_care": { + "type": [ + "string", + "null" + ] + }, + "service_facility_id": { + "type": [ + "string", + "null" + ] + }, + "end_on": { + "type": [ + "string", + "null" + ] + }, + "service_facility": { + "type": [ + "string", + "null" + ] + }, + "level_of_care_id": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "Marital Status": { + "type": [ + "string", + "null" + ] + }, + "Member Id": { + "type": [ + "string", + "null" + ] + }, + "Mri": { + "type": [ + "string", + "null" + ] + }, + "Nickname ": { + "type": [ + "string", + "null" + ] + }, + "Nurse": { + "type": [ + "string", + "null" + ] + }, + "Opportunity Legacy Id": { + "type": [ + "string", + "null" + ] + }, + "Opportunityid": { + "type": [ + "string", + "null" + ] + }, + "Patient Address1": { + "type": [ + "string", + "null" + ] + }, + "Patient City": { + "type": [ + "string", + "null" + ] + }, + "Patient Date Of Birth": { + "type": [ + "string", + "null" + ] + }, + "Patient Email": { + "type": [ + "string", + "null" + ] + }, + "Patient First Name": { + "type": [ + "string", + "null" + ] + }, + "Patient Gender Code": { + "type": [ + "string", + "null" + ] + }, + "Patient Id": { + "type": [ + "string", + "null" + ] + }, + "Patient Last Name": { + "type": [ + "string", + "null" + ] + }, + "Patient Middle Name": { + "type": [ + "string", + "null" + ] + }, + "Patient Name": { + "type": [ + "string", + "null" + ] + }, + "Patient Phone Home": { + "type": [ + "string", + "null" + ] + }, + "Patient Phone Mobile": { + "type": [ + "string", + "null" + ] + }, + "Patient State": { + "type": [ + "string", + "null" + ] + }, + "Patient Zip": { + "type": [ + "string", + "null" + ] + }, + "Payment Method": { + "type": [ + "string", + "null" + ] + }, + "Planned Discharge": { + "type": [ + "string", + "null" + ] + }, + "Policy Type": { + "type": [ + "string", + "null" + ] + }, + "Preadmission Notes ": { + "type": [ + "string", + "null" + ] + }, + "Previous Admission Id": { + "type": [ + "string", + "null" + ] + }, + "Primary Counselor": { + "type": [ + "string", + "null" + ] + }, + "Program": { + "type": [ + "string", + "null" + ] + }, + "Provider": { + "type": [ + "string", + "null" + ] + }, + "Race": { + "type": [ + "string", + "null" + ] + }, + "Realm": { + "type": [ + "string", + "null" + ] + }, + "Referral Destination": { + "type": [ + "string", + "null" + ] + }, + "Referral Source": { + "type": [ + "string", + "null" + ] + }, + "Referral Source Account Owner": { + "type": [ + "string", + "null" + ] + }, + "Referral Source Contact": { + "type": [ + "string", + "null" + ] + }, + "Referral Source Id": { + "type": [ + "string", + "null" + ] + }, + "Religion ": { + "type": [ + "string", + "null" + ] + }, + "Room Name": { + "type": [ + "string", + "null" + ] + }, + "Secondary Insurance Group Number": { + "type": [ + "string", + "null" + ] + }, + "Secondary Insurance Name": { + "type": [ + "string", + "null" + ] + }, + "Secondary Member Id": { + "type": [ + "string", + "null" + ] + }, + "Secondary Submit Name": { + "type": [ + "string", + "null" + ] + }, + "Secondary Term Date ": { + "type": [ + "string", + "null" + ] + }, + "Service Facility Id": { + "type": [ + "string", + "null" + ] + }, + "Start On": { + "type": [ + "string", + "null" + ] + }, + "Submit Name": { + "type": [ + "string", + "null" + ] + }, + "Target Service Facility": { + "type": [ + "string", + "null" + ] + }, + "Temporal Detox": { + "type": [ + "string", + "null" + ] + }, + "Term Date ": { + "type": [ + "string", + "null" + ] + }, + "Transfer From": { + "type": [ + "string", + "null" + ] + }, + "Transferred": { + "type": [ + "string", + "null" + ] + } + } +} \ No newline at end of file diff --git a/tap_sunwave/schemas/form.json b/tap_sunwave/schemas/form.json index fea4b44..aec60dc 100644 --- a/tap_sunwave/schemas/form.json +++ b/tap_sunwave/schemas/form.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/schema#", "type": [ "object", "null" diff --git a/tap_sunwave/schemas/opportunity.json b/tap_sunwave/schemas/opportunity.json index 3677ac9..3a5caaf 100644 --- a/tap_sunwave/schemas/opportunity.json +++ b/tap_sunwave/schemas/opportunity.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/schema#", "type": [ "object", "null" diff --git a/tap_sunwave/schemas/opportunity_timeline.json b/tap_sunwave/schemas/opportunity_timeline.json new file mode 100644 index 0000000..bb3008e --- /dev/null +++ b/tap_sunwave/schemas/opportunity_timeline.json @@ -0,0 +1,237 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": [ + "object", + "null" + ], + "properties": { + "workflow_status": { + "type": [ + "string", + "null" + ] + }, + "from_phone": { + "type": [ + "string", + "null" + ] + }, + "associated_with": { + "type": [ + "string", + "null" + ] + }, + "associated_with_id": { + "type": [ + "string", + "null" + ] + }, + "task_subject": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "created_by_name": { + "type": [ + "string", + "null" + ] + }, + "contact_id": { + "type": [ + "string", + "null" + ] + }, + "contact_email": { + "type": [ + "null", + "string" + ] + }, + "related_task_id": { + "type": [ + "null", + "string" + ] + }, + "campaign_name": { + "type": [ + "string", + "null" + ] + }, + "contact_type": { + "type": [ + "string", + "null" + ] + }, + "google_reminders": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "string", + "null" + ] + }, + "text": { + "type": [ + "string", + "null" + ] + }, + "reminder_date_time": { + "type": [ + "string", + "null" + ] + }, + "contact_phones": { + "type": [ + "string", + "null" + ] + }, + "assigned_to": { + "type": [ + "string", + "null" + ] + }, + "call_outcome": { + "type": [ + "null", + "string" + ] + }, + "task_status": { + "type": [ + "string", + "null" + ] + }, + "call_direction": { + "type": [ + "string", + "null" + ] + }, + "editable": { + "type": [ + "string", + "null" + ] + }, + "task_due_date": { + "type": [ + "string", + "null" + ] + }, + "activity_date": { + "type": [ + "string", + "null" + ] + }, + "google_calendar_link": { + "type": [ + "string", + "null" + ] + }, + "other_log_type": { + "type": [ + "null", + "string" + ] + }, + "created_by": { + "type": [ + "null", + "string" + ] + }, + "task_type_id": { + "type": [ + "string", + "null" + ] + }, + "created_on": { + "type": [ + "string", + "null" + ] + }, + "associated_contacts": { + "type": [ + "array", + "null" + ] + }, + "contact_display": { + "type": [ + "null", + "string" + ] + }, + "caller_id": { + "type": [ + "string", + "null" + ] + }, + "to_phone": { + "type": [ + "string", + "null" + ] + }, + "task_completed_on": { + "type": [ + "string", + "null" + ] + }, + "task_type": { + "type": [ + "string", + "null" + ] + }, + "assigned_to_name": { + "type": [ + "string", + "null" + ] + }, + "entity": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "null", + "null" + ] + } + } +} \ No newline at end of file diff --git a/tap_sunwave/streams.py b/tap_sunwave/streams.py index 2a4d972..68c0b0b 100644 --- a/tap_sunwave/streams.py +++ b/tap_sunwave/streams.py @@ -18,7 +18,6 @@ class UserStream(SunwaveStream): path = "/api/users" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = None - schema_filepath = SCHEMAS_DIR / f"{name}.json" class ReferralStream(SunwaveStream): @@ -26,7 +25,6 @@ class ReferralStream(SunwaveStream): path = "/api/referrals/status/{status}" primary_keys: t.ClassVar[list[str]] = ["id"] replication_key = None - schema_filepath = SCHEMAS_DIR / f"{name}.json" class FormsStream(SunwaveStream): @@ -36,14 +34,6 @@ class FormsStream(SunwaveStream): path = "/api/forms" primary_keys = ["id"] replication_key = None - - @property - def schema(self): - """ - Retrieve the schema for Forms directly from Swagger at: - #/components/schemas/FormStandardResponse - """ - return self._get_swagger_schema("#/components/schemas/FormStandardResponse") class OpportunitiesStream(SunwaveStream): @@ -53,10 +43,6 @@ class OpportunitiesStream(SunwaveStream): name = "opportunity" primary_keys = ["opportunity_id"] replication_key = None - - @property - def schema(self): - return self._get_swagger_schema("#/components/schemas/Opportunities") @property def path(self): @@ -83,10 +69,6 @@ class OpportunityTimelineStream(SunwaveStream): replication_key = None parent_stream_type = OpportunitiesStream path = "/api/opportunities/{opportunity_id}/timeline" - - @property - def schema(self): - return self._get_swagger_schema("#/components/schemas/OpportunitiesTimeline") def parse_response(self, response: requests.Response) -> t.Iterable[dict]: """Parse the response and return an iterator of result records. @@ -120,8 +102,4 @@ def path(self): start=start_date.strftime('%Y-%m-%d'), end=end_date.strftime('%Y-%m-%d') ) - - @property - def schema(self): - return self._get_swagger_schema("#/components/schemas/Census") \ No newline at end of file From e0c6da98e0edd640e16e9bb853e25056c8f3d69b Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Thu, 16 Jan 2025 15:30:58 -0500 Subject: [PATCH 3/4] Static schemas defined in one place --- meltano.yml | 2 -- tap_sunwave/client.py | 26 ++++---------------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/meltano.yml b/meltano.yml index 4d463f5..8c4bf86 100644 --- a/meltano.yml +++ b/meltano.yml @@ -21,8 +21,6 @@ plugins: - name: client_secret sensitive: true - name: clinic_id - select: - - opportunity_timeline.* loaders: - name: target-jsonl variant: andyh1203 diff --git a/tap_sunwave/client.py b/tap_sunwave/client.py index 1c67af6..5badb0b 100644 --- a/tap_sunwave/client.py +++ b/tap_sunwave/client.py @@ -28,8 +28,9 @@ class SunwaveStream(RESTStream): url_base = "https://emr.sunwavehealth.com/SunwaveEMR" auth_errors = 0 - schema_filepath = SCHEMAS_DIR / f"{name}.json" - + @property + def schema_filepath(self): + return SCHEMAS_DIR / f"{self.name}.json" def _request(self, prepared_request: requests.PreparedRequest, context: Context | None) -> requests.Response: """Auth wasn't getting ran for every retried request, so we need to do it here""" @@ -107,23 +108,4 @@ def _cleanup_schema(self, schema_fragment): elif isinstance(schema_fragment, list): for item in schema_fragment: - self._cleanup_schema(item) - - def _get_swagger_schema(self, ref: str) -> dict: - """Get schema from packaged swagger file.""" - # Use package resources to access swagger.json - with resources.files(__package__).joinpath("docs/swagger.json").open("r", encoding="utf-8") as f: - swagger_doc = json.load(f) - - # Example: ref => "#/components/schemas/FormStandardResponse" - # Strip the leading "#/" and split by "/" - path_parts = ref.lstrip("#/").split("/") - - # Traverse the swagger doc to get the nested object - data = swagger_doc - for part in path_parts: - data = data[part] - - cleaned_schema = self._cleanup_schema(data["properties"]) - assert len(data["properties"]) > 1, "No properties found in schema" - return data \ No newline at end of file + self._cleanup_schema(item) \ No newline at end of file From 2c907b1885f58ba515138d1eeb4a036e2fd38427 Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Thu, 16 Jan 2025 15:32:48 -0500 Subject: [PATCH 4/4] Fix type for opportunity_timeline --- tap_sunwave/schemas/opportunity_timeline.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tap_sunwave/schemas/opportunity_timeline.json b/tap_sunwave/schemas/opportunity_timeline.json index bb3008e..09b944d 100644 --- a/tap_sunwave/schemas/opportunity_timeline.json +++ b/tap_sunwave/schemas/opportunity_timeline.json @@ -1,5 +1,4 @@ { - "$schema": "http://json-schema.org/schema#", "type": [ "object", "null" @@ -230,7 +229,7 @@ "status": { "type": [ "null", - "null" + "string" ] } }