diff --git a/component_config/configSchema.json b/component_config/configSchema.json index 878a433..2c51785 100644 --- a/component_config/configSchema.json +++ b/component_config/configSchema.json @@ -78,7 +78,18 @@ }, "required": true, "description": "Full Sync downloads all data from the source every run, Incremental Sync downloads data (tickets, ticket_comments and ticket_audits) by parameter start_time described here. The start time is taken from the last successful run.", - "propertyOrder": 20 + "propertyOrder": 10 + }, + "date_from": { + "propertyOrder": 20, + "type": "string", + "title": "From date", + "description": "Date from. Date in YYYY-MM-DD format or a string i.e. 5 days ago, 1 month ago, yesterday, etc. If left empty, all records from 2000-01-01 are downloaded.", + "options": { + "dependencies": { + "sync_mode": "incremental_sync" + } + } } } }, diff --git a/src/component.py b/src/component.py index 8ff7feb..3202693 100644 --- a/src/component.py +++ b/src/component.py @@ -3,6 +3,7 @@ from collections import OrderedDict from typing import List +import dateparser import dlt from dlt.common import pendulum from dlt.common.time import ensure_pendulum_datetime @@ -45,7 +46,7 @@ def run(self): # get the previous start time if self.params.sync_options.is_incremental: - previous_start: int = self.get_state_file().get("time", {}).get("previousStart", DEFAULT_START_DATE) + previous_start = self._def_start_timestamp() logging.info("Incremental mode") else: previous_start = DEFAULT_START_DATE @@ -72,6 +73,23 @@ def run(self): logging.info(f"Saving the state file with the actual start date {actual_start}") self.write_state_file({"time": {"previousStart": actual_start}}) + def _def_start_timestamp(self): + if self.params.sync_options.date_from: + return self._parse_date(self.params.sync_options.date_from) + else: + return int(self.get_state_file().get("time", {}).get("previousStart", DEFAULT_START_DATE)) + + @staticmethod + def _parse_date(date_to_parse: str) -> int: + try: + parsed_date = dateparser.parse(date_to_parse) + if parsed_date.tzinfo is None: + parsed_date = parsed_date.replace(tzinfo=pendulum.UTC) + return int(parsed_date.timestamp()) + except (AttributeError, TypeError) as err: + raise UserException(f"Failed to parse date {date_to_parse}, make sure the date is either in YYYY-MM-DD " + f"format or relative date i.e. 5 days ago, 1 month ago, yesterday, etc.") from err + def _set_dlt(self): # prepare the temporary directories os.makedirs(DLT_TMP_DIR, exist_ok=True) diff --git a/src/configuration.py b/src/configuration.py index 3ee6e49..8b31ec9 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -12,6 +12,7 @@ class Authentication(BaseModel): class SyncOptions(BaseModel): sync_mode: str + date_from: str = Field(default=None) @computed_field def is_incremental(self) -> bool: