Skip to content

Commit

Permalink
Move retry to parameter, fix wait time flag
Browse files Browse the repository at this point in the history
  • Loading branch information
Erik Johnson committed Nov 30, 2024
1 parent d33a9b6 commit 82db74d
Showing 6 changed files with 15 additions and 13 deletions.
1 change: 1 addition & 0 deletions bdfr/__main__.py
Original file line number Diff line number Diff line change
@@ -51,6 +51,7 @@
_downloader_options = [
click.option("--make-hard-links", is_flag=True, default=None),
click.option("--max-wait-time", type=int, default=None),
click.option("--max-wait-retry", type=int, default=None),
click.option("--no-dupes", is_flag=True, default=None),
click.option("--search-existing", is_flag=True, default=None),
click.option("--skip", default=None, multiple=True),
6 changes: 3 additions & 3 deletions bdfr/archiver.py
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl

def download(self):
for generator in self.reddit_lists:
for retry in range(5):
for retry in range(self.args.max_wait_retry or 1):
try:
for submission in generator:
try:
@@ -53,8 +53,8 @@ def download(self):
break
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
sleep(self.args.max_wait_time)

def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = []
6 changes: 3 additions & 3 deletions bdfr/cloner.py
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl

def download(self):
for generator in self.reddit_lists:
for retry in range(5):
for retry in range(self.args.max_wait_retry or 1):
try:
for submission in generator:
try:
@@ -31,5 +31,5 @@ def download(self):
break
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
sleep(self.args.max_wait_time)
3 changes: 2 additions & 1 deletion bdfr/configuration.py
Original file line number Diff line number Diff line change
@@ -31,7 +31,8 @@ def __init__(self):
self.link: list[str] = []
self.log: Optional[str] = None
self.make_hard_links = False
self.max_wait_time = None
self.max_wait_time = 60
self.max_wait_retry = 5
self.multireddit: list[str] = []
self.no_dupes: bool = False
self.saved: bool = False
6 changes: 3 additions & 3 deletions bdfr/connector.py
Original file line number Diff line number Diff line change
@@ -362,7 +362,7 @@ def get_user_data(self) -> list[Iterator]:
return []
generators = []
for user in self.args.user:
for retry in range(5):
for retry in range(self.args.max_wait_retry or 1):
try:
try:
self.check_user_existence(user)
@@ -388,8 +388,8 @@ def get_user_data(self) -> list[Iterator]:
break
except prawcore.PrawcoreException as e:
logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
sleep(self.args.max_wait_time)
return generators
else:
return []
6 changes: 3 additions & 3 deletions bdfr/downloader.py
Original file line number Diff line number Diff line change
@@ -44,7 +44,7 @@ def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handl

def download(self):
for generator in self.reddit_lists:
for retry in range(5):
for retry in range(self.args.max_wait_retry or 1):
try:
for submission in generator:
try:
@@ -54,8 +54,8 @@ def download(self):
break
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
logger.debug(f"Waiting {self.args.max_wait_time} seconds to continue")
sleep(self.args.max_wait_time)

def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids:

0 comments on commit 82db74d

Please sign in to comment.