diff --git a/docs/configuration.rst b/docs/configuration.rst index e3388f778e..43c453e9fb 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2415,6 +2415,16 @@ Description Extract `TwitPic `__ embeds. +extractor.twitter.unique +------------------------ +Type + ``bool`` +Default + ``true`` +Description + Ignore previously seen Tweets. + + extractor.twitter.users ----------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index d447bdbfa6..1492653493 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -288,6 +288,7 @@ "strategy": null, "text-tweets": false, "twitpic": false, + "unique": true, "users": "timeline", "videos": true }, diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 33e9e10d33..fc81f1cd44 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -64,6 +64,11 @@ def items(self): tweets = self._expand_tweets(self.tweets()) self.tweets = lambda : tweets + if self.config("unique", True): + seen_tweets = set() + else: + seen_tweets = None + for tweet in self.tweets(): if "legacy" in tweet: @@ -71,6 +76,11 @@ def items(self): else: data = tweet + if seen_tweets is not None: + if data["id_str"] in seen_tweets: + continue + seen_tweets.add(data["id_str"]) + if not self.retweets and "retweeted_status_id_str" in data: self.log.debug("Skipping %s (retweet)", data["id_str"]) continue