From 630c02fac3cfaec0ec22c639d83576b9ed96c350 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Thu, 22 Feb 2024 03:35:57 -0800 Subject: [PATCH 01/54] wip: initial commit --- application/build.gradle | 3 + .../org/togetherjava/tjbot/config/Config.java | 16 ++ .../togetherjava/tjbot/config/RSSFeed.java | 13 ++ .../features/javamail/JavaMailRSSRoutine.java | 167 ++++++++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java create mode 100644 application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java diff --git a/application/build.gradle b/application/build.gradle index 4d88d97b39..9380b24de3 100644 --- a/application/build.gradle +++ b/application/build.gradle @@ -77,6 +77,9 @@ dependencies { implementation 'org.kohsuke:github-api:1.321' + implementation 'org.apache.commons:commons-text:1.11.0' + implementation 'com.apptasticsoftware:rssreader:3.6.0' + testImplementation 'org.mockito:mockito-core:5.11.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0' testImplementation 'org.junit.jupiter:junit-jupiter-params:5.10.0' diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index d14c6279d0..8f6ae5b130 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -42,6 +42,8 @@ public final class Config { private final String sourceCodeBaseUrl; private final JShellConfig jshell; private final FeatureBlacklistConfig featureBlacklistConfig; + private final List rssFeeds; + private final String javaNewsChannelPattern; private final String selectRolesChannelPattern; private final String memberCountCategoryPattern; @@ -90,6 +92,10 @@ private Config(@JsonProperty(value = "token", required = true) String token, required = true) String memberCountCategoryPattern, @JsonProperty(value = "featureBlacklist", required = true) FeatureBlacklistConfig featureBlacklistConfig, + @JsonProperty(value = "javaNewsChannelPattern", + required = true) String javaNewsChannelPattern, + @JsonProperty(value = "rssFeeds", + required = true) List rssFeeds, @JsonProperty(value = "selectRolesChannelPattern", required = true) String selectRolesChannelPattern) { this.token = Objects.requireNonNull(token); @@ -122,6 +128,8 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.sourceCodeBaseUrl = Objects.requireNonNull(sourceCodeBaseUrl); this.jshell = Objects.requireNonNull(jshell); this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); + this.rssFeeds = Objects.requireNonNull(rssFeeds); + this.javaNewsChannelPattern = Objects.requireNonNull(javaNewsChannelPattern); this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); } @@ -405,4 +413,12 @@ public String getSelectRolesChannelPattern() { public String getMemberCountCategoryPattern() { return memberCountCategoryPattern; } + + public String getJavaNewsChannelPattern() { + return javaNewsChannelPattern; + } + + public List getRssFeeds() { + return rssFeeds; + } } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java new file mode 100644 index 0000000000..5068a46b09 --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -0,0 +1,13 @@ +package org.togetherjava.tjbot.config; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +public record RSSFeed( + @JsonProperty(value = "url", required = true) String url) { + + public RSSFeed { + Objects.requireNonNull(url); + } +} diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java new file mode 100644 index 0000000000..55c7b3085b --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -0,0 +1,167 @@ +package org.togetherjava.tjbot.features.javamail; + +import com.apptasticsoftware.rssreader.Item; +import com.apptasticsoftware.rssreader.RssReader; +import com.ctc.wstx.shaded.msv.org_isorelax.verifier.impl.SAXEventGenerator; +import net.dv8tion.jda.api.EmbedBuilder; +import net.dv8tion.jda.api.JDA; +import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; +import org.apache.commons.text.StringEscapeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.togetherjava.tjbot.config.Config; +import org.togetherjava.tjbot.config.RSSFeed; +import org.togetherjava.tjbot.features.Routine; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public final class JavaMailRSSRoutine implements Routine { + + private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); + private static final RssReader RSS_READER = new RssReader(); + private static Predicate javaNewsChannelPredicate; + private final List feeds; + + public JavaMailRSSRoutine(Config config) { + this.feeds = config.getRssFeeds(); + + javaNewsChannelPredicate = + Pattern.compile(config.getJavaNewsChannelPattern()).asPredicate(); + } + + @Override + public Schedule createSchedule() { + // TODO: Make this adjustable in the future + return new Schedule(ScheduleMode.FIXED_DELAY, 0, 10, TimeUnit.MINUTES); + } + + @Override + public void runRoutine(JDA jda) { + // TODO: REMOVE ME + // 1. make http request and check if the latest rss feed id matches last saved RSS feed id + // 2. if it doesn't then post the new one and store it in db. + + final TextChannel channel = jda.getTextChannelCache() + .stream() + .filter(c -> javaNewsChannelPredicate.test(c.getName())) + .findFirst() + .orElse(null); + + if (channel == null) { + logger.warn("Could not find the Java news channel"); + return; + } + + // parse latest channel message and get the date that the rss feed from that is from. + // footer format is {DATE} | https://wiki.openjdk.org (RSS CHANNEL ID) + List lastDates = channel.getIterableHistory() + .stream() + .filter(message -> message.getAuthor().getId().equals(jda.getSelfUser().getId())) + .map(message -> message.getEmbeds().getFirst().getFooter().getText().split(" | ")) + .filter(footer -> Arrays.stream(footer).findAny().isPresent()) + .collect(Collectors.toList()); + + // loop through lastDates and check if we are still tracking that rss feed and then call + // getAfter + for (String[] footer : lastDates) { + RSSFeed feed = null; + if (footer.length != 2) { + continue; + } + + // check if we are still tracking + long date = parseDate(footer[0]); + for (RSSFeed trackedFeed : feeds) { + // they likely won't match exactly. + if (trackedFeed.url().contains(footer[1])) { + feed = trackedFeed; + } + } + + if (feed == null) { + continue; + } + + // now we can handle all posts for this specific rss feed + List posts = getAfter(feed.url(), date); + } + + + } + + private static void sendRSSPost() { + + } + + @SuppressWarnings("static-access") + private static EmbedBuilder constructEmbedMessage(Item item, TextChannel channel) { + final EmbedBuilder embedBuilder = new EmbedBuilder(); + + // TODO: You do this stuff + embedBuilder.setTitle(item.getTitle().get(), item.getLink().get()); + embedBuilder.setDescription( + StringEscapeUtils.unescapeHtml4(item.getDescription().get().substring(0, 150)) + + "..."); + embedBuilder + .setFooter("%s | %d".format(item.getPubDate().get(), item.getChannel().getLink())); + return embedBuilder; + } + + + private static List getAfter(String rssUrl, long date) { + List rssList = fetchRss(rssUrl).orElse(null); + final Predicate rssListPredicate = item -> { + var pubDateTime = item.getPubDate(); + if (pubDateTime.isEmpty()) { + return false; + } + + long pubDate = parseDate(pubDateTime.get()); + + return pubDate > date; + }; + + if (rssList == null) { + return List.of(); + } + + return rssList.stream().filter(rssListPredicate).collect(Collectors.toList()); + } + + private static Optional getLatest(String rssUrl) throws IOException { + return fetchRss(rssUrl).orElseThrow().stream().findFirst(); + } + + /** + * Fetches new items from a given RSS url. + */ + private static Optional> fetchRss(String rssUrl) { + try { + return Optional.of(RSS_READER.read(rssUrl).collect(Collectors.toList())); + } catch (Exception e) { + logger.warn("Could not fetch RSS from URL ({})", rssUrl); + return Optional.empty(); + } + } + + private static long parseDate(String date) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + try { + return dateFormat.parse(date).getTime(); + } catch (Exception e) { + logger.error("Could not parse date, {}", e.getMessage()); + } + return 0; + } +} From 03036e443da1ad8353f4ad8e48c0dcbde6643cfb Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 13:53:10 +0200 Subject: [PATCH 02/54] style(spotlessApply): run uncalled task --- .../src/main/java/org/togetherjava/tjbot/config/Config.java | 5 ++--- .../src/main/java/org/togetherjava/tjbot/config/RSSFeed.java | 3 +-- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 8f6ae5b130..e2aa4ba1be 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -93,9 +93,8 @@ private Config(@JsonProperty(value = "token", required = true) String token, @JsonProperty(value = "featureBlacklist", required = true) FeatureBlacklistConfig featureBlacklistConfig, @JsonProperty(value = "javaNewsChannelPattern", - required = true) String javaNewsChannelPattern, - @JsonProperty(value = "rssFeeds", - required = true) List rssFeeds, + required = true) String javaNewsChannelPattern, + @JsonProperty(value = "rssFeeds", required = true) List rssFeeds, @JsonProperty(value = "selectRolesChannelPattern", required = true) String selectRolesChannelPattern) { this.token = Objects.requireNonNull(token); diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index 5068a46b09..bb50150ab8 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -4,8 +4,7 @@ import java.util.Objects; -public record RSSFeed( - @JsonProperty(value = "url", required = true) String url) { +public record RSSFeed(@JsonProperty(value = "url", required = true) String url) { public RSSFeed { Objects.requireNonNull(url); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 55c7b3085b..92c7d5419d 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -2,23 +2,20 @@ import com.apptasticsoftware.rssreader.Item; import com.apptasticsoftware.rssreader.RssReader; -import com.ctc.wstx.shaded.msv.org_isorelax.verifier.impl.SAXEventGenerator; import net.dv8tion.jda.api.EmbedBuilder; import net.dv8tion.jda.api.JDA; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import org.apache.commons.text.StringEscapeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.togetherjava.tjbot.config.Config; import org.togetherjava.tjbot.config.RSSFeed; import org.togetherjava.tjbot.features.Routine; import java.io.IOException; -import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; From 28b6160a6c06e0f05fee931c8731f655ccd97160 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 18:51:23 +0200 Subject: [PATCH 03/54] fix: register routine in features list --- .../src/main/java/org/togetherjava/tjbot/features/Features.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/Features.java b/application/src/main/java/org/togetherjava/tjbot/features/Features.java index ea6b908b90..f47a9c3040 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/Features.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/Features.java @@ -24,6 +24,7 @@ import org.togetherjava.tjbot.features.github.GitHubCommand; import org.togetherjava.tjbot.features.github.GitHubReference; import org.togetherjava.tjbot.features.help.*; +import org.togetherjava.tjbot.features.javamail.JavaMailRSSRoutine; import org.togetherjava.tjbot.features.jshell.JShellCommand; import org.togetherjava.tjbot.features.jshell.JShellEval; import org.togetherjava.tjbot.features.mathcommands.TeXCommand; @@ -108,6 +109,7 @@ public static Collection createFeatures(JDA jda, Database database, Con features.add(new HelpThreadActivityUpdater(helpSystemHelper)); features.add(new HelpThreadAutoArchiver(helpSystemHelper)); features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem)); + features.add(new JavaMailRSSRoutine(config)); features.add(new MemberCountDisplayRoutine(config)); // Message receivers From b494cd3163a8f5c53035619dece3d147f3af7d93 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 18:54:59 +0200 Subject: [PATCH 04/54] feat: update config.json.template --- application/config.json.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/config.json.template b/application/config.json.template index b584f10f6a..456c69be30 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -105,4 +105,6 @@ }, "memberCountCategoryPattern": "Info", "selectRolesChannelPattern": "select-your-roles" + "javaNewsChannelPattern": "java-news-and-changes", + "rssFeeds": [] } From c746bfaf9100b665fd47ec64ab3bd4e9c2ded806 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 19:53:21 +0200 Subject: [PATCH 05/54] wip(rss-config): change config design --- application/config.json.template | 8 +- .../togetherjava/tjbot/config/RSSFeed.java | 5 +- .../features/javamail/JavaMailRSSRoutine.java | 94 +++++++------------ 3 files changed, 45 insertions(+), 62 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 456c69be30..7fceaa5711 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -105,6 +105,10 @@ }, "memberCountCategoryPattern": "Info", "selectRolesChannelPattern": "select-your-roles" - "javaNewsChannelPattern": "java-news-and-changes", - "rssFeeds": [] + "rssFeeds": [ + { + "url": "", + "targetChannelPattern", "" + } + ] } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index bb50150ab8..9fb0ea2876 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -4,9 +4,12 @@ import java.util.Objects; -public record RSSFeed(@JsonProperty(value = "url", required = true) String url) { +public record RSSFeed(@JsonProperty(value = "url", required = true) String url, + @JsonProperty(value = "targetChannelPattern", + required = true) String targetChannelPattern) { public RSSFeed { Objects.requireNonNull(url); + Objects.requireNonNull(targetChannelPattern); } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 92c7d5419d..dccc1db59f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -4,6 +4,7 @@ import com.apptasticsoftware.rssreader.RssReader; import net.dv8tion.jda.api.EmbedBuilder; import net.dv8tion.jda.api.JDA; +import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import org.apache.commons.text.StringEscapeUtils; import org.slf4j.Logger; @@ -13,9 +14,11 @@ import org.togetherjava.tjbot.config.RSSFeed; import org.togetherjava.tjbot.features.Routine; +import javax.annotation.Nonnull; + import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -27,14 +30,16 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); - private static Predicate javaNewsChannelPredicate; private final List feeds; + private final List> targetChannelPatterns = new ArrayList<>(); public JavaMailRSSRoutine(Config config) { this.feeds = config.getRssFeeds(); - javaNewsChannelPredicate = - Pattern.compile(config.getJavaNewsChannelPattern()).asPredicate(); + this.feeds.forEach(feed -> { + var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); + targetChannelPatterns.add(predicate); + }); } @Override @@ -44,69 +49,39 @@ public Schedule createSchedule() { } @Override - public void runRoutine(JDA jda) { - // TODO: REMOVE ME - // 1. make http request and check if the latest rss feed id matches last saved RSS feed id - // 2. if it doesn't then post the new one and store it in db. + public void runRoutine(@Nonnull JDA jda) { + feeds.forEach(feed -> sendRSS(jda, feed)); + } - final TextChannel channel = jda.getTextChannelCache() - .stream() - .filter(c -> javaNewsChannelPredicate.test(c.getName())) - .findFirst() - .orElse(null); + private void sendRSS(JDA jda, RSSFeed feed) { + var textChannel = getTextChannelFromFeed(jda, feed); + var items = fetchRss(feed.url()); - if (channel == null) { - logger.warn("Could not find the Java news channel"); + if (textChannel.isEmpty()) { + logger.warn("Tried to sendRss, got empty response (channel {} not found)", + feed.targetChannelPattern()); return; } - // parse latest channel message and get the date that the rss feed from that is from. - // footer format is {DATE} | https://wiki.openjdk.org (RSS CHANNEL ID) - List lastDates = channel.getIterableHistory() - .stream() - .filter(message -> message.getAuthor().getId().equals(jda.getSelfUser().getId())) - .map(message -> message.getEmbeds().getFirst().getFooter().getText().split(" | ")) - .filter(footer -> Arrays.stream(footer).findAny().isPresent()) - .collect(Collectors.toList()); - - // loop through lastDates and check if we are still tracking that rss feed and then call - // getAfter - for (String[] footer : lastDates) { - RSSFeed feed = null; - if (footer.length != 2) { - continue; - } - - // check if we are still tracking - long date = parseDate(footer[0]); - for (RSSFeed trackedFeed : feeds) { - // they likely won't match exactly. - if (trackedFeed.url().contains(footer[1])) { - feed = trackedFeed; - } - } - - if (feed == null) { - continue; - } - - // now we can handle all posts for this specific rss feed - List posts = getAfter(feed.url(), date); - } - - + items.forEach(item -> { + MessageEmbed embed = constructEmbedMessage(item, textChannel.get()).build(); + textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); + }); } - private static void sendRSSPost() { - + private static Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { + return jda.getTextChannelCache() + .stream() + .filter(c -> feed.targetChannelPattern().equals(c.getName())) + .findFirst(); } @SuppressWarnings("static-access") private static EmbedBuilder constructEmbedMessage(Item item, TextChannel channel) { final EmbedBuilder embedBuilder = new EmbedBuilder(); - // TODO: You do this stuff embedBuilder.setTitle(item.getTitle().get(), item.getLink().get()); + // TODO: don't hardcode substring to 150 embedBuilder.setDescription( StringEscapeUtils.unescapeHtml4(item.getDescription().get().substring(0, 150)) + "..."); @@ -116,8 +91,9 @@ private static EmbedBuilder constructEmbedMessage(Item item, TextChannel channel } - private static List getAfter(String rssUrl, long date) { - List rssList = fetchRss(rssUrl).orElse(null); + // TODO: make this use DateTime + private static List getPostsAfterDate(String rssUrl, long date) { + List rssList = fetchRss(rssUrl); final Predicate rssListPredicate = item -> { var pubDateTime = item.getPubDate(); if (pubDateTime.isEmpty()) { @@ -137,18 +113,18 @@ private static List getAfter(String rssUrl, long date) { } private static Optional getLatest(String rssUrl) throws IOException { - return fetchRss(rssUrl).orElseThrow().stream().findFirst(); + return fetchRss(rssUrl).stream().findFirst(); } /** * Fetches new items from a given RSS url. */ - private static Optional> fetchRss(String rssUrl) { + private static List fetchRss(String rssUrl) { try { - return Optional.of(RSS_READER.read(rssUrl).collect(Collectors.toList())); + return RSS_READER.read(rssUrl).collect(Collectors.toList()); } catch (Exception e) { logger.warn("Could not fetch RSS from URL ({})", rssUrl); - return Optional.empty(); + return List.of(); } } From cbaa40b047666a789df6a4086deed3cc5f5585c0 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 20:07:40 +0200 Subject: [PATCH 06/54] feat: make targetChannelPattern into a Map --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index dccc1db59f..4d40202ce0 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -18,9 +18,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -31,14 +29,14 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); private final List feeds; - private final List> targetChannelPatterns = new ArrayList<>(); + private final Map> targetChannelPatterns = new HashMap<>(); public JavaMailRSSRoutine(Config config) { this.feeds = config.getRssFeeds(); this.feeds.forEach(feed -> { var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); - targetChannelPatterns.add(predicate); + targetChannelPatterns.put(feed, predicate); }); } @@ -69,10 +67,10 @@ private void sendRSS(JDA jda, RSSFeed feed) { }); } - private static Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { + private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { return jda.getTextChannelCache() .stream() - .filter(c -> feed.targetChannelPattern().equals(c.getName())) + .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) .findFirst(); } From dbcd59b0b36465dd86ad01e20b2e16adeddcd11a Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 20:09:56 +0200 Subject: [PATCH 07/54] refactor: don't hardcode number --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 4d40202ce0..7e0a978a60 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -28,6 +28,7 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); + private static final int MAX_CONTENTS = 150; private final List feeds; private final Map> targetChannelPatterns = new HashMap<>(); @@ -79,10 +80,8 @@ private static EmbedBuilder constructEmbedMessage(Item item, TextChannel channel final EmbedBuilder embedBuilder = new EmbedBuilder(); embedBuilder.setTitle(item.getTitle().get(), item.getLink().get()); - // TODO: don't hardcode substring to 150 - embedBuilder.setDescription( - StringEscapeUtils.unescapeHtml4(item.getDescription().get().substring(0, 150)) - + "..."); + embedBuilder.setDescription(StringEscapeUtils + .unescapeHtml4(item.getDescription().get().substring(0, MAX_CONTENTS)) + "..."); embedBuilder .setFooter("%s | %d".format(item.getPubDate().get(), item.getChannel().getLink())); return embedBuilder; From 6e5f34a1d9693a3a9b5b7b42ddeabd85993633bb Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 20:17:07 +0200 Subject: [PATCH 08/54] feat: improve constructEmbedMessage --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 7e0a978a60..52301838cc 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -6,6 +6,7 @@ import net.dv8tion.jda.api.JDA; import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +64,7 @@ private void sendRSS(JDA jda, RSSFeed feed) { } items.forEach(item -> { - MessageEmbed embed = constructEmbedMessage(item, textChannel.get()).build(); + MessageEmbed embed = constructEmbedMessage(item).build(); textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); }); } @@ -76,12 +77,14 @@ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { } @SuppressWarnings("static-access") - private static EmbedBuilder constructEmbedMessage(Item item, TextChannel channel) { + private static EmbedBuilder constructEmbedMessage(Item item) { final EmbedBuilder embedBuilder = new EmbedBuilder(); + String title = item.getTitle().orElse("No title"); + String rawDescription = item.getDescription().orElse("No description"); + String description = StringEscapeUtils.unescapeHtml4(rawDescription); - embedBuilder.setTitle(item.getTitle().get(), item.getLink().get()); - embedBuilder.setDescription(StringEscapeUtils - .unescapeHtml4(item.getDescription().get().substring(0, MAX_CONTENTS)) + "..."); + embedBuilder.setTitle(title, item.getLink().orElse("")); + embedBuilder.setDescription(StringUtils.abbreviate(description, MAX_CONTENTS)); embedBuilder .setFooter("%s | %d".format(item.getPubDate().get(), item.getChannel().getLink())); return embedBuilder; From 2d93e9f4fe8123a007a16dbc120653ca063db4bc Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 21:26:28 +0200 Subject: [PATCH 09/54] feat: improve the description constructing --- .../features/javamail/JavaMailRSSRoutine.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 52301838cc..8cb8566eb1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -6,8 +6,10 @@ import net.dv8tion.jda.api.JDA; import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; +import org.jooq.tools.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,21 +78,31 @@ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { .findFirst(); } - @SuppressWarnings("static-access") private static EmbedBuilder constructEmbedMessage(Item item) { final EmbedBuilder embedBuilder = new EmbedBuilder(); String title = item.getTitle().orElse("No title"); - String rawDescription = item.getDescription().orElse("No description"); - String description = StringEscapeUtils.unescapeHtml4(rawDescription); + String titleLink = item.getLink().orElse(""); + Optional rawDescription = item.getDescription(); + + embedBuilder.setTitle(title, titleLink); + + if (rawDescription.isPresent()) { + Document fullDescription = + Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.get())); + StringBuilder finalDescription = new StringBuilder(); + fullDescription.body().select("p").forEach(p -> { + finalDescription.append(p.text()).append(". "); + }); + + embedBuilder + .setDescription(StringUtils.abbreviate(finalDescription.toString(), MAX_CONTENTS)); + return embedBuilder; + } - embedBuilder.setTitle(title, item.getLink().orElse("")); - embedBuilder.setDescription(StringUtils.abbreviate(description, MAX_CONTENTS)); - embedBuilder - .setFooter("%s | %d".format(item.getPubDate().get(), item.getChannel().getLink())); + embedBuilder.setDescription("No description"); return embedBuilder; } - // TODO: make this use DateTime private static List getPostsAfterDate(String rssUrl, long date) { List rssList = fetchRss(rssUrl); From 2da57d6f356538b1e472f9e9c64bdad26fb42b8f Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 21:27:11 +0200 Subject: [PATCH 10/54] feat: increase MAX_CONTENTS to 300 --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 8cb8566eb1..1a246f5304 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -31,7 +31,7 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); - private static final int MAX_CONTENTS = 150; + private static final int MAX_CONTENTS = 300; private final List feeds; private final Map> targetChannelPatterns = new HashMap<>(); From 066b2e9c624b6ecbdbb02cf4aa4b4c3636ada515 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 21:40:45 +0200 Subject: [PATCH 11/54] wip: changes --- .../src/main/java/org/togetherjava/tjbot/config/Config.java | 4 ++++ .../tjbot/features/javamail/JavaMailRSSRoutine.java | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index e2aa4ba1be..90775ed01a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Objects; + /** * Configuration of the application. Create instances using {@link #load(Path)}. */ @@ -44,6 +45,7 @@ public final class Config { private final FeatureBlacklistConfig featureBlacklistConfig; private final List rssFeeds; private final String javaNewsChannelPattern; + private final int rssPollInterval; private final String selectRolesChannelPattern; private final String memberCountCategoryPattern; @@ -95,6 +97,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, @JsonProperty(value = "javaNewsChannelPattern", required = true) String javaNewsChannelPattern, @JsonProperty(value = "rssFeeds", required = true) List rssFeeds, + @JsonProperty(value = "rssPollInterval", required = true) int rssPollInterval, @JsonProperty(value = "selectRolesChannelPattern", required = true) String selectRolesChannelPattern) { this.token = Objects.requireNonNull(token); @@ -129,6 +132,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); this.rssFeeds = Objects.requireNonNull(rssFeeds); this.javaNewsChannelPattern = Objects.requireNonNull(javaNewsChannelPattern); + this.rssPollInterval = Objects.requireNonNull(rssPollInterval); this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 1a246f5304..7de1a1ed1a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -1,5 +1,6 @@ package org.togetherjava.tjbot.features.javamail; +import com.apptasticsoftware.rssreader.DateTime; import com.apptasticsoftware.rssreader.Item; import com.apptasticsoftware.rssreader.RssReader; import net.dv8tion.jda.api.EmbedBuilder; @@ -104,7 +105,7 @@ private static EmbedBuilder constructEmbedMessage(Item item) { } // TODO: make this use DateTime - private static List getPostsAfterDate(String rssUrl, long date) { + private static List getPostsAfterDate(String rssUrl, DateTime date) { List rssList = fetchRss(rssUrl); final Predicate rssListPredicate = item -> { var pubDateTime = item.getPubDate(); From 728d08b15493b4f2a6fc7f6639efa425ee078017 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Thu, 22 Feb 2024 11:58:11 -0800 Subject: [PATCH 12/54] feat(rss-feed): convert polling interval to be configurable --- application/config.json.template | 3 ++- .../org/togetherjava/tjbot/config/Config.java | 5 +++++ .../features/javamail/JavaMailRSSRoutine.java | 19 ++++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 7fceaa5711..a60bbe6d5b 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -103,8 +103,9 @@ "special": [ ] }, + "selectRolesChannelPattern": "select-your-roles", + "rssPollInterval": 5, "memberCountCategoryPattern": "Info", - "selectRolesChannelPattern": "select-your-roles" "rssFeeds": [ { "url": "", diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 90775ed01a..ee9833e817 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -424,4 +424,9 @@ public String getJavaNewsChannelPattern() { public List getRssFeeds() { return rssFeeds; } + + // TODO: Make JavaDocs on this + public int getRssPollInterval() { + return rssPollInterval; + } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 7de1a1ed1a..f11d37a1d5 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -35,9 +35,11 @@ public final class JavaMailRSSRoutine implements Routine { private static final int MAX_CONTENTS = 300; private final List feeds; private final Map> targetChannelPatterns = new HashMap<>(); + private final int interval; public JavaMailRSSRoutine(Config config) { this.feeds = config.getRssFeeds(); + this.interval = config.getRssPollInterval(); this.feeds.forEach(feed -> { var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); @@ -47,8 +49,7 @@ public JavaMailRSSRoutine(Config config) { @Override public Schedule createSchedule() { - // TODO: Make this adjustable in the future - return new Schedule(ScheduleMode.FIXED_DELAY, 0, 10, TimeUnit.MINUTES); + return new Schedule(ScheduleMode.FIXED_DELAY, 0, interval, TimeUnit.MINUTES); } @Override @@ -108,14 +109,14 @@ private static EmbedBuilder constructEmbedMessage(Item item) { private static List getPostsAfterDate(String rssUrl, DateTime date) { List rssList = fetchRss(rssUrl); final Predicate rssListPredicate = item -> { - var pubDateTime = item.getPubDate(); - if (pubDateTime.isEmpty()) { + var pubDateString = item.getPubDate(); + if (pubDateString.isEmpty()) { return false; } + DateTime pubDate = parseDate(pubDateString.get()); - long pubDate = parseDate(pubDateTime.get()); - - return pubDate > date; + // If pubDate is after `date`, return true + return pubDate > date. }; if (rssList == null) { @@ -141,13 +142,13 @@ private static List fetchRss(String rssUrl) { } } - private static long parseDate(String date) { + private static long parseDateT(String date) { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); try { return dateFormat.parse(date).getTime(); } catch (Exception e) { logger.error("Could not parse date, {}", e.getMessage()); } - return 0; + return; } } From cae7e38ec66d483878881b611823a671b448236e Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 22:13:10 +0200 Subject: [PATCH 13/54] feat: improve date parsing Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> --- .../features/javamail/JavaMailRSSRoutine.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index f11d37a1d5..3773e344ec 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -1,6 +1,5 @@ package org.togetherjava.tjbot.features.javamail; -import com.apptasticsoftware.rssreader.DateTime; import com.apptasticsoftware.rssreader.Item; import com.apptasticsoftware.rssreader.RssReader; import net.dv8tion.jda.api.EmbedBuilder; @@ -21,7 +20,8 @@ import javax.annotation.Nonnull; import java.io.IOException; -import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -33,6 +33,8 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); private static final int MAX_CONTENTS = 300; + private static final DateTimeFormatter RSS_DATE_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); private final List feeds; private final Map> targetChannelPatterns = new HashMap<>(); private final int interval; @@ -105,18 +107,17 @@ private static EmbedBuilder constructEmbedMessage(Item item) { return embedBuilder; } - // TODO: make this use DateTime - private static List getPostsAfterDate(String rssUrl, DateTime date) { + private static List getPostsAfterDate(String rssUrl, Date afterDate) { List rssList = fetchRss(rssUrl); final Predicate rssListPredicate = item -> { var pubDateString = item.getPubDate(); if (pubDateString.isEmpty()) { return false; } - DateTime pubDate = parseDate(pubDateString.get()); + var pubDate = parseDateTime(pubDateString.get()); - // If pubDate is after `date`, return true - return pubDate > date. + // If pubDate is after the afterDate, it passes + return pubDate.compareTo(afterDate) > 0; }; if (rssList == null) { @@ -142,13 +143,7 @@ private static List fetchRss(String rssUrl) { } } - private static long parseDateT(String date) { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - try { - return dateFormat.parse(date).getTime(); - } catch (Exception e) { - logger.error("Could not parse date, {}", e.getMessage()); - } - return; + private static Date parseDateTime(String date) { + return Date.from(RSS_DATE_FORMAT.parse(date, Instant::from)); } } From 9eb905ba323c27744d0d5c69df466edc85f78b86 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 22:23:35 +0200 Subject: [PATCH 14/54] feat: update config.json.template --- application/config.json.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/config.json.template b/application/config.json.template index a60bbe6d5b..11cba887e1 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -111,5 +111,6 @@ "url": "", "targetChannelPattern", "" } - ] + ], + "rssPollInterval": 10 } From a75be809247ff3bb892f3b3eac92c165c6a04609 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 23:06:53 +0200 Subject: [PATCH 15/54] feat: improve date handling and add embed timestamp --- .../org/togetherjava/tjbot/config/Config.java | 8 +-- .../features/javamail/JavaMailRSSRoutine.java | 71 +++++++++++++------ .../resources/db/V14__Add_Rss_Feed_Cache.sql | 5 ++ 3 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index ee9833e817..2f27e10674 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -417,10 +417,6 @@ public String getMemberCountCategoryPattern() { return memberCountCategoryPattern; } - public String getJavaNewsChannelPattern() { - return javaNewsChannelPattern; - } - public List getRssFeeds() { return rssFeeds; } @@ -429,4 +425,8 @@ public List getRssFeeds() { public int getRssPollInterval() { return rssPollInterval; } + + public String getJavaNewsChannelPattern() { + return javaNewsChannelPattern; + } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 3773e344ec..e9907a11da 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -15,18 +15,20 @@ import org.togetherjava.tjbot.config.Config; import org.togetherjava.tjbot.config.RSSFeed; +import org.togetherjava.tjbot.db.Database; +import org.togetherjava.tjbot.db.generated.tables.records.RssFeedRecord; import org.togetherjava.tjbot.features.Routine; import javax.annotation.Nonnull; -import java.io.IOException; -import java.time.Instant; +import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.regex.Pattern; -import java.util.stream.Collectors; + +import static org.togetherjava.tjbot.db.generated.tables.RssFeed.RSS_FEED; public final class JavaMailRSSRoutine implements Routine { @@ -38,11 +40,12 @@ public final class JavaMailRSSRoutine implements Routine { private final List feeds; private final Map> targetChannelPatterns = new HashMap<>(); private final int interval; + private final Database database; - public JavaMailRSSRoutine(Config config) { + public JavaMailRSSRoutine(Config config, Database database) { this.feeds = config.getRssFeeds(); this.interval = config.getRssPollInterval(); - + this.database = database; this.feeds.forEach(feed -> { var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); targetChannelPatterns.put(feed, predicate); @@ -61,7 +64,18 @@ public void runRoutine(@Nonnull JDA jda) { private void sendRSS(JDA jda, RSSFeed feed) { var textChannel = getTextChannelFromFeed(jda, feed); - var items = fetchRss(feed.url()); + + RssFeedRecord entry = database.read(context -> context.selectFrom(RSS_FEED) + .where(RSS_FEED.URL.eq(feed.url())) + .limit(1) + .fetchOne()); + + List items; + if (entry == null) { + items = fetchRss(feed.url()); + } else { + items = fetchRssAfterDate(feed.url(), entry.getLastDate()); + } if (textChannel.isEmpty()) { logger.warn("Tried to sendRss, got empty response (channel {} not found)", @@ -73,6 +87,21 @@ private void sendRSS(JDA jda, RSSFeed feed) { MessageEmbed embed = constructEmbedMessage(item).build(); textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); }); + + String lastDate = items.getFirst().getPubDate().orElseThrow(); + if (entry == null) { + // Insert + database.write(context -> context.newRecord(RSS_FEED) + .setUrl(feed.url()) + .setLastDate(lastDate) + .insert()); + return; + } + + database.write(context -> context.update(RSS_FEED) + .set(RSS_FEED.LAST_DATE, lastDate) + .where(RSS_FEED.URL.eq(feed.url())) + .executeAsync()); } private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { @@ -88,15 +117,17 @@ private static EmbedBuilder constructEmbedMessage(Item item) { String titleLink = item.getLink().orElse(""); Optional rawDescription = item.getDescription(); + item.getPubDate().ifPresent(date -> embedBuilder.setTimestamp(getLocalDateTime(date))); + embedBuilder.setTitle(title, titleLink); if (rawDescription.isPresent()) { Document fullDescription = Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.get())); StringBuilder finalDescription = new StringBuilder(); - fullDescription.body().select("p").forEach(p -> { - finalDescription.append(p.text()).append(". "); - }); + fullDescription.body() + .select("p") + .forEach(p -> finalDescription.append(p.text()).append(". ")); embedBuilder .setDescription(StringUtils.abbreviate(finalDescription.toString(), MAX_CONTENTS)); @@ -107,28 +138,24 @@ private static EmbedBuilder constructEmbedMessage(Item item) { return embedBuilder; } - private static List getPostsAfterDate(String rssUrl, Date afterDate) { + private static List fetchRssAfterDate(String rssUrl, String afterDate) { + final LocalDateTime afterDateTime = getLocalDateTime(afterDate); List rssList = fetchRss(rssUrl); - final Predicate rssListPredicate = item -> { + final Predicate itemAfterDate = item -> { var pubDateString = item.getPubDate(); if (pubDateString.isEmpty()) { return false; } - var pubDate = parseDateTime(pubDateString.get()); + var pubDate = getLocalDateTime(pubDateString.get()); - // If pubDate is after the afterDate, it passes - return pubDate.compareTo(afterDate) > 0; + return pubDate.isAfter(afterDateTime); }; if (rssList == null) { return List.of(); } - return rssList.stream().filter(rssListPredicate).collect(Collectors.toList()); - } - - private static Optional getLatest(String rssUrl) throws IOException { - return fetchRss(rssUrl).stream().findFirst(); + return rssList.stream().filter(itemAfterDate).toList(); } /** @@ -136,14 +163,14 @@ private static Optional getLatest(String rssUrl) throws IOException { */ private static List fetchRss(String rssUrl) { try { - return RSS_READER.read(rssUrl).collect(Collectors.toList()); + return RSS_READER.read(rssUrl).toList(); } catch (Exception e) { logger.warn("Could not fetch RSS from URL ({})", rssUrl); return List.of(); } } - private static Date parseDateTime(String date) { - return Date.from(RSS_DATE_FORMAT.parse(date, Instant::from)); + private static LocalDateTime getLocalDateTime(String date) { + return LocalDateTime.parse(date, RSS_DATE_FORMAT); } } diff --git a/application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql b/application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql new file mode 100644 index 0000000000..ed65d30fb0 --- /dev/null +++ b/application/src/main/resources/db/V14__Add_Rss_Feed_Cache.sql @@ -0,0 +1,5 @@ +CREATE TABLE rss_feed +( + url TEXT NOT NULL PRIMARY KEY, + last_date TEXT NOT NULL +) \ No newline at end of file From faf8e85d5557982abf7f7717444dba36b4a2371a Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:18:19 -0800 Subject: [PATCH 16/54] fix: malformed `config.json.template` --- application/config.json.template | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 11cba887e1..5940a5e913 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -104,12 +104,11 @@ ] }, "selectRolesChannelPattern": "select-your-roles", - "rssPollInterval": 5, "memberCountCategoryPattern": "Info", "rssFeeds": [ { "url": "", - "targetChannelPattern", "" + "targetChannelPattern": "" } ], "rssPollInterval": 10 From b77955f79128427867a7cf623de9cfeab4c51867 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:02:27 -0800 Subject: [PATCH 17/54] fix: now correctly finds the latest date --- .../features/javamail/JavaMailRSSRoutine.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index e9907a11da..07cc471102 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -7,6 +7,7 @@ import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.Nullable; import org.jooq.tools.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -88,7 +89,12 @@ private void sendRSS(JDA jda, RSSFeed feed) { textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); }); - String lastDate = items.getFirst().getPubDate().orElseThrow(); + String lastDate = getLatestDate(items); + + if (lastDate == null) { + return; + } + if (entry == null) { // Insert database.write(context -> context.newRecord(RSS_FEED) @@ -104,6 +110,26 @@ private void sendRSS(JDA jda, RSSFeed feed) { .executeAsync()); } + @Nullable + private static String getLatestDate(List items) { + String lastDate = null; + + for (Item item : items) { + if (lastDate == null) { + lastDate = item.getPubDate().orElseThrow(); + continue; + } + + LocalDateTime formattedLastDate = getLocalDateTime(lastDate); + LocalDateTime itemDate = getLocalDateTime(item.getPubDate().orElseThrow()); + + if (itemDate.isAfter(formattedLastDate)) { + lastDate = item.getPubDate().orElseThrow(); + } + } + return lastDate; + } + private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { return jda.getTextChannelCache() .stream() From d068973636c50cf5d0260034ab36cae09fa1c1c2 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 22 Feb 2024 23:06:53 +0200 Subject: [PATCH 18/54] feat: improve date handling and add embed timestamp --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 07cc471102..1896ecb1d8 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -89,12 +89,7 @@ private void sendRSS(JDA jda, RSSFeed feed) { textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); }); - String lastDate = getLatestDate(items); - - if (lastDate == null) { - return; - } - + String lastDate = items.getFirst().getPubDate().orElseThrow(); if (entry == null) { // Insert database.write(context -> context.newRecord(RSS_FEED) From 964c0806a05e64ec474c3683957bcd5d16634b75 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:02:27 -0800 Subject: [PATCH 19/54] fix: now correctly finds the latest date --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 1896ecb1d8..07cc471102 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -89,7 +89,12 @@ private void sendRSS(JDA jda, RSSFeed feed) { textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); }); - String lastDate = items.getFirst().getPubDate().orElseThrow(); + String lastDate = getLatestDate(items); + + if (lastDate == null) { + return; + } + if (entry == null) { // Insert database.write(context -> context.newRecord(RSS_FEED) From 5a19fe98d10f2b81896d530b006f7b25496c86d7 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 21:39:40 -0800 Subject: [PATCH 20/54] feat: you can now optionally declare a specific channel for a feed to go to --- .../org/togetherjava/tjbot/config/Config.java | 2 +- .../org/togetherjava/tjbot/config/RSSFeed.java | 5 +++-- .../features/javamail/JavaMailRSSRoutine.java | 18 +++++++++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 2f27e10674..9dbb42b971 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -132,7 +132,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); this.rssFeeds = Objects.requireNonNull(rssFeeds); this.javaNewsChannelPattern = Objects.requireNonNull(javaNewsChannelPattern); - this.rssPollInterval = Objects.requireNonNull(rssPollInterval); + this.rssPollInterval = rssPollInterval; this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index 9fb0ea2876..7ed3746ce0 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -2,11 +2,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.util.Objects; public record RSSFeed(@JsonProperty(value = "url", required = true) String url, - @JsonProperty(value = "targetChannelPattern", - required = true) String targetChannelPattern) { + @JsonProperty(value = "targetChannelPattern") @Nullable String targetChannelPattern) { public RSSFeed { Objects.requireNonNull(url); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 07cc471102..1bf297c4c1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -39,6 +39,7 @@ public final class JavaMailRSSRoutine implements Routine { private static final DateTimeFormatter RSS_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); private final List feeds; + private final Predicate defaultChannelPattern; private final Map> targetChannelPatterns = new HashMap<>(); private final int interval; private final Database database; @@ -47,9 +48,13 @@ public JavaMailRSSRoutine(Config config, Database database) { this.feeds = config.getRssFeeds(); this.interval = config.getRssPollInterval(); this.database = database; + this.defaultChannelPattern = + Pattern.compile(config.getJavaNewsChannelPattern()).asMatchPredicate(); this.feeds.forEach(feed -> { - var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); - targetChannelPatterns.put(feed, predicate); + if (feed.targetChannelPattern() != null) { + var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); + targetChannelPatterns.put(feed, predicate); + } }); } @@ -131,9 +136,16 @@ private static String getLatestDate(List items) { } private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { + if (feed.targetChannelPattern() != null) { + return jda.getTextChannelCache() + .stream() + .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) + .findFirst(); + } + return jda.getTextChannelCache() .stream() - .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) + .filter(channel -> defaultChannelPattern.test(channel.getName())) .findFirst(); } From 0a3c2933d5d6808cbd807a56b5095f00e51e7f1d Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 21:51:35 -0800 Subject: [PATCH 21/54] finished javadoc todos --- .../java/org/togetherjava/tjbot/config/Config.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 9dbb42b971..399d17efa1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -417,15 +417,27 @@ public String getMemberCountCategoryPattern() { return memberCountCategoryPattern; } + /** + * @return A list of all currently tracked RSS feeds + */ public List getRssFeeds() { return rssFeeds; } - // TODO: Make JavaDocs on this + /** + * Gets the interval in which RSS feeds will be polled from the source. + * + * @return The interval in seconds + */ public int getRssPollInterval() { return rssPollInterval; } + /** + * The channel name that rss feeds will be sent to if they are not specified. + * + * @return The channel name pattern + */ public String getJavaNewsChannelPattern() { return javaNewsChannelPattern; } From b52e95e9827bda150b2492d76433ce43d11772ef Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Wed, 28 Feb 2024 22:05:05 -0800 Subject: [PATCH 22/54] improved embed --- .../togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 1bf297c4c1..05b1258c53 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -158,6 +158,7 @@ private static EmbedBuilder constructEmbedMessage(Item item) { item.getPubDate().ifPresent(date -> embedBuilder.setTimestamp(getLocalDateTime(date))); embedBuilder.setTitle(title, titleLink); + embedBuilder.setAuthor(item.getChannel().getLink()); if (rawDescription.isPresent()) { Document fullDescription = From 66a43502a1db93d906a0405bdbd0b93f5eb07086 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 18:32:06 +0200 Subject: [PATCH 23/54] feat(rss-routine): working version - Added JavaDocs on most parts - Simplified the code and cleaned up unused stuff --- application/config.json.template | 3 +- .../togetherjava/tjbot/config/RSSFeed.java | 5 +- .../features/javamail/JavaMailRSSRoutine.java | 203 ++++++++++++------ 3 files changed, 139 insertions(+), 72 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 5940a5e913..c4208a31fb 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -108,7 +108,8 @@ "rssFeeds": [ { "url": "", - "targetChannelPattern": "" + "targetChannelPattern": "", + "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss zzz" } ], "rssPollInterval": 10 diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index 7ed3746ce0..1ebbd80e6f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -7,10 +7,13 @@ import java.util.Objects; public record RSSFeed(@JsonProperty(value = "url", required = true) String url, - @JsonProperty(value = "targetChannelPattern") @Nullable String targetChannelPattern) { + @JsonProperty(value = "targetChannelPattern") @Nullable String targetChannelPattern, + @JsonProperty(value = "dateFormatterPattern", + required = true) String dateFormatterPattern) { public RSSFeed { Objects.requireNonNull(url); Objects.requireNonNull(targetChannelPattern); + Objects.requireNonNull(dateFormatterPattern); } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 05b1258c53..3c343b52f2 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -7,7 +7,9 @@ import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jooq.Result; import org.jooq.tools.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -23,9 +25,13 @@ import javax.annotation.Nonnull; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -36,8 +42,8 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); private static final int MAX_CONTENTS = 300; - private static final DateTimeFormatter RSS_DATE_FORMAT = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + private static final ZonedDateTime ZONED_TIME_MIN = + ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); private final List feeds; private final Predicate defaultChannelPattern; private final Map> targetChannelPatterns = new HashMap<>(); @@ -68,74 +74,135 @@ public void runRoutine(@Nonnull JDA jda) { feeds.forEach(feed -> sendRSS(jda, feed)); } - private void sendRSS(JDA jda, RSSFeed feed) { - var textChannel = getTextChannelFromFeed(jda, feed); - - RssFeedRecord entry = database.read(context -> context.selectFrom(RSS_FEED) - .where(RSS_FEED.URL.eq(feed.url())) - .limit(1) - .fetchOne()); + /** + * Sends all the necessary posts from a given RSS feed. + *

+ * This handles fetching the latest posts from the given URL, checking which ones have already + * been posted by reading information from the database and updating the last posted date. + */ + private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException { + // Don't proceed if the text channel was not found + var textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); + if (textChannel == null) { + logger.warn("Tried to sendRss, got empty response (channel {} not found)", + feedConfig.targetChannelPattern()); + return; + } - List items; - if (entry == null) { - items = fetchRss(feed.url()); - } else { - items = fetchRssAfterDate(feed.url(), entry.getLastDate()); + // Acquire the list of RSS items from the configured URL + List rssItems = fetchRss(feedConfig.url()); + if (rssItems.isEmpty()) { + return; } - if (textChannel.isEmpty()) { - logger.warn("Tried to sendRss, got empty response (channel {} not found)", - feed.targetChannelPattern()); + // Do not proceed if the configured date format does not match + // the actual date format provided by the feed's contents + if (!isValidDateFormat(rssItems, feedConfig)) { + logger.warn("Could not find valid date format for RSS feed {}", feedConfig.url()); return; } - items.forEach(item -> { - MessageEmbed embed = constructEmbedMessage(item).build(); - textChannel.get().sendMessageEmbeds(List.of(embed)).queue(); - }); + // Attempt to find any stored information regarding the provided RSS URL + Result dateResult = database.read(context -> context.selectFrom(RSS_FEED) + .where(RSS_FEED.URL.eq(feedConfig.url())) + .limit(1) + .fetch()); + + String dateStr = dateResult.isEmpty() ? null : dateResult.getFirst().getLastDate(); + ZonedDateTime lastSavedDate = getLocalDateTime(dateStr, feedConfig.dateFormatterPattern()); - String lastDate = getLatestDate(items); + final Predicate shouldItemBePosted = item -> { + ZonedDateTime itemPubDate = + getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); + return itemPubDate.isAfter(lastSavedDate); + }; + + // Date that will be stored in the database at the end + AtomicReference lastPostedDate = new AtomicReference<>(lastSavedDate); + + // Send each item that should be posted and concurrently + // find the post with the latest date + rssItems.stream().filter(shouldItemBePosted).forEach(item -> { + MessageEmbed embed = constructEmbedMessage(item, feedConfig).build(); + textChannel.sendMessageEmbeds(List.of(embed)).queue(); + + // Get the last posted date so that we update the database + ZonedDateTime pubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); + if (pubDate.isAfter(lastPostedDate.get())) { + lastPostedDate.set(pubDate); + } + }); - if (lastDate == null) { + // Don't write anything to the database if there were no RSS items + if (rssItems.isEmpty()) { return; } - if (entry == null) { - // Insert + // Finally, save the last posted date to the database. + var dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); + String lastDateStr = lastPostedDate.get().format(dateTimeFormatter); + if (dateResult.isEmpty()) { database.write(context -> context.newRecord(RSS_FEED) - .setUrl(feed.url()) - .setLastDate(lastDate) + .setUrl(feedConfig.url()) + .setLastDate(lastDateStr) .insert()); return; } + // If we already have an existing record with the given URL, + // now is the time to update it database.write(context -> context.update(RSS_FEED) - .set(RSS_FEED.LAST_DATE, lastDate) - .where(RSS_FEED.URL.eq(feed.url())) + .set(RSS_FEED.LAST_DATE, lastDateStr) + .where(RSS_FEED.URL.eq(feedConfig.url())) .executeAsync()); } - @Nullable - private static String getLatestDate(List items) { - String lastDate = null; + /** + * Attempts to get a {@link ZonedDateTime} from an {@link Item} with a provided string date time + * format. + *

+ * If either of the function inputs are null, the oldest-possible {@link ZonedDateTime} will get + * returned instead. + * + * @return The computed {@link ZonedDateTime} + */ + private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) { + var pubDate = item.getPubDate().orElse(null); - for (Item item : items) { - if (lastDate == null) { - lastDate = item.getPubDate().orElseThrow(); - continue; - } + if (pubDate == null || dateTimeFormat == null) { + return ZONED_TIME_MIN; + } + + return getLocalDateTime(pubDate, dateTimeFormat); + } - LocalDateTime formattedLastDate = getLocalDateTime(lastDate); - LocalDateTime itemDate = getLocalDateTime(item.getPubDate().orElseThrow()); + /** + * Given a list of RSS feed items, this function checks if the dates are valid + *

+ * This assumes that all items share the same date format (as is usual in an RSS feed response), + * therefore it only checks the first item's date for the final result. + */ + private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig) { + try { + final var firstRssFeed = rssFeeds.getFirst(); + String firstRssFeedPubDate = firstRssFeed.getPubDate().orElse(null); - if (itemDate.isAfter(formattedLastDate)) { - lastDate = item.getPubDate().orElseThrow(); + if (firstRssFeedPubDate == null) { + return false; } + + getLocalDateTime(firstRssFeedPubDate, feedConfig.dateFormatterPattern()); + } catch (Exception e) { + return false; } - return lastDate; + return true; } + /** + * Attempts to find a text channel from a given RSS feed configuration. + */ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { + // Attempt to find the target channel if (feed.targetChannelPattern() != null) { return jda.getTextChannelCache() .stream() @@ -143,29 +210,36 @@ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { .findFirst(); } + // If the target channel was not found, use the fallback return jda.getTextChannelCache() .stream() .filter(channel -> defaultChannelPattern.test(channel.getName())) .findFirst(); } - private static EmbedBuilder constructEmbedMessage(Item item) { + /** + * Provides the {@link EmbedBuilder} from an RSS item used for sending RSS posts. + */ + private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) { final EmbedBuilder embedBuilder = new EmbedBuilder(); String title = item.getTitle().orElse("No title"); String titleLink = item.getLink().orElse(""); Optional rawDescription = item.getDescription(); - item.getPubDate().ifPresent(date -> embedBuilder.setTimestamp(getLocalDateTime(date))); + // Set the item's timestamp to the embed if found + item.getPubDate() + .ifPresent(date -> embedBuilder + .setTimestamp(getLocalDateTime(date, feedConfig.dateFormatterPattern()))); embedBuilder.setTitle(title, titleLink); - embedBuilder.setAuthor(item.getChannel().getLink()); - if (rawDescription.isPresent()) { + // Process embed's description if a raw description was provided + if (rawDescription.isPresent() && !rawDescription.get().isEmpty()) { Document fullDescription = Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.get())); StringBuilder finalDescription = new StringBuilder(); fullDescription.body() - .select("p") + .select("*") .forEach(p -> finalDescription.append(p.text()).append(". ")); embedBuilder @@ -173,32 +247,13 @@ private static EmbedBuilder constructEmbedMessage(Item item) { return embedBuilder; } + // Fill the description with a placeholder if the description was empty embedBuilder.setDescription("No description"); return embedBuilder; } - private static List fetchRssAfterDate(String rssUrl, String afterDate) { - final LocalDateTime afterDateTime = getLocalDateTime(afterDate); - List rssList = fetchRss(rssUrl); - final Predicate itemAfterDate = item -> { - var pubDateString = item.getPubDate(); - if (pubDateString.isEmpty()) { - return false; - } - var pubDate = getLocalDateTime(pubDateString.get()); - - return pubDate.isAfter(afterDateTime); - }; - - if (rssList == null) { - return List.of(); - } - - return rssList.stream().filter(itemAfterDate).toList(); - } - /** - * Fetches new items from a given RSS url. + * Fetches a list of {@link Item} from a given RSS url. */ private static List fetchRss(String rssUrl) { try { @@ -209,7 +264,15 @@ private static List fetchRss(String rssUrl) { } } - private static LocalDateTime getLocalDateTime(String date) { - return LocalDateTime.parse(date, RSS_DATE_FORMAT); + /** + * Helper function for parsing a given date value to a {@link ZonedDateTime} with a given + * format. + */ + private static ZonedDateTime getLocalDateTime(@Nullable String date, @NotNull String format) { + if (date == null) { + return ZONED_TIME_MIN; + } + + return ZonedDateTime.parse(date, DateTimeFormatter.ofPattern(format)); } } From 56eb1b8858d5ddba194727eebcfb90b02a8ed924 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 18:33:40 +0200 Subject: [PATCH 24/54] feat(rss-routine): increase MAX_CONTENTS to 1000 --- .../tjbot/features/javamail/JavaMailRSSRoutine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java index 3c343b52f2..845e3b9bd1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java @@ -41,7 +41,7 @@ public final class JavaMailRSSRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); private static final RssReader RSS_READER = new RssReader(); - private static final int MAX_CONTENTS = 300; + private static final int MAX_CONTENTS = 1000; private static final ZonedDateTime ZONED_TIME_MIN = ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); private final List feeds; From 0c0770a304a5b735ef823159c17de2673c16591a Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 18:41:45 +0200 Subject: [PATCH 25/54] refactor(rss-routine): rename to RSSHandlerRoutine While the original idea was to add a Java news and changes RSS feed, this was expanded to all types of RSS feeds, so a more appropriate name makes things more clear. --- .../main/java/org/togetherjava/tjbot/features/Features.java | 4 ++-- .../{JavaMailRSSRoutine.java => RSSHandlerRoutine.java} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename application/src/main/java/org/togetherjava/tjbot/features/javamail/{JavaMailRSSRoutine.java => RSSHandlerRoutine.java} (98%) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/Features.java b/application/src/main/java/org/togetherjava/tjbot/features/Features.java index f47a9c3040..9b837799ef 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/Features.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/Features.java @@ -24,7 +24,7 @@ import org.togetherjava.tjbot.features.github.GitHubCommand; import org.togetherjava.tjbot.features.github.GitHubReference; import org.togetherjava.tjbot.features.help.*; -import org.togetherjava.tjbot.features.javamail.JavaMailRSSRoutine; +import org.togetherjava.tjbot.features.javamail.RSSHandlerRoutine; import org.togetherjava.tjbot.features.jshell.JShellCommand; import org.togetherjava.tjbot.features.jshell.JShellEval; import org.togetherjava.tjbot.features.mathcommands.TeXCommand; @@ -109,8 +109,8 @@ public static Collection createFeatures(JDA jda, Database database, Con features.add(new HelpThreadActivityUpdater(helpSystemHelper)); features.add(new HelpThreadAutoArchiver(helpSystemHelper)); features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem)); - features.add(new JavaMailRSSRoutine(config)); features.add(new MemberCountDisplayRoutine(config)); + features.add(new RSSHandlerRoutine(config, database)); // Message receivers features.add(new TopHelpersMessageListener(database, config)); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java similarity index 98% rename from application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java rename to application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 845e3b9bd1..e6ae17903f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/JavaMailRSSRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -37,9 +37,9 @@ import static org.togetherjava.tjbot.db.generated.tables.RssFeed.RSS_FEED; -public final class JavaMailRSSRoutine implements Routine { +public final class RSSHandlerRoutine implements Routine { - private static final Logger logger = LoggerFactory.getLogger(JavaMailRSSRoutine.class); + private static final Logger logger = LoggerFactory.getLogger(RSSHandlerRoutine.class); private static final RssReader RSS_READER = new RssReader(); private static final int MAX_CONTENTS = 1000; private static final ZonedDateTime ZONED_TIME_MIN = @@ -50,7 +50,7 @@ public final class JavaMailRSSRoutine implements Routine { private final int interval; private final Database database; - public JavaMailRSSRoutine(Config config, Database database) { + public RSSHandlerRoutine(Config config, Database database) { this.feeds = config.getRssFeeds(); this.interval = config.getRssPollInterval(); this.database = database; From 9629b357ef0b89a1dfe7d8845535b2ceece1bb67 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 18:58:16 +0200 Subject: [PATCH 26/54] docs(rss-routine): add JavaDocs for constructor and class --- .../features/javamail/RSSHandlerRoutine.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index e6ae17903f..dae5f6732c 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -37,6 +37,32 @@ import static org.togetherjava.tjbot.db.generated.tables.RssFeed.RSS_FEED; +/** + * This class orchestrates the retrieval, organization, and distribution of RSS feed posts sourced + * from various channels, all of which can be easily configured via the {@code config.json} + *

+ * To include a new RSS feed, simply define an {@link RSSFeed} entry in the {@code "rssFeeds"} array + * within the configuration file, adhering to the format shown below: + * + * 

+ * {@code
+ * {
+ *     "url": "https://example.com/feed",
+ *     "targetChannelPattern": "example",
+ *     "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss Z"
+ * }
+ * }
+ *
+ * + * Where: + *
    + *
  • {@code url} represents the URL of the RSS feed.
    • + *
  • {@code targetChannelPattern} specifies the pattern to identify the target channel for the + * feed posts.
    • + *
  • {@code dateFormatterPattern} denotes the pattern for parsing the date and time information in + * the feed.
    • + *
+ */ public final class RSSHandlerRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(RSSHandlerRoutine.class); @@ -50,6 +76,12 @@ public final class RSSHandlerRoutine implements Routine { private final int interval; private final Database database; + /** + * Constructs an RSSHandlerRoutine with the provided configuration and database. + * + * @param config The configuration containing RSS feed details. + * @param database The database for storing RSS feed data. + */ public RSSHandlerRoutine(Config config, Database database) { this.feeds = config.getRssFeeds(); this.interval = config.getRssPollInterval(); From 19befa6cea35a7844876e91a49a0f5a7070ed2c6 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 18:59:30 +0200 Subject: [PATCH 27/54] feat: add @NotNull annotation --- .../togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index dae5f6732c..aef71a1366 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -97,6 +97,7 @@ public RSSHandlerRoutine(Config config, Database database) { } @Override + @NotNull public Schedule createSchedule() { return new Schedule(ScheduleMode.FIXED_DELAY, 0, interval, TimeUnit.MINUTES); } From fc682ca726abc081177e4e47337668ed86542649 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:05:09 -0800 Subject: [PATCH 28/54] Update RSSHandlerRoutine.java --- .../togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index aef71a1366..5ac93a11ce 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -265,6 +265,7 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) .setTimestamp(getLocalDateTime(date, feedConfig.dateFormatterPattern()))); embedBuilder.setTitle(title, titleLink); + embedBuilder.setAuthor(item.getChannel().getLink()); // Process embed's description if a raw description was provided if (rawDescription.isPresent() && !rawDescription.get().isEmpty()) { From 184aee710a07c6c83486494d5c36dd344126cda2 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:17:59 -0800 Subject: [PATCH 29/54] fix: reverse feed so it posts in correct order --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 5ac93a11ce..12aaed9403 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -153,9 +153,10 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException // Date that will be stored in the database at the end AtomicReference lastPostedDate = new AtomicReference<>(lastSavedDate); + // Send each item that should be posted and concurrently // find the post with the latest date - rssItems.stream().filter(shouldItemBePosted).forEach(item -> { + rssItems.reversed().stream().filter(shouldItemBePosted).forEach(item -> { MessageEmbed embed = constructEmbedMessage(item, feedConfig).build(); textChannel.sendMessageEmbeds(List.of(embed)).queue(); From 44df6b426765da46e1a7f69d15bb1c1e92b524a2 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 19:47:12 +0200 Subject: [PATCH 30/54] refactor: use variable types instead of var --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 12aaed9403..58e67c27e1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -90,7 +90,8 @@ public RSSHandlerRoutine(Config config, Database database) { Pattern.compile(config.getJavaNewsChannelPattern()).asMatchPredicate(); this.feeds.forEach(feed -> { if (feed.targetChannelPattern() != null) { - var predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); + Predicate predicate = + Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); targetChannelPatterns.put(feed, predicate); } }); @@ -115,7 +116,7 @@ public void runRoutine(@Nonnull JDA jda) { */ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException { // Don't proceed if the text channel was not found - var textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); + TextChannel textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); if (textChannel == null) { logger.warn("Tried to sendRss, got empty response (channel {} not found)", feedConfig.targetChannelPattern()); @@ -173,7 +174,8 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException } // Finally, save the last posted date to the database. - var dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); + DateTimeFormatter dateTimeFormatter = + DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); String lastDateStr = lastPostedDate.get().format(dateTimeFormatter); if (dateResult.isEmpty()) { database.write(context -> context.newRecord(RSS_FEED) @@ -201,7 +203,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException * @return The computed {@link ZonedDateTime} */ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) { - var pubDate = item.getPubDate().orElse(null); + String pubDate = item.getPubDate().orElse(null); if (pubDate == null || dateTimeFormat == null) { return ZONED_TIME_MIN; @@ -218,7 +220,7 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma */ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig) { try { - final var firstRssFeed = rssFeeds.getFirst(); + final Item firstRssFeed = rssFeeds.getFirst(); String firstRssFeedPubDate = firstRssFeed.getPubDate().orElse(null); if (firstRssFeedPubDate == null) { From 39a6986f2ef2b337055a3f3c779bef13fe95fe88 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 19:51:57 +0200 Subject: [PATCH 31/54] feat: use fetchAny() instead of fetch() --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 58e67c27e1..63ff43b536 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -9,7 +9,6 @@ import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jooq.Result; import org.jooq.tools.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -137,12 +136,12 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException } // Attempt to find any stored information regarding the provided RSS URL - Result dateResult = database.read(context -> context.selectFrom(RSS_FEED) + RssFeedRecord dateResult = database.read(context -> context.selectFrom(RSS_FEED) .where(RSS_FEED.URL.eq(feedConfig.url())) .limit(1) - .fetch()); + .fetchAny()); - String dateStr = dateResult.isEmpty() ? null : dateResult.getFirst().getLastDate(); + String dateStr = dateResult == null ? null : dateResult.getLastDate(); ZonedDateTime lastSavedDate = getLocalDateTime(dateStr, feedConfig.dateFormatterPattern()); final Predicate shouldItemBePosted = item -> { @@ -177,7 +176,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); String lastDateStr = lastPostedDate.get().format(dateTimeFormatter); - if (dateResult.isEmpty()) { + if (dateResult == null) { database.write(context -> context.newRecord(RSS_FEED) .setUrl(feedConfig.url()) .setLastDate(lastDateStr) From 061ab509f9d5276836629ffdb227eddce6e34506 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 22:18:54 +0200 Subject: [PATCH 32/54] fix(rss-handler): remove redundant empty check Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 63ff43b536..83381b528b 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -167,9 +167,6 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException } }); - // Don't write anything to the database if there were no RSS items - if (rssItems.isEmpty()) { - return; } // Finally, save the last posted date to the database. From 15510bf8c856760e03fefcb256dcc3e5aa2b2873 Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 29 Feb 2024 22:23:42 +0200 Subject: [PATCH 33/54] refactor(rss-handler): remove AtomicReference usage Co-authored-by: Ethan McCue <5004262+bowbahdoe@users.noreply.github.com> --- .../features/javamail/RSSHandlerRoutine.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 83381b528b..63af0811a2 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -30,7 +30,6 @@ import java.time.format.DateTimeParseException; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -151,28 +150,29 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException }; // Date that will be stored in the database at the end - AtomicReference lastPostedDate = new AtomicReference<>(lastSavedDate); - + ZonedDateTime lastPostedDate = lastSavedDate; // Send each item that should be posted and concurrently // find the post with the latest date - rssItems.reversed().stream().filter(shouldItemBePosted).forEach(item -> { + for (Item item : rssItems.reversed()) { + if (!shouldItemBePosted.test(item)) { + continue; + } + MessageEmbed embed = constructEmbedMessage(item, feedConfig).build(); textChannel.sendMessageEmbeds(List.of(embed)).queue(); // Get the last posted date so that we update the database ZonedDateTime pubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); - if (pubDate.isAfter(lastPostedDate.get())) { - lastPostedDate.set(pubDate); + if (pubDate.isAfter(lastPostedDate)) { + lastPostedDate = pubDate; } - }); - } // Finally, save the last posted date to the database. DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); - String lastDateStr = lastPostedDate.get().format(dateTimeFormatter); + String lastDateStr = lastPostedDate.format(dateTimeFormatter); if (dateResult == null) { database.write(context -> context.newRecord(RSS_FEED) .setUrl(feedConfig.url()) From 31bf19747bc1d5ba3330fc5e4130f7142462a1c6 Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:40:43 -0800 Subject: [PATCH 34/54] resolve a couple issues --- .../features/javamail/RSSHandlerRoutine.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 63af0811a2..b07907ef24 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; +import java.io.IOException; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -64,13 +65,13 @@ public final class RSSHandlerRoutine implements Routine { private static final Logger logger = LoggerFactory.getLogger(RSSHandlerRoutine.class); - private static final RssReader RSS_READER = new RssReader(); private static final int MAX_CONTENTS = 1000; private static final ZonedDateTime ZONED_TIME_MIN = ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); + private final RssReader rssReader; private final List feeds; private final Predicate defaultChannelPattern; - private final Map> targetChannelPatterns = new HashMap<>(); + private final Map> targetChannelPatterns; private final int interval; private final Database database; @@ -86,6 +87,7 @@ public RSSHandlerRoutine(Config config, Database database) { this.database = database; this.defaultChannelPattern = Pattern.compile(config.getJavaNewsChannelPattern()).asMatchPredicate(); + this.targetChannelPatterns = new HashMap<>(); this.feeds.forEach(feed -> { if (feed.targetChannelPattern() != null) { Predicate predicate = @@ -93,6 +95,7 @@ public RSSHandlerRoutine(Config config, Database database) { targetChannelPatterns.put(feed, predicate); } }); + this.rssReader = new RssReader(); } @Override @@ -141,7 +144,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException .fetchAny()); String dateStr = dateResult == null ? null : dateResult.getLastDate(); - ZonedDateTime lastSavedDate = getLocalDateTime(dateStr, feedConfig.dateFormatterPattern()); + ZonedDateTime lastSavedDate = getZonedDateTime(dateStr, feedConfig.dateFormatterPattern()); final Predicate shouldItemBePosted = item -> { ZonedDateTime itemPubDate = @@ -205,7 +208,7 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma return ZONED_TIME_MIN; } - return getLocalDateTime(pubDate, dateTimeFormat); + return getZonedDateTime(pubDate, dateTimeFormat); } /** @@ -223,7 +226,7 @@ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig return false; } - getLocalDateTime(firstRssFeedPubDate, feedConfig.dateFormatterPattern()); + getZonedDateTime(firstRssFeedPubDate, feedConfig.dateFormatterPattern()); } catch (Exception e) { return false; } @@ -261,7 +264,7 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) // Set the item's timestamp to the embed if found item.getPubDate() .ifPresent(date -> embedBuilder - .setTimestamp(getLocalDateTime(date, feedConfig.dateFormatterPattern()))); + .setTimestamp(getZonedDateTime(date, feedConfig.dateFormatterPattern()))); embedBuilder.setTitle(title, titleLink); embedBuilder.setAuthor(item.getChannel().getLink()); @@ -288,10 +291,10 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) /** * Fetches a list of {@link Item} from a given RSS url. */ - private static List fetchRss(String rssUrl) { + private List fetchRss(String rssUrl) { try { - return RSS_READER.read(rssUrl).toList(); - } catch (Exception e) { + return rssReader.read(rssUrl).toList(); + } catch (IOException e) { logger.warn("Could not fetch RSS from URL ({})", rssUrl); return List.of(); } @@ -300,8 +303,9 @@ private static List fetchRss(String rssUrl) { /** * Helper function for parsing a given date value to a {@link ZonedDateTime} with a given * format. + * */ - private static ZonedDateTime getLocalDateTime(@Nullable String date, @NotNull String format) { + private static ZonedDateTime getZonedDateTime(@Nullable String date, @NotNull String format) throws DateTimeParseException { if (date == null) { return ZONED_TIME_MIN; } From 851fb06f810a9e52cc1906a5001791e70bfbba90 Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 19:47:41 +0200 Subject: [PATCH 35/54] refactor(rss): switch to using a record for the config --- application/config.json.template | 21 ++++++----- .../org/togetherjava/tjbot/config/Config.java | 37 +++---------------- .../tjbot/config/RSSFeedsConfig.java | 31 ++++++++++++++++ .../features/javamail/RSSHandlerRoutine.java | 16 ++++---- 4 files changed, 58 insertions(+), 47 deletions(-) create mode 100644 application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java diff --git a/application/config.json.template b/application/config.json.template index c4208a31fb..9fd79e14a6 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -104,13 +104,16 @@ ] }, "selectRolesChannelPattern": "select-your-roles", - "memberCountCategoryPattern": "Info", - "rssFeeds": [ - { - "url": "", - "targetChannelPattern": "", - "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss zzz" - } - ], - "rssPollInterval": 10 + "rssConfig": { + "feeds": [ + { + "url": "", + "targetChannelPattern": "", + "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss zzz" + } + ], + "javaNewsChannelPattern": "", + "rssPollInterval": 10 + } + "memberCountCategoryPattern": "Info" } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 399d17efa1..376a1d5149 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -43,9 +43,7 @@ public final class Config { private final String sourceCodeBaseUrl; private final JShellConfig jshell; private final FeatureBlacklistConfig featureBlacklistConfig; - private final List rssFeeds; - private final String javaNewsChannelPattern; - private final int rssPollInterval; + private final RSSFeedsConfig rssFeedsConfig; private final String selectRolesChannelPattern; private final String memberCountCategoryPattern; @@ -94,10 +92,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, required = true) String memberCountCategoryPattern, @JsonProperty(value = "featureBlacklist", required = true) FeatureBlacklistConfig featureBlacklistConfig, - @JsonProperty(value = "javaNewsChannelPattern", - required = true) String javaNewsChannelPattern, - @JsonProperty(value = "rssFeeds", required = true) List rssFeeds, - @JsonProperty(value = "rssPollInterval", required = true) int rssPollInterval, + @JsonProperty(value = "rssConfig", required = true) RSSFeedsConfig rssFeedsConfig, @JsonProperty(value = "selectRolesChannelPattern", required = true) String selectRolesChannelPattern) { this.token = Objects.requireNonNull(token); @@ -130,9 +125,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.sourceCodeBaseUrl = Objects.requireNonNull(sourceCodeBaseUrl); this.jshell = Objects.requireNonNull(jshell); this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); - this.rssFeeds = Objects.requireNonNull(rssFeeds); - this.javaNewsChannelPattern = Objects.requireNonNull(javaNewsChannelPattern); - this.rssPollInterval = rssPollInterval; + this.rssFeedsConfig = rssFeedsConfig; this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); } @@ -418,27 +411,9 @@ public String getMemberCountCategoryPattern() { } /** - * @return A list of all currently tracked RSS feeds + * @return the RSS feeds configuration */ - public List getRssFeeds() { - return rssFeeds; - } - - /** - * Gets the interval in which RSS feeds will be polled from the source. - * - * @return The interval in seconds - */ - public int getRssPollInterval() { - return rssPollInterval; - } - - /** - * The channel name that rss feeds will be sent to if they are not specified. - * - * @return The channel name pattern - */ - public String getJavaNewsChannelPattern() { - return javaNewsChannelPattern; + public RSSFeedsConfig getRSSFeedsConfig() { + return rssFeedsConfig; } } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java new file mode 100644 index 0000000000..c7988b8ba4 --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -0,0 +1,31 @@ +package org.togetherjava.tjbot.config; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Objects; + +/** + * Represents the configuration for an RSS feed, which includes the list of feeds to subscribe to, a + * pattern for identifying Java news channels, and the interval (in minutes) for polling the feeds. + */ +public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, + @JsonProperty(value = "javaNewsChannelPattern", + required = true) String javaNewsChannelPattern, + @JsonProperty(value = "rssPollInterval", required = true) int rssPollInterval) { + + /** + * Constructs a new {@link RSSFeedsConfig}. + * + * @param feeds The list of RSS feeds to subscribe to. + * @param javaNewsChannelPattern The pattern used to identify Java news channels within the RSS + * feeds. + * @param rssPollInterval The interval (in minutes) for polling the RSS feeds for updates. + * @throws NullPointerException if any of the parameters (feeds or javaNewsChannelPattern) are + * null + */ + public RSSFeedsConfig { + Objects.requireNonNull(feeds); + Objects.requireNonNull(javaNewsChannelPattern); + } +} diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index b07907ef24..7c7a5eb582 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -17,6 +17,7 @@ import org.togetherjava.tjbot.config.Config; import org.togetherjava.tjbot.config.RSSFeed; +import org.togetherjava.tjbot.config.RSSFeedsConfig; import org.togetherjava.tjbot.db.Database; import org.togetherjava.tjbot.db.generated.tables.records.RssFeedRecord; import org.togetherjava.tjbot.features.Routine; @@ -69,7 +70,7 @@ public final class RSSHandlerRoutine implements Routine { private static final ZonedDateTime ZONED_TIME_MIN = ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); private final RssReader rssReader; - private final List feeds; + private final RSSFeedsConfig config; private final Predicate defaultChannelPattern; private final Map> targetChannelPatterns; private final int interval; @@ -82,13 +83,13 @@ public final class RSSHandlerRoutine implements Routine { * @param database The database for storing RSS feed data. */ public RSSHandlerRoutine(Config config, Database database) { - this.feeds = config.getRssFeeds(); - this.interval = config.getRssPollInterval(); + this.config = config.getRSSFeedsConfig(); + this.interval = this.config.rssPollInterval(); this.database = database; this.defaultChannelPattern = - Pattern.compile(config.getJavaNewsChannelPattern()).asMatchPredicate(); + Pattern.compile(this.config.javaNewsChannelPattern()).asMatchPredicate(); this.targetChannelPatterns = new HashMap<>(); - this.feeds.forEach(feed -> { + this.config.feeds().forEach(feed -> { if (feed.targetChannelPattern() != null) { Predicate predicate = Pattern.compile(feed.targetChannelPattern()).asMatchPredicate(); @@ -106,7 +107,7 @@ public Schedule createSchedule() { @Override public void runRoutine(@Nonnull JDA jda) { - feeds.forEach(feed -> sendRSS(jda, feed)); + this.config.feeds().forEach(feed -> sendRSS(jda, feed)); } /** @@ -305,7 +306,8 @@ private List fetchRss(String rssUrl) { * format. * */ - private static ZonedDateTime getZonedDateTime(@Nullable String date, @NotNull String format) throws DateTimeParseException { + private static ZonedDateTime getZonedDateTime(@Nullable String date, @NotNull String format) + throws DateTimeParseException { if (date == null) { return ZONED_TIME_MIN; } From 2b6bc7ca9b464d7caa29085355634322d8b3aaae Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 20:00:11 +0200 Subject: [PATCH 36/54] feat: rename to fallbackChannelPattern --- application/config.json.template | 2 +- .../org/togetherjava/tjbot/config/RSSFeedsConfig.java | 11 +++++------ .../tjbot/features/javamail/RSSHandlerRoutine.java | 8 ++++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 9fd79e14a6..472150f481 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -112,7 +112,7 @@ "dateFormatterPattern": "EEE, dd MMM yyyy HH:mm:ss zzz" } ], - "javaNewsChannelPattern": "", + "fallbackChannelPattern": "", "rssPollInterval": 10 } "memberCountCategoryPattern": "Info" diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index c7988b8ba4..c19b036de6 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -10,22 +10,21 @@ * pattern for identifying Java news channels, and the interval (in minutes) for polling the feeds. */ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, - @JsonProperty(value = "javaNewsChannelPattern", - required = true) String javaNewsChannelPattern, + @JsonProperty(value = "fallbackChannelPattern", + required = true) String fallbackChannelPattern, @JsonProperty(value = "rssPollInterval", required = true) int rssPollInterval) { /** * Constructs a new {@link RSSFeedsConfig}. * * @param feeds The list of RSS feeds to subscribe to. - * @param javaNewsChannelPattern The pattern used to identify Java news channels within the RSS - * feeds. + * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. * @param rssPollInterval The interval (in minutes) for polling the RSS feeds for updates. - * @throws NullPointerException if any of the parameters (feeds or javaNewsChannelPattern) are + * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are * null */ public RSSFeedsConfig { Objects.requireNonNull(feeds); - Objects.requireNonNull(javaNewsChannelPattern); + Objects.requireNonNull(fallbackChannelPattern); } } diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 7c7a5eb582..aa94990912 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -71,7 +71,7 @@ public final class RSSHandlerRoutine implements Routine { ZonedDateTime.of(LocalDateTime.MIN, ZoneId.systemDefault()); private final RssReader rssReader; private final RSSFeedsConfig config; - private final Predicate defaultChannelPattern; + private final Predicate fallbackChannelPattern; private final Map> targetChannelPatterns; private final int interval; private final Database database; @@ -86,8 +86,8 @@ public RSSHandlerRoutine(Config config, Database database) { this.config = config.getRSSFeedsConfig(); this.interval = this.config.rssPollInterval(); this.database = database; - this.defaultChannelPattern = - Pattern.compile(this.config.javaNewsChannelPattern()).asMatchPredicate(); + this.fallbackChannelPattern = + Pattern.compile(this.config.fallbackChannelPattern()).asMatchPredicate(); this.targetChannelPatterns = new HashMap<>(); this.config.feeds().forEach(feed -> { if (feed.targetChannelPattern() != null) { @@ -249,7 +249,7 @@ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { // If the target channel was not found, use the fallback return jda.getTextChannelCache() .stream() - .filter(channel -> defaultChannelPattern.test(channel.getName())) + .filter(channel -> fallbackChannelPattern.test(channel.getName())) .findFirst(); } From e3028faf94bea85f9b6cb3b8e956ec5dcac7f161 Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 20:07:39 +0200 Subject: [PATCH 37/54] refactor: remove unnecessary throws exception in method signature --- .../togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index aa94990912..56b9cd1fb2 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -116,7 +116,7 @@ public void runRoutine(@Nonnull JDA jda) { * This handles fetching the latest posts from the given URL, checking which ones have already * been posted by reading information from the database and updating the last posted date. */ - private void sendRSS(JDA jda, RSSFeed feedConfig) throws DateTimeParseException { + private void sendRSS(JDA jda, RSSFeed feedConfig) { // Don't proceed if the text channel was not found TextChannel textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); if (textChannel == null) { From 34974e5a76b4c36ba24c64b3aab0611c6c42758c Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 20:31:08 +0200 Subject: [PATCH 38/54] docs: add a few missing parameters and fix typos --- .../togetherjava/tjbot/config/RSSFeed.java | 10 +++++++ .../features/javamail/RSSHandlerRoutine.java | 26 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index 1ebbd80e6f..382274646b 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -6,11 +6,21 @@ import java.util.Objects; +/** + * Represents an RSS feed configuration. + */ public record RSSFeed(@JsonProperty(value = "url", required = true) String url, @JsonProperty(value = "targetChannelPattern") @Nullable String targetChannelPattern, @JsonProperty(value = "dateFormatterPattern", required = true) String dateFormatterPattern) { + /** + * Constructs an RSSFeed object. + * + * @param url the URL of the RSS feed + * @param targetChannelPattern the target channel pattern + * @param dateFormatterPattern the date formatter pattern + */ public RSSFeed { Objects.requireNonNull(url); Objects.requireNonNull(targetChannelPattern); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 56b9cd1fb2..844ef3a3d8 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -115,6 +115,9 @@ public void runRoutine(@Nonnull JDA jda) { *

* This handles fetching the latest posts from the given URL, checking which ones have already * been posted by reading information from the database and updating the last posted date. + * + * @param jda The JDA instance. + * @param feedConfig The configuration object for the RSS feed. */ private void sendRSS(JDA jda, RSSFeed feedConfig) { // Don't proceed if the text channel was not found @@ -200,6 +203,8 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { * If either of the function inputs are null, the oldest-possible {@link ZonedDateTime} will get * returned instead. * + * @param item The {@link Item} from which to extract the date. + * @param dateTimeFormat The format of the date time string. * @return The computed {@link ZonedDateTime} */ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) { @@ -213,10 +218,14 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma } /** - * Given a list of RSS feed items, this function checks if the dates are valid + * Given a list of RSS feed items, this function checks if the dates are valid. *

* This assumes that all items share the same date format (as is usual in an RSS feed response), * therefore it only checks the first item's date for the final result. + * + * @param rssFeeds the list of RSS feed items + * @param feedConfig the RSS feed configuration containing date formatter pattern + * @return true if the date format is valid, false otherwise */ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig) { try { @@ -236,6 +245,10 @@ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig /** * Attempts to find a text channel from a given RSS feed configuration. + * + * @param jda the JDA instance + * @param feed the RSS feed configuration to search for a text channel + * @return an {@link Optional} containing the found text channel if it exists, otherwise empty */ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { // Attempt to find the target channel @@ -255,6 +268,10 @@ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { /** * Provides the {@link EmbedBuilder} from an RSS item used for sending RSS posts. + * + * @param item the RSS item to construct the embed message from + * @param feedConfig the configuration of the RSS feed + * @return the constructed {@link EmbedBuilder} containing information from the RSS item */ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) { final EmbedBuilder embedBuilder = new EmbedBuilder(); @@ -291,6 +308,9 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) /** * Fetches a list of {@link Item} from a given RSS url. + * + * @param rssUrl the URL of the RSS feed to fetch + * @return a list of {@link Item} parsed from the RSS feed */ private List fetchRss(String rssUrl) { try { @@ -305,6 +325,10 @@ private List fetchRss(String rssUrl) { * Helper function for parsing a given date value to a {@link ZonedDateTime} with a given * format. * + * @param date the date value to parse, can be null + * @param format the format pattern to use for parsing + * @return the parsed {@link ZonedDateTime} object + * @throws DateTimeParseException if the date cannot be parsed */ private static ZonedDateTime getZonedDateTime(@Nullable String date, @NotNull String format) throws DateTimeParseException { From e2843157a40f51b7614e97da2e646cd7cd7a6b32 Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 20:47:58 +0200 Subject: [PATCH 39/54] refactor: reduce try-catch scope and added clarifying comment --- .../features/javamail/RSSHandlerRoutine.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 844ef3a3d8..e06b5aec17 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -228,16 +228,19 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma * @return true if the date format is valid, false otherwise */ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig) { - try { - final Item firstRssFeed = rssFeeds.getFirst(); - String firstRssFeedPubDate = firstRssFeed.getPubDate().orElse(null); + final Item firstRssFeed = rssFeeds.getFirst(); + String firstRssFeedPubDate = firstRssFeed.getPubDate().orElse(null); - if (firstRssFeedPubDate == null) { - return false; - } + if (firstRssFeedPubDate == null) { + return false; + } + try { + // If this throws a DateTimeParseException then it's certain + // that the format pattern defined in the config and the + // feed's actual format differ. getZonedDateTime(firstRssFeedPubDate, feedConfig.dateFormatterPattern()); - } catch (Exception e) { + } catch (DateTimeParseException e) { return false; } return true; From 9deaa0261248b17d14f4deff93718db4f6f14d58 Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 21:05:40 +0200 Subject: [PATCH 40/54] perf: use Stream instead of StringBuilder --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index e06b5aec17..3f21ec5c49 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -12,6 +12,7 @@ import org.jooq.tools.StringUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.togetherjava.tjbot.db.generated.tables.RssFeed.RSS_FEED; @@ -294,13 +296,13 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) if (rawDescription.isPresent() && !rawDescription.get().isEmpty()) { Document fullDescription = Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.get())); - StringBuilder finalDescription = new StringBuilder(); - fullDescription.body() + String finalDescription = fullDescription.body() .select("*") - .forEach(p -> finalDescription.append(p.text()).append(". ")); + .stream() + .map(Element::text) + .collect(Collectors.joining(". ")); - embedBuilder - .setDescription(StringUtils.abbreviate(finalDescription.toString(), MAX_CONTENTS)); + embedBuilder.setDescription(StringUtils.abbreviate(finalDescription, MAX_CONTENTS)); return embedBuilder; } From 237d6ba5e5737c6b82b8839e056efcef11302f3d Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 21:09:26 +0200 Subject: [PATCH 41/54] refactor: put fallback case into an else statement --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 3f21ec5c49..a05f75f2cb 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -256,19 +256,18 @@ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig * @return an {@link Optional} containing the found text channel if it exists, otherwise empty */ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { - // Attempt to find the target channel + // Attempt to find the target channel, use the fallback otherwise if (feed.targetChannelPattern() != null) { return jda.getTextChannelCache() .stream() .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) .findFirst(); + } else { + return jda.getTextChannelCache() + .stream() + .filter(channel -> fallbackChannelPattern.test(channel.getName())) + .findFirst(); } - - // If the target channel was not found, use the fallback - return jda.getTextChannelCache() - .stream() - .filter(channel -> fallbackChannelPattern.test(channel.getName())) - .findFirst(); } /** From b48c32759bd32098b077298ac00bf22ebdc8c002 Mon Sep 17 00:00:00 2001 From: christolis Date: Fri, 1 Mar 2024 21:14:40 +0200 Subject: [PATCH 42/54] refactor: switch to Map#containsKey() for targetChannelPatterns --- .../togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index a05f75f2cb..ec7e1d479c 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -257,7 +257,7 @@ private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig */ private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { // Attempt to find the target channel, use the fallback otherwise - if (feed.targetChannelPattern() != null) { + if (targetChannelPatterns.containsKey(feed)) { return jda.getTextChannelCache() .stream() .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) From e257595995e5a0d1bd7bd0a62f14b4ab18d2a73b Mon Sep 17 00:00:00 2001 From: christolis Date: Sat, 2 Mar 2024 00:33:15 +0200 Subject: [PATCH 43/54] feat: modularize sendRss() method and improve variable names This also changes the functionality of how new RSS feeds get dealt with for the first time. Before this commit, all items would get posted as embeds on Discord, and as a result, that would bombard the target channel with RSS posts once the routine executes. This commit changes this behavior by assuming that all RSS posts have been posted and it should only consider posting posts newer than the registered date. --- .../features/javamail/RSSHandlerRoutine.java | 150 +++++++++++++----- 1 file changed, 107 insertions(+), 43 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index ec7e1d479c..a1240f2e9f 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -122,7 +122,6 @@ public void runRoutine(@Nonnull JDA jda) { * @param feedConfig The configuration object for the RSS feed. */ private void sendRSS(JDA jda, RSSFeed feedConfig) { - // Don't proceed if the text channel was not found TextChannel textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); if (textChannel == null) { logger.warn("Tried to sendRss, got empty response (channel {} not found)", @@ -130,59 +129,130 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { return; } - // Acquire the list of RSS items from the configured URL - List rssItems = fetchRss(feedConfig.url()); + List rssItems = fetchRSSItemsFromURL(feedConfig.url()); if (rssItems.isEmpty()) { return; } - // Do not proceed if the configured date format does not match - // the actual date format provided by the feed's contents - if (!isValidDateFormat(rssItems, feedConfig)) { + // We only get the first RSS item from the list since we are + // assuming that the whole RSS feed uses the same format + if (!isValidDateFormat(rssItems.getFirst(), feedConfig)) { logger.warn("Could not find valid date format for RSS feed {}", feedConfig.url()); return; } - // Attempt to find any stored information regarding the provided RSS URL - RssFeedRecord dateResult = database.read(context -> context.selectFrom(RSS_FEED) - .where(RSS_FEED.URL.eq(feedConfig.url())) - .limit(1) - .fetchAny()); + RssFeedRecord rssFeedRecord = getRssFeedRecordFromDatabase(feedConfig); + Optional lastSavedDate = getLastSavedDateFromDatabaseRecord(rssFeedRecord, + feedConfig.dateFormatterPattern()); - String dateStr = dateResult == null ? null : dateResult.getLastDate(); - ZonedDateTime lastSavedDate = getZonedDateTime(dateStr, feedConfig.dateFormatterPattern()); + ZonedDateTime lastPostedDate = + getLatestPostDateFromItems(rssItems, feedConfig.dateFormatterPattern()); + updateLastDateToDatabase(feedConfig, rssFeedRecord, lastPostedDate); + + if (lastSavedDate.isEmpty()) { + return; + } final Predicate shouldItemBePosted = item -> { ZonedDateTime itemPubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); - return itemPubDate.isAfter(lastSavedDate); + return itemPubDate.isAfter(lastSavedDate.get()); }; - // Date that will be stored in the database at the end - ZonedDateTime lastPostedDate = lastSavedDate; + rssItems.reversed() + .stream() + .filter(shouldItemBePosted) + .forEach(item -> postItem(textChannel, item, feedConfig)); + } - // Send each item that should be posted and concurrently - // find the post with the latest date - for (Item item : rssItems.reversed()) { - if (!shouldItemBePosted.test(item)) { - continue; - } + /** + * Retrieves an RSS feed record from the database based on the provided RSS feed configuration. + * + * @param feedConfig the RSS feed configuration to retrieve the record for + * @return the RSS feed record retrieved from the database + */ + private RssFeedRecord getRssFeedRecordFromDatabase(RSSFeed feedConfig) { + return database.read(context -> context.selectFrom(RSS_FEED) + .where(RSS_FEED.URL.eq(feedConfig.url())) + .limit(1) + .fetchAny()); + } - MessageEmbed embed = constructEmbedMessage(item, feedConfig).build(); - textChannel.sendMessageEmbeds(List.of(embed)).queue(); + /** + * Retrieves the last saved date from the database record associated with the given RSS feed + * record. + *

+ * If the RSS feed record is null, returns an empty {@link Optional}. + * + * @param rssRecord the RSS feed record to retrieve the last saved date from + * @param dateFormatterPattern the pattern used to parse the date from the database record + * @return An {@link Optional} containing the last saved date if it could be retrieved and + * parsed successfully, otherwise an empty {@link Optional} + */ + private Optional getLastSavedDateFromDatabaseRecord(RssFeedRecord rssRecord, + String dateFormatterPattern) { + if (rssRecord == null) { + return Optional.empty(); + } - // Get the last posted date so that we update the database - ZonedDateTime pubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); - if (pubDate.isAfter(lastPostedDate)) { - lastPostedDate = pubDate; - } + try { + ZonedDateTime savedDate = + getZonedDateTime(rssRecord.getLastDate(), dateFormatterPattern); + return Optional.of(savedDate); + } catch (DateTimeParseException e) { + return Optional.empty(); + } + } + + /** + * Retrieves the latest post date from the given list of items. + * + * @param items the list of items to retrieve the latest post date from + * @param dateFormatterPattern the pattern used to parse the date from the database record + * @return the latest post date as a {@link ZonedDateTime} object, or null if the list is empty + */ + private ZonedDateTime getLatestPostDateFromItems(List items, + String dateFormatterPattern) { + return items.stream() + .map(item -> getDateTimeFromItem(item, dateFormatterPattern)) + .max(ZonedDateTime::compareTo) + .orElse(null); + } + + /** + * Posts an RSS item to a text channel. + * + * @param textChannel the text channel to which the item will be posted + * @param rssItem the RSS item to post + * @param feedConfig the RSS feed configuration + */ + private void postItem(TextChannel textChannel, Item rssItem, RSSFeed feedConfig) { + MessageEmbed embed = constructEmbedMessage(rssItem, feedConfig).build(); + textChannel.sendMessageEmbeds(List.of(embed)).queue(); + } + + /** + * Updates the last posted date to the database for the specified RSS feed configuration. + *

+ * This will insert a new entry to the database if the provided {@link RssFeedRecord} is + * null. + * + * @param feedConfig the RSS feed configuration + * @param rssFeedRecord the record representing the RSS feed. can be null if not found in the + * database + * @param lastPostedDate the last posted date to be updated + */ + private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecord rssFeedRecord, + ZonedDateTime lastPostedDate) { + if (lastPostedDate == null) { + return; } - // Finally, save the last posted date to the database. DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); String lastDateStr = lastPostedDate.format(dateTimeFormatter); - if (dateResult == null) { + + if (rssFeedRecord == null) { database.write(context -> context.newRecord(RSS_FEED) .setUrl(feedConfig.url()) .setLastDate(lastDateStr) @@ -190,8 +260,6 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { return; } - // If we already have an existing record with the given URL, - // now is the time to update it database.write(context -> context.update(RSS_FEED) .set(RSS_FEED.LAST_DATE, lastDateStr) .where(RSS_FEED.URL.eq(feedConfig.url())) @@ -220,18 +288,14 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma } /** - * Given a list of RSS feed items, this function checks if the dates are valid. - *

- * This assumes that all items share the same date format (as is usual in an RSS feed response), - * therefore it only checks the first item's date for the final result. + * Checks if the dates between an RSS item and the provided config match. * - * @param rssFeeds the list of RSS feed items - * @param feedConfig the RSS feed configuration containing date formatter pattern + * @param rssItem the RSS feed item + * @param feedConfig the RSS feed configuration containing the date formatter pattern * @return true if the date format is valid, false otherwise */ - private static boolean isValidDateFormat(List rssFeeds, RSSFeed feedConfig) { - final Item firstRssFeed = rssFeeds.getFirst(); - String firstRssFeedPubDate = firstRssFeed.getPubDate().orElse(null); + private static boolean isValidDateFormat(Item rssItem, RSSFeed feedConfig) { + String firstRssFeedPubDate = rssItem.getPubDate().orElse(null); if (firstRssFeedPubDate == null) { return false; @@ -316,7 +380,7 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) * @param rssUrl the URL of the RSS feed to fetch * @return a list of {@link Item} parsed from the RSS feed */ - private List fetchRss(String rssUrl) { + private List fetchRSSItemsFromURL(String rssUrl) { try { return rssReader.read(rssUrl).toList(); } catch (IOException e) { From 18135e6c81729fa309dbade3dc6d1af9801a12a3 Mon Sep 17 00:00:00 2001 From: christolis Date: Sat, 2 Mar 2024 01:30:18 +0200 Subject: [PATCH 44/54] refactor: remove star import --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index a1240f2e9f..9c6a37a500 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -31,7 +31,10 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.regex.Pattern; From 48f0e76f7693f1329e89eb954436d74e24c95d87 Mon Sep 17 00:00:00 2001 From: christolis Date: Tue, 5 Mar 2024 12:56:56 +0200 Subject: [PATCH 45/54] fix: add Objects#requireNonNull() on rssConfig --- .../src/main/java/org/togetherjava/tjbot/config/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 376a1d5149..a3d37bffe0 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -125,7 +125,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.sourceCodeBaseUrl = Objects.requireNonNull(sourceCodeBaseUrl); this.jshell = Objects.requireNonNull(jshell); this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); - this.rssFeedsConfig = rssFeedsConfig; + this.rssFeedsConfig = Objects.requireNonNull(rssFeedsConfig); this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); } From 6f9e2ddee0f10a9b8e844af171b37e826fc43269 Mon Sep 17 00:00:00 2001 From: christolis Date: Sun, 10 Mar 2024 21:18:08 +0200 Subject: [PATCH 46/54] changes --- .../features/javamail/RSSHandlerRoutine.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 9c6a37a500..97b7f6b6d0 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -150,7 +150,10 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { ZonedDateTime lastPostedDate = getLatestPostDateFromItems(rssItems, feedConfig.dateFormatterPattern()); - updateLastDateToDatabase(feedConfig, rssFeedRecord, lastPostedDate); + + if (lastPostedDate != null) { + updateLastDateToDatabase(feedConfig, rssFeedRecord, lastPostedDate); + } if (lastSavedDate.isEmpty()) { return; @@ -193,7 +196,7 @@ private RssFeedRecord getRssFeedRecordFromDatabase(RSSFeed feedConfig) { * parsed successfully, otherwise an empty {@link Optional} */ private Optional getLastSavedDateFromDatabaseRecord(RssFeedRecord rssRecord, - String dateFormatterPattern) { + String dateFormatterPattern) throws DateTimeParseException { if (rssRecord == null) { return Optional.empty(); } @@ -241,16 +244,14 @@ private void postItem(TextChannel textChannel, Item rssItem, RSSFeed feedConfig) * null. * * @param feedConfig the RSS feed configuration - * @param rssFeedRecord the record representing the RSS feed. can be null if not found in the + * @param rssFeedRecord the record representing the RSS feed, can be null if not found in the * database * @param lastPostedDate the last posted date to be updated + * + * @throws DateTimeParseException if the date cannot be parsed */ private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecord rssFeedRecord, ZonedDateTime lastPostedDate) { - if (lastPostedDate == null) { - return; - } - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(feedConfig.dateFormatterPattern()); String lastDateStr = lastPostedDate.format(dateTimeFormatter); @@ -369,11 +370,10 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) .collect(Collectors.joining(". ")); embedBuilder.setDescription(StringUtils.abbreviate(finalDescription, MAX_CONTENTS)); - return embedBuilder; + } else { + embedBuilder.setDescription("No description"); } - // Fill the description with a placeholder if the description was empty - embedBuilder.setDescription("No description"); return embedBuilder; } From 7f14bc2e8deb54faf81c20b539c6c66ae27061f5 Mon Sep 17 00:00:00 2001 From: christolis Date: Sun, 10 Mar 2024 22:10:12 +0200 Subject: [PATCH 47/54] feat: improvements from code reviews Co-authored-by: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> --- .../org/togetherjava/tjbot/config/Config.java | 2 + .../togetherjava/tjbot/config/RSSFeed.java | 1 + .../tjbot/config/RSSFeedsConfig.java | 4 ++ .../features/javamail/RSSHandlerRoutine.java | 65 +++++++++---------- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index a3d37bffe0..e819f8e7d1 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -411,6 +411,8 @@ public String getMemberCountCategoryPattern() { } /** + * Gets the RSS feeds configuration. + * * @return the RSS feeds configuration */ public RSSFeedsConfig getRSSFeedsConfig() { diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java index 382274646b..a9f68361a6 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeed.java @@ -20,6 +20,7 @@ public record RSSFeed(@JsonProperty(value = "url", required = true) String url, * @param url the URL of the RSS feed * @param targetChannelPattern the target channel pattern * @param dateFormatterPattern the date formatter pattern + * @throws NullPointerException if any of the parameters are null */ public RSSFeed { Objects.requireNonNull(url); diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index c19b036de6..343e9f2c83 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -8,6 +8,10 @@ /** * Represents the configuration for an RSS feed, which includes the list of feeds to subscribe to, a * pattern for identifying Java news channels, and the interval (in minutes) for polling the feeds. + * + * @param feeds The list of RSS feeds to subscribe to. + * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. + * @param rssPollInterval The interval (in minutes) for polling the RSS feeds for updates. */ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 97b7f6b6d0..644722aea5 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -127,7 +127,7 @@ public void runRoutine(@Nonnull JDA jda) { private void sendRSS(JDA jda, RSSFeed feedConfig) { TextChannel textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); if (textChannel == null) { - logger.warn("Tried to sendRss, got empty response (channel {} not found)", + logger.warn("Tried to send an RSS post, got empty response (channel {} not found)", feedConfig.targetChannelPattern()); return; } @@ -137,24 +137,28 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { return; } - // We only get the first RSS item from the list since we are - // assuming that the whole RSS feed uses the same format - if (!isValidDateFormat(rssItems.getFirst(), feedConfig)) { - logger.warn("Could not find valid date format for RSS feed {}", feedConfig.url()); - return; + for (Item item : rssItems) { + if (!isValidDateFormat(item, feedConfig)) { + logger.warn("Could not find valid or matching date format for RSS feed {}", + feedConfig.url()); + return; + } } - RssFeedRecord rssFeedRecord = getRssFeedRecordFromDatabase(feedConfig); - Optional lastSavedDate = getLastSavedDateFromDatabaseRecord(rssFeedRecord, - feedConfig.dateFormatterPattern()); - - ZonedDateTime lastPostedDate = + Optional rssFeedRecord = getRssFeedRecordFromDatabase(feedConfig); + Optional lastPostedDate = getLatestPostDateFromItems(rssItems, feedConfig.dateFormatterPattern()); - if (lastPostedDate != null) { - updateLastDateToDatabase(feedConfig, rssFeedRecord, lastPostedDate); + lastPostedDate.ifPresent( + date -> updateLastDateToDatabase(feedConfig, rssFeedRecord.orElse(null), date)); + + if (rssFeedRecord.isEmpty()) { + return; } + Optional lastSavedDate = getLastSavedDateFromDatabaseRecord( + rssFeedRecord.get(), feedConfig.dateFormatterPattern()); + if (lastSavedDate.isEmpty()) { return; } @@ -162,7 +166,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { final Predicate shouldItemBePosted = item -> { ZonedDateTime itemPubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); - return itemPubDate.isAfter(lastSavedDate.get()); + return itemPubDate.isAfter(lastSavedDate.orElseThrow()); }; rssItems.reversed() @@ -175,32 +179,27 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { * Retrieves an RSS feed record from the database based on the provided RSS feed configuration. * * @param feedConfig the RSS feed configuration to retrieve the record for - * @return the RSS feed record retrieved from the database + * @return an optional RSS feed record retrieved from the database */ - private RssFeedRecord getRssFeedRecordFromDatabase(RSSFeed feedConfig) { - return database.read(context -> context.selectFrom(RSS_FEED) + private Optional getRssFeedRecordFromDatabase(RSSFeed feedConfig) { + return Optional.ofNullable(database.read(context -> context.selectFrom(RSS_FEED) .where(RSS_FEED.URL.eq(feedConfig.url())) .limit(1) - .fetchAny()); + .fetchAny())); } /** * Retrieves the last saved date from the database record associated with the given RSS feed * record. - *

- * If the RSS feed record is null, returns an empty {@link Optional}. * - * @param rssRecord the RSS feed record to retrieve the last saved date from + * @param rssRecord an existing RSS feed record to retrieve the last saved date from * @param dateFormatterPattern the pattern used to parse the date from the database record * @return An {@link Optional} containing the last saved date if it could be retrieved and * parsed successfully, otherwise an empty {@link Optional} */ - private Optional getLastSavedDateFromDatabaseRecord(RssFeedRecord rssRecord, - String dateFormatterPattern) throws DateTimeParseException { - if (rssRecord == null) { - return Optional.empty(); - } - + private Optional getLastSavedDateFromDatabaseRecord( + @NotNull RssFeedRecord rssRecord, String dateFormatterPattern) + throws DateTimeParseException { try { ZonedDateTime savedDate = getZonedDateTime(rssRecord.getLastDate(), dateFormatterPattern); @@ -217,12 +216,11 @@ private Optional getLastSavedDateFromDatabaseRecord(RssFeedRecord * @param dateFormatterPattern the pattern used to parse the date from the database record * @return the latest post date as a {@link ZonedDateTime} object, or null if the list is empty */ - private ZonedDateTime getLatestPostDateFromItems(List items, + private Optional getLatestPostDateFromItems(List items, String dateFormatterPattern) { return items.stream() .map(item -> getDateTimeFromItem(item, dateFormatterPattern)) - .max(ZonedDateTime::compareTo) - .orElse(null); + .max(ZonedDateTime::compareTo); } /** @@ -274,8 +272,8 @@ private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecor * Attempts to get a {@link ZonedDateTime} from an {@link Item} with a provided string date time * format. *

- * If either of the function inputs are null, the oldest-possible {@link ZonedDateTime} will get - * returned instead. + * If either of the function inputs are null or a {@link DateTimeParseException} is caught, the + * oldest-possible {@link ZonedDateTime} will get returned instead. * * @param item The {@link Item} from which to extract the date. * @param dateTimeFormat The format of the date time string. @@ -381,7 +379,8 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) * Fetches a list of {@link Item} from a given RSS url. * * @param rssUrl the URL of the RSS feed to fetch - * @return a list of {@link Item} parsed from the RSS feed + * @return a list of {@link Item} parsed from the RSS feed, or an empty list if there's an + * {@link IOException} */ private List fetchRSSItemsFromURL(String rssUrl) { try { From 82fd9b7e0d688363115b22d8b04888614f67f1e6 Mon Sep 17 00:00:00 2001 From: christolis Date: Wed, 20 Mar 2024 12:32:19 +0200 Subject: [PATCH 48/54] feat: add DateTimeParseException in signature --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 644722aea5..7c89a49d4c 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -278,8 +278,10 @@ private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecor * @param item The {@link Item} from which to extract the date. * @param dateTimeFormat The format of the date time string. * @return The computed {@link ZonedDateTime} + * @throws DateTimeParseException if the date cannot be parsed */ - private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) { + private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) + throws DateTimeParseException { String pubDate = item.getPubDate().orElse(null); if (pubDate == null || dateTimeFormat == null) { From 32f5f9bdbfaa3032489165b060fe2ff22206a90e Mon Sep 17 00:00:00 2001 From: christolis Date: Thu, 4 Apr 2024 09:44:47 +0300 Subject: [PATCH 49/54] refactor: use .orElseThrow() instead of .get() --- .../tjbot/features/javamail/RSSHandlerRoutine.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 7c89a49d4c..77d49ebdbf 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -157,7 +157,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { } Optional lastSavedDate = getLastSavedDateFromDatabaseRecord( - rssFeedRecord.get(), feedConfig.dateFormatterPattern()); + rssFeedRecord.orElseThrow(), feedConfig.dateFormatterPattern()); if (lastSavedDate.isEmpty()) { return; @@ -360,9 +360,9 @@ private static EmbedBuilder constructEmbedMessage(Item item, RSSFeed feedConfig) embedBuilder.setAuthor(item.getChannel().getLink()); // Process embed's description if a raw description was provided - if (rawDescription.isPresent() && !rawDescription.get().isEmpty()) { + if (rawDescription.isPresent() && !rawDescription.orElseThrow().isEmpty()) { Document fullDescription = - Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.get())); + Jsoup.parse(StringEscapeUtils.unescapeHtml4(rawDescription.orElseThrow())); String finalDescription = fullDescription.body() .select("*") .stream() From cf67e4d2a2ec660c29a5fa04985c0bca015e04de Mon Sep 17 00:00:00 2001 From: Nathan Weisz <151496498+nateweisz@users.noreply.github.com> Date: Tue, 9 Apr 2024 06:32:41 -0700 Subject: [PATCH 50/54] fixed missing coma after rss configuration Co-authored-by: Tanish --- application/config.json.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/config.json.template b/application/config.json.template index 472150f481..765a7317c2 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -114,6 +114,6 @@ ], "fallbackChannelPattern": "", "rssPollInterval": 10 - } + }, "memberCountCategoryPattern": "Info" } From 2fb249f2f326de8464c341d93f387c1afb169fcd Mon Sep 17 00:00:00 2001 From: Nathan Weisz Date: Tue, 9 Apr 2024 06:44:47 -0700 Subject: [PATCH 51/54] fix(rss): feeds support multiple channels fix(rss): polling interval is now clear on unit of time fix(rss): posting messages now uses forEachOrdered --- application/config.json.template | 2 +- .../tjbot/config/RSSFeedsConfig.java | 6 +-- .../features/javamail/RSSHandlerRoutine.java | 38 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index 765a7317c2..3071633109 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -113,7 +113,7 @@ } ], "fallbackChannelPattern": "", - "rssPollInterval": 10 + "pollIntervalInMinutes": 10 }, "memberCountCategoryPattern": "Info" } diff --git a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java index 343e9f2c83..1c3371f71a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/RSSFeedsConfig.java @@ -11,19 +11,19 @@ * * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. - * @param rssPollInterval The interval (in minutes) for polling the RSS feeds for updates. + * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. */ public record RSSFeedsConfig(@JsonProperty(value = "feeds", required = true) List feeds, @JsonProperty(value = "fallbackChannelPattern", required = true) String fallbackChannelPattern, - @JsonProperty(value = "rssPollInterval", required = true) int rssPollInterval) { + @JsonProperty(value = "pollIntervalInMinutes", required = true) int pollIntervalInMinutes) { /** * Constructs a new {@link RSSFeedsConfig}. * * @param feeds The list of RSS feeds to subscribe to. * @param fallbackChannelPattern The pattern used to identify the fallback text channel to use. - * @param rssPollInterval The interval (in minutes) for polling the RSS feeds for updates. + * @param pollIntervalInMinutes The interval (in minutes) for polling the RSS feeds for updates. * @throws NullPointerException if any of the parameters (feeds or fallbackChannelPattern) are * null */ diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 77d49ebdbf..0e32886e50 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -89,7 +89,7 @@ public final class RSSHandlerRoutine implements Routine { */ public RSSHandlerRoutine(Config config, Database database) { this.config = config.getRSSFeedsConfig(); - this.interval = this.config.rssPollInterval(); + this.interval = this.config.pollIntervalInMinutes(); this.database = database; this.fallbackChannelPattern = Pattern.compile(this.config.fallbackChannelPattern()).asMatchPredicate(); @@ -125,8 +125,8 @@ public void runRoutine(@Nonnull JDA jda) { * @param feedConfig The configuration object for the RSS feed. */ private void sendRSS(JDA jda, RSSFeed feedConfig) { - TextChannel textChannel = getTextChannelFromFeed(jda, feedConfig).orElse(null); - if (textChannel == null) { + List textChannels = getTextChannelsFromFeed(jda, feedConfig).orElse(null); + if (textChannels == null || textChannels.isEmpty()) { logger.warn("Tried to send an RSS post, got empty response (channel {} not found)", feedConfig.targetChannelPattern()); return; @@ -172,7 +172,7 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { rssItems.reversed() .stream() .filter(shouldItemBePosted) - .forEach(item -> postItem(textChannel, item, feedConfig)); + .forEachOrdered(item -> postItem(textChannels, item, feedConfig)); } /** @@ -226,13 +226,13 @@ private Optional getLatestPostDateFromItems(List items, /** * Posts an RSS item to a text channel. * - * @param textChannel the text channel to which the item will be posted + * @param textChannels the text channels to which the item will be posted * @param rssItem the RSS item to post * @param feedConfig the RSS feed configuration */ - private void postItem(TextChannel textChannel, Item rssItem, RSSFeed feedConfig) { + private void postItem(List textChannels, Item rssItem, RSSFeed feedConfig) { MessageEmbed embed = constructEmbedMessage(rssItem, feedConfig).build(); - textChannel.sendMessageEmbeds(List.of(embed)).queue(); + textChannels.forEach(channel -> channel.sendMessageEmbeds(List.of(embed)).queue()); } /** @@ -317,24 +317,24 @@ private static boolean isValidDateFormat(Item rssItem, RSSFeed feedConfig) { } /** - * Attempts to find a text channel from a given RSS feed configuration. + * Attempts to find text channels from a given RSS feed configuration. * * @param jda the JDA instance - * @param feed the RSS feed configuration to search for a text channel - * @return an {@link Optional} containing the found text channel if it exists, otherwise empty + * @param feed the RSS feed configuration to search for text channels + * @return an {@link Optional} containing a list of text channels found, or empty if none are found */ - private Optional getTextChannelFromFeed(JDA jda, RSSFeed feed) { + private Optional> getTextChannelsFromFeed(JDA jda, RSSFeed feed) { // Attempt to find the target channel, use the fallback otherwise if (targetChannelPatterns.containsKey(feed)) { - return jda.getTextChannelCache() - .stream() - .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) - .findFirst(); + return Optional.of(jda.getTextChannelCache() + .stream() + .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) + .toList()); } else { - return jda.getTextChannelCache() - .stream() - .filter(channel -> fallbackChannelPattern.test(channel.getName())) - .findFirst(); + return Optional.of(jda.getTextChannelCache() + .stream() + .filter(channel -> fallbackChannelPattern.test(channel.getName())) + .toList()); } } From 0c7231a07b08917fd3b4944ae3f1e95ca329c0b8 Mon Sep 17 00:00:00 2001 From: Nathan Weisz Date: Tue, 9 Apr 2024 08:09:59 -0700 Subject: [PATCH 52/54] Optional> -> List --- .../features/javamail/RSSHandlerRoutine.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 0e32886e50..8959a58626 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -125,8 +125,8 @@ public void runRoutine(@Nonnull JDA jda) { * @param feedConfig The configuration object for the RSS feed. */ private void sendRSS(JDA jda, RSSFeed feedConfig) { - List textChannels = getTextChannelsFromFeed(jda, feedConfig).orElse(null); - if (textChannels == null || textChannels.isEmpty()) { + List textChannels = getTextChannelsFromFeed(jda, feedConfig); + if (textChannels.isEmpty()) { logger.warn("Tried to send an RSS post, got empty response (channel {} not found)", feedConfig.targetChannelPattern()); return; @@ -321,20 +321,20 @@ private static boolean isValidDateFormat(Item rssItem, RSSFeed feedConfig) { * * @param jda the JDA instance * @param feed the RSS feed configuration to search for text channels - * @return an {@link Optional} containing a list of text channels found, or empty if none are found + * @return an {@link List} of the text channels found, or empty if none are found */ - private Optional> getTextChannelsFromFeed(JDA jda, RSSFeed feed) { + private List getTextChannelsFromFeed(JDA jda, RSSFeed feed) { // Attempt to find the target channel, use the fallback otherwise if (targetChannelPatterns.containsKey(feed)) { - return Optional.of(jda.getTextChannelCache() + return jda.getTextChannelCache() .stream() .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) - .toList()); + .toList(); } else { - return Optional.of(jda.getTextChannelCache() + return jda.getTextChannelCache() .stream() .filter(channel -> fallbackChannelPattern.test(channel.getName())) - .toList()); + .toList(); } } From 49bdf539e5deeed8d8d251648a26599b808eabf0 Mon Sep 17 00:00:00 2001 From: Nathan Weisz Date: Tue, 9 Apr 2024 08:24:53 -0700 Subject: [PATCH 53/54] extract item post predicate to seperate function --- .../features/javamail/RSSHandlerRoutine.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index 8959a58626..d9fcee9d14 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -145,6 +145,15 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { } } + final Predicate shouldItemBePosted = prepareItemPostPredicate(feedConfig, rssItems); + if (shouldItemBePosted == null) return; + rssItems.reversed() + .stream() + .filter(shouldItemBePosted) + .forEachOrdered(item -> postItem(textChannels, item, feedConfig)); + } + + private Predicate prepareItemPostPredicate(RSSFeed feedConfig, List rssItems) { Optional rssFeedRecord = getRssFeedRecordFromDatabase(feedConfig); Optional lastPostedDate = getLatestPostDateFromItems(rssItems, feedConfig.dateFormatterPattern()); @@ -153,26 +162,21 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { date -> updateLastDateToDatabase(feedConfig, rssFeedRecord.orElse(null), date)); if (rssFeedRecord.isEmpty()) { - return; + return null; } Optional lastSavedDate = getLastSavedDateFromDatabaseRecord( rssFeedRecord.orElseThrow(), feedConfig.dateFormatterPattern()); if (lastSavedDate.isEmpty()) { - return; + return null; } - final Predicate shouldItemBePosted = item -> { + return item -> { ZonedDateTime itemPubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); return itemPubDate.isAfter(lastSavedDate.orElseThrow()); }; - - rssItems.reversed() - .stream() - .filter(shouldItemBePosted) - .forEachOrdered(item -> postItem(textChannels, item, feedConfig)); } /** From dfcd27e6c2d581a50a294fd3698f33f0862a85f4 Mon Sep 17 00:00:00 2001 From: Nathan Weisz Date: Tue, 9 Apr 2024 16:27:25 -0700 Subject: [PATCH 54/54] various changes from null to optional --- .../features/javamail/RSSHandlerRoutine.java | 53 +++++++++---------- .../tjbot/features/javamail/package-info.java | 10 ++++ 2 files changed, 35 insertions(+), 28 deletions(-) create mode 100644 application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java index d9fcee9d14..8ab9c2d528 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/RSSHandlerRoutine.java @@ -7,7 +7,6 @@ import net.dv8tion.jda.api.entities.MessageEmbed; import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; import org.apache.commons.text.StringEscapeUtils; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jooq.tools.StringUtils; import org.jsoup.Jsoup; @@ -105,7 +104,6 @@ public RSSHandlerRoutine(Config config, Database database) { } @Override - @NotNull public Schedule createSchedule() { return new Schedule(ScheduleMode.FIXED_DELAY, 0, interval, TimeUnit.MINUTES); } @@ -145,15 +143,18 @@ private void sendRSS(JDA jda, RSSFeed feedConfig) { } } - final Predicate shouldItemBePosted = prepareItemPostPredicate(feedConfig, rssItems); - if (shouldItemBePosted == null) return; + final Optional> shouldItemBePosted = + prepareItemPostPredicate(feedConfig, rssItems); + if (shouldItemBePosted.isEmpty()) + return; rssItems.reversed() .stream() - .filter(shouldItemBePosted) + .filter(shouldItemBePosted.get()) .forEachOrdered(item -> postItem(textChannels, item, feedConfig)); } - private Predicate prepareItemPostPredicate(RSSFeed feedConfig, List rssItems) { + private Optional> prepareItemPostPredicate(RSSFeed feedConfig, + List rssItems) { Optional rssFeedRecord = getRssFeedRecordFromDatabase(feedConfig); Optional lastPostedDate = getLatestPostDateFromItems(rssItems, feedConfig.dateFormatterPattern()); @@ -162,21 +163,21 @@ private Predicate prepareItemPostPredicate(RSSFeed feedConfig, List date -> updateLastDateToDatabase(feedConfig, rssFeedRecord.orElse(null), date)); if (rssFeedRecord.isEmpty()) { - return null; + return Optional.empty(); } Optional lastSavedDate = getLastSavedDateFromDatabaseRecord( rssFeedRecord.orElseThrow(), feedConfig.dateFormatterPattern()); if (lastSavedDate.isEmpty()) { - return null; + return Optional.empty(); } - return item -> { + return Optional.of(item -> { ZonedDateTime itemPubDate = getDateTimeFromItem(item, feedConfig.dateFormatterPattern()); return itemPubDate.isAfter(lastSavedDate.orElseThrow()); - }; + }); } /** @@ -201,9 +202,8 @@ private Optional getRssFeedRecordFromDatabase(RSSFeed feedConfig) * @return An {@link Optional} containing the last saved date if it could be retrieved and * parsed successfully, otherwise an empty {@link Optional} */ - private Optional getLastSavedDateFromDatabaseRecord( - @NotNull RssFeedRecord rssRecord, String dateFormatterPattern) - throws DateTimeParseException { + private Optional getLastSavedDateFromDatabaseRecord(RssFeedRecord rssRecord, + String dateFormatterPattern) throws DateTimeParseException { try { ZonedDateTime savedDate = getZonedDateTime(rssRecord.getLastDate(), dateFormatterPattern); @@ -286,13 +286,10 @@ private void updateLastDateToDatabase(RSSFeed feedConfig, @Nullable RssFeedRecor */ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeFormat) throws DateTimeParseException { - String pubDate = item.getPubDate().orElse(null); + Optional pubDate = item.getPubDate(); - if (pubDate == null || dateTimeFormat == null) { - return ZONED_TIME_MIN; - } + return pubDate.map(s -> getZonedDateTime(s, dateTimeFormat)).orElse(ZONED_TIME_MIN); - return getZonedDateTime(pubDate, dateTimeFormat); } /** @@ -303,9 +300,9 @@ private static ZonedDateTime getDateTimeFromItem(Item item, String dateTimeForma * @return true if the date format is valid, false otherwise */ private static boolean isValidDateFormat(Item rssItem, RSSFeed feedConfig) { - String firstRssFeedPubDate = rssItem.getPubDate().orElse(null); + Optional firstRssFeedPubDate = rssItem.getPubDate(); - if (firstRssFeedPubDate == null) { + if (firstRssFeedPubDate.isEmpty()) { return false; } @@ -313,7 +310,7 @@ private static boolean isValidDateFormat(Item rssItem, RSSFeed feedConfig) { // If this throws a DateTimeParseException then it's certain // that the format pattern defined in the config and the // feed's actual format differ. - getZonedDateTime(firstRssFeedPubDate, feedConfig.dateFormatterPattern()); + getZonedDateTime(firstRssFeedPubDate.get(), feedConfig.dateFormatterPattern()); } catch (DateTimeParseException e) { return false; } @@ -331,14 +328,14 @@ private List getTextChannelsFromFeed(JDA jda, RSSFeed feed) { // Attempt to find the target channel, use the fallback otherwise if (targetChannelPatterns.containsKey(feed)) { return jda.getTextChannelCache() - .stream() - .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) - .toList(); + .stream() + .filter(channel -> targetChannelPatterns.get(feed).test(channel.getName())) + .toList(); } else { return jda.getTextChannelCache() - .stream() - .filter(channel -> fallbackChannelPattern.test(channel.getName())) - .toList(); + .stream() + .filter(channel -> fallbackChannelPattern.test(channel.getName())) + .toList(); } } @@ -406,7 +403,7 @@ private List fetchRSSItemsFromURL(String rssUrl) { * @return the parsed {@link ZonedDateTime} object * @throws DateTimeParseException if the date cannot be parsed */ - private static ZonedDateTime getZonedDateTime(@Nullable String date, @NotNull String format) + private static ZonedDateTime getZonedDateTime(@Nullable String date, String format) throws DateTimeParseException { if (date == null) { return ZONED_TIME_MIN; diff --git a/application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java b/application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java new file mode 100644 index 0000000000..36305399af --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/features/javamail/package-info.java @@ -0,0 +1,10 @@ +/** + * This package forwards rss feeds to discord + */ +@MethodsReturnNonnullByDefault +@ParametersAreNonnullByDefault +package org.togetherjava.tjbot.features.javamail; + +import org.togetherjava.tjbot.annotations.MethodsReturnNonnullByDefault; + +import javax.annotation.ParametersAreNonnullByDefault;