✨ News API 1.0

pom.xml

@@ -16,7 +16,16 @@
   <properties> 
     <java.version>11</java.version> 
   </properties>  
-  <dependencies> 
+  <dependencies>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-cache</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-xml</artifactId>
+      <version>2.14.1</version>
+    </dependency>
     <dependency> 
       <groupId>org.springframework.boot</groupId>  
       <artifactId>spring-boot-starter-web</artifactId> 

src/main/java/dev/wms/pwrapi/api/NewsAPI.java

@@ -0,0 +1,44 @@
+package dev.wms.pwrapi.api;
+
+import dev.wms.pwrapi.dto.news.Channel;
+import dev.wms.pwrapi.dto.news.FacultyType;
+import dev.wms.pwrapi.service.news.NewsService;
+import io.swagger.v3.oas.annotations.Operation;
+import lombok.AllArgsConstructor;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+
+@RestController
+@RequestMapping(value = "/api/news", produces = "application/json")
+@AllArgsConstructor
+public class NewsAPI {
+
+    private NewsService newsService;
+
+    @GetMapping("/general")
+    @Operation(summary = "Returns recent news from main PWr website",
+    description = "This endpoint implementation is based on RSS scraping, transforming and caching. " +
+            "RSS is taken from https://pwr.edu.pl/rss/pl/24.xml and which, as we observed, is a copy of news available on " +
+            "https://pwr.edu.pl/uczelnia/aktualnosci The data is cached for 15 minutes from last call, so recent news can be displayed with maximum 15 " +
+            "minutes delay.")
+    public ResponseEntity<Channel> fetchGeneralNews(){
+        return ResponseEntity.ok(newsService.fetchGeneralNews());
+    }
+
+    @GetMapping("/faculty")
+    @Operation(summary = "Returns recent news from faculty website",
+    description = "Implementation of this endpoint is based on RSS scraping or HTTP scraping. If the website supports RSS, " +
+            "rss is used, if not, the traditional HTTP scraping and HTML parsing is used. Detailed info about website and " +
+            "used method for certain faculties can be viewed here https://github.com/komp15/PWr-API/blob/feature_newsAPI/src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java " +
+            "The data is cached for 15 minutes from last call for all faculties, so recent news can be displayed with maximum 15 " +
+            "minutes delay.")
+    public ResponseEntity<Channel> fetchFacultyNews(
+            @RequestParam FacultyType faculty){
+        return ResponseEntity.ok(newsService.fetchNewsForFaculty(faculty));
+    }
+
+}

src/main/java/dev/wms/pwrapi/dao/news/NewsDAO.java

@@ -0,0 +1,97 @@
+package dev.wms.pwrapi.dao.news;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import dev.wms.pwrapi.dto.news.*;
+import dev.wms.pwrapi.utils.http.HttpUtils;
+import okhttp3.OkHttpClient;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.stereotype.Repository;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Repository
+public class NewsDAO {
+
+    private final DateTimeFormatter rssFormatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH);
+    private final DateTimeFormatter goalFormatter = DateTimeFormatter.ofPattern("dd.MM.yyyy");
+    private final Pattern datePattern = Pattern.compile("\\d{2} [a-zA-Z]{3} \\d{4}");
+
+    public Channel parsePwrRSS(String rssUrl) {
+        OkHttpClient client = new OkHttpClient();
+        String response = HttpUtils.makeRequestWithClientAndGetString(client, rssUrl);
+        XmlMapper xmlMapper = new XmlMapper();
+        xmlMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+        xmlMapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true);
+        xmlMapper.enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT);
+        xmlMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        try {
+            Rss items = xmlMapper.readValue(response, Rss.class);
+            for(Item item : items.getChannel().getItem()) reformatDate(item);
+
+            return items.getChannel();
+        } catch (JsonProcessingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void reformatDate(Item item){
+        Matcher matcher = datePattern.matcher(item.getPubDate());
+        matcher.find();
+        LocalDate parsedDate = LocalDate.parse(matcher.group(), rssFormatter);
+        item.setPubDate(parsedDate.format(goalFormatter));
+    }
+
+    public Channel getFacultyNews(FacultyType faculty) {
+        if(faculty.isRss){
+            return parsePwrRSS(faculty.url);
+        } else {
+            return parsePwrHTML(faculty.url);
+        }
+    }
+
+    private Channel parsePwrHTML(String url) {
+        OkHttpClient client = new OkHttpClient();
+        Document document = HttpUtils.makeRequestWithClientAndGetDocument(client, url);
+
+        Elements newsBoxes = document.getElementsByClass("news-box");
+        newsBoxes.removeIf(box -> box.text().isEmpty());
+
+        List<Item> channelItems = newsBoxes.parallelStream()
+                .map(newsBox -> parseItem(newsBox, url))
+                .toList();
+
+
+        return Channel.builder()
+                .title(document.getElementsByClass("portal-title").first().text())
+                .link(url)
+                .description("")
+                .item(channelItems)
+                .build();
+    }
+
+    private Item parseItem(Element element, String url){
+        Element textDiv = element.getElementsByClass("col-text").first();
+        return Item.builder()
+                .title(textDiv.getElementsByClass("title").first().attr("title"))
+                .link(getDomainFromNewsUrl(url) + textDiv.getElementsByClass("title").first().attr("href"))
+                .pubDate(textDiv.getElementsByClass("date").text().replace("Data: ","").strip()
+                        .split("Kategoria:")[0].strip())
+                .description(textDiv.getElementsByTag("p").get(1).text().replace("... więcej","")
+                        .replace("więcej","").strip())
+                .build();
+    }
+
+    private String getDomainFromNewsUrl(String url){
+        return url.split(".pl")[0] + ".pl";
+    }
+}

src/main/java/dev/wms/pwrapi/dto/news/Channel.java

@@ -0,0 +1,21 @@
+package dev.wms.pwrapi.dto.news;
+
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class Channel {
+    public String title;
+    public String link;
+    public String description;
+    @JacksonXmlElementWrapper(useWrapping = false)
+    public List<Item> item;
+}

src/main/java/dev/wms/pwrapi/dto/news/FacultyType.java

@@ -0,0 +1,31 @@
+package dev.wms.pwrapi.dto.news;
+
+public enum FacultyType {
+
+    INFORMATYKI_I_TELEKOMUNIKACJI("https://wit.pwr.edu.pl/rss/pl/189.xml", true),
+    ZARZADZANIA("https://wz.pwr.edu.pl/rss/pl/127.xml", true),
+    ELEKTRONIKI_MIKROSYSTEMOW_I_FOTONIKI("https://wefim.pwr.edu.pl/rss/pl/5.xml", true),
+    ARCHITEKTURY("https://wa.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    BUDOWNICTWA_LADOWEGO_I_WODNEGO("https://wbliw.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    CHEMICZNY("https://wch.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    ELEKTRYCZNY("https://weny.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    GEOINZYNIERII_GORNICTWA_I_GEOLOGII("https://wggg.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    INZYNIERII_SRODOWISKA("https://wis.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    MECHANICZNO_ENERGETYCZNY("https://wme.pwr.edu.pl/aktualnosci", false),
+    MECHANICZNY("https://wm.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    PODSTAWOWYCH_PROBLEMOW_TECHNIKI("https://wppt.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    MATEMATYKI("https://wmat.pwr.edu.pl/o-wydziale/aktualnosci", false),
+    FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_JELENIEJ_GORZE("https://jelenia-gora.pwr.edu.pl/o-filii/aktualnosci", false),
+    FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_WALBRZYCHU("https://walbrzych.pwr.edu.pl/o-filii/aktualnosci", false),
+    FILIA_POLITECHNIKI_WROCLAWSKIEJ_W_LEGNICY("https://legnica.pwr.edu.pl/o-wydziale/aktualnosci", false);
+
+
+
+    public final String url;
+    public final boolean isRss;
+
+    FacultyType(String url, boolean isRss) {
+        this.url = url;
+        this.isRss = isRss;
+    }
+}

src/main/java/dev/wms/pwrapi/dto/news/Item.java

@@ -0,0 +1,21 @@
+package dev.wms.pwrapi.dto.news;
+
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.extern.jackson.Jacksonized;
+
+@Data
+@Jacksonized
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class Item {
+    public String title;
+    public String link;
+    public String pubDate;
+    public String description;
+}
+

src/main/java/dev/wms/pwrapi/dto/news/Rss.java

@@ -0,0 +1,14 @@
+package dev.wms.pwrapi.dto.news;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class Rss {
+    public Channel channel;
+    public double version;
+    public String text;
+}

src/main/java/dev/wms/pwrapi/service/news/NewsService.java

@@ -0,0 +1,27 @@
+package dev.wms.pwrapi.service.news;
+
+import dev.wms.pwrapi.dao.news.NewsDAO;
+import dev.wms.pwrapi.dto.news.Channel;
+import dev.wms.pwrapi.dto.news.FacultyType;
+import lombok.AllArgsConstructor;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Service;
+
+@Service
+@AllArgsConstructor
+public class NewsService {
+
+    private NewsDAO newsDAO;
+
+    @Cacheable("pwr-news")
+    public Channel fetchGeneralNews() {
+        System.out.println("FETCHING NEW NEWSES");
+        return newsDAO.parsePwrRSS("https://pwr.edu.pl/rss/pl/24.xml");
+    }
+
+    @Cacheable("pwr-news")
+    public Channel fetchNewsForFaculty(FacultyType faculty) {
+        System.out.println("FETCHING NEW NEWSES");
+        return newsDAO.getFacultyNews(faculty);
+    }
+}

src/main/java/dev/wms/pwrapi/utils/config/CachingConfig.java

@@ -0,0 +1,35 @@
+package dev.wms.pwrapi.utils.config;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+
+@Configuration
+@EnableCaching
+@EnableScheduling
+@Slf4j
+public class CachingConfig implements CacheManagerCustomizer<ConcurrentMapCacheManager> {
+
+    @Override
+    public void customize(ConcurrentMapCacheManager cacheManager) {
+        cacheManager.setCacheNames(List.of("pwr-news"));
+    }
+
+    @CacheEvict(allEntries = true, cacheNames = "pwr-news")
+    @Scheduled(fixedDelayString = "${pwr-api.news.cacheTTL}")
+    public void reportCacheEvict(){
+    }
+
+}

src/main/java/dev/wms/pwrapi/utils/config/DeserializationConfig.java

@@ -0,0 +1,29 @@
+package dev.wms.pwrapi.utils.config;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.MediaType;
+import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
+import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+@Configuration
+public class DeserializationConfig {
+
+    @Bean
+    Jackson2ObjectMapperBuilder getDefaultDeserializer(){
+        return Jackson2ObjectMapperBuilder.json();
+    }
+
+    @Bean
+    public WebMvcConfigurer customWebMvcConfigurer(){
+        return new WebMvcConfigurer() {
+            @Override
+            public void configureContentNegotiation(ContentNegotiationConfigurer configurer) {
+                configurer.defaultContentType(MediaType.APPLICATION_JSON);
+            }
+        };
+    }
+
+}

src/main/java/dev/wms/pwrapi/utils/generalExceptionHandling/GeneralAdvice.java

@@ -32,6 +32,7 @@ public class GeneralAdvice {
     @Order(Ordered.LOWEST_PRECEDENCE)
     public ResponseEntity<ExceptionMessagingDTO> handleGeneralException(Throwable t){
         log.info("Handling unexpected exception " + t);
+        t.printStackTrace();
         return ResponseEntity.status(500).body(new ExceptionMessagingDTO(t.getMessage()));
     }
 

src/main/java/dev/wms/pwrapi/utils/http/HttpUtils.java

@@ -20,14 +20,27 @@ public static String makeRequestWithClientAndGetString(OkHttpClient client, Requ
         }
     }
 
+    public static String makeRequestWithClientAndGetString(OkHttpClient client, String url){
+
+        Request request = new Request.Builder()
+                .url(url)
+                .build();
+
+        try(Response response = client.newCall(request).execute()){
+            return response.body().string();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Makes request with OkHttp's client and parses it to Jsoup's Document. Needed for proper response closing
      * @param client OkHttp client which will execute the request
      * @param url URL that will be requested
      * @return Jsoup's Document containing parsed html from OkHttp response
      * @throws IOException when parsing goes wrong
      */
-    public static Document makeRequestWithClientAndGetDocument(OkHttpClient client, String url) throws IOException {
+    public static Document makeRequestWithClientAndGetDocument(OkHttpClient client, String url)  {
 
         Request financeRequest = new Request.Builder()
                 .url(url)
@@ -39,6 +52,8 @@ public static Document makeRequestWithClientAndGetDocument(OkHttpClient client,
             responseString = response.body().string();
         } catch (SocketTimeoutException e){
             throw new SystemTimeoutException();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
 
         return Jsoup.parse(responseString);