Merge pull request #940 from mediathekview/feature/configfilename

update configurationmanager to support configuration file name

MServer-Config.yaml

@@ -22,6 +22,7 @@ senderIncluded:
   #- KIKA
   - DW
   #- BR
+  #- PHOENIX
 
 # If set the server will be awake after the crawler run and restarts the run after the given amount.
 #schedules:
@@ -52,7 +53,7 @@ filmlistSaveFormats:
 
 # The paths where which filmlist should be safed to.
 filmlistSavePaths:
-#  JSON: target/filmlists/filmliste.json
+  JSON: target/filmlists/filmliste.json
   OLD_JSON: target/filmlists/filmliste_old.json
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old.json.xz
@@ -64,8 +65,8 @@ filmlistSavePaths:
 # The paths where which diff film list should be safed to.
 # If not set no difference film lists will be safed.
 filmlistDiffSavePaths:
-#  JSON: target/filmlists/filmliste_diff.json
-#  OLD_JSON: target/filmlists/filmliste_old_diff.json
+  JSON: target/filmlists/filmliste_diff.json
+  OLD_JSON: target/filmlists/filmliste_old_diff.json
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste_diff.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old_diff.json.xz
 #  JSON_COMPRESSED_GZIP: target/filmlists/filmliste_diff.json.gz
@@ -76,12 +77,12 @@ filmlistDiffSavePaths:
 #Sets if a filmlist hash file should be written
 writeFilmlistHashFileEnabled: true
 #The filmlist hash file path
-filmlistHashFilePath: target/filmlists/filmlist.hash
+filmlistHashFilePath: target/filmlists/filmlist.hash.xx
 
 #Sets if a filmlist id file should be written
 writeFilmlistIdFileEnabled: true
 #The fimlist id file path
-filmlistIdFilePath: target/filmlists/filmlist.id
+filmlistIdFilePath: target/filmlists/filmlist.id.xx
 
 
 # import additional filmlist sources
@@ -181,7 +182,7 @@ copySettings:
   # En- / disables FTP
   copyEnabled: false
 
-  # The paths where to safe the film list files.SrfTopicOverviewTask
+  # The paths where to safe the film list files.
   # WARNING: You can only set the path for film list formats you listed in "filmlistSaveFormats".
   # Required if enabled
   copyTargetFilePaths:

src/main/java/de/mediathekview/mserver/base/config/Log4JConfigurationFactory.java

@@ -45,9 +45,19 @@ public class Log4JConfigurationFactory extends ConfigurationFactory {
 
   private static MServerLogSettingsDTO logSettings;
 
+  public Log4JConfigurationFactory(MServerLogSettingsDTO logSettings) {
+    Log4JConfigurationFactory.logSettings = logSettings;
+  }
+
+
+  public Log4JConfigurationFactory() {
+    if (Log4JConfigurationFactory.logSettings == null) {
+      Log4JConfigurationFactory.logSettings = new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE).getConfig().getLogSettings();
+    }
+  }
+
   static Configuration createConfiguration(
       final String name, final ConfigurationBuilder<BuiltConfiguration> aBuilder) {
-    logSettings = new MServerConfigManager().getConfig().getLogSettings();
 
     aBuilder.setConfigurationName(name);
 

src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java

@@ -15,7 +15,7 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private final Boolean writeFilmlistIdFileEnabled;
   private final String filmlistIdFilePath;
   /** ignore certain film by title **/
-  private final String ignoreFilmlistPath;
+  private String ignoreFilmlistPath;
   /** add livestreams from external list **/
   private final ImportLivestreamConfiguration importLivestreamConfiguration;
   /** add additional filmlist from external **/
@@ -147,6 +147,10 @@ public Map<FilmlistFormats, String> getFilmlistSavePaths() {
   public void setFilmlistSavePaths(final Map<FilmlistFormats, String> filmlistSavePaths) {
     this.filmlistSavePaths = filmlistSavePaths;
   }
+
+  public void setIgnoreFilmlistPath(final String ignoreFilmlistPath) {
+    this.ignoreFilmlistPath = ignoreFilmlistPath;
+  }
 
   public MServerLogSettingsDTO getLogSettings() {
     return logSettings;

src/main/java/de/mediathekview/mserver/base/config/MServerConfigManager.java

@@ -14,10 +14,6 @@ public MServerConfigManager(final String fileName) {
     configFileName = fileName;
   }
 
-  public MServerConfigManager() {
-    this(DEFAULT_CONFIG_FILE);
-  }
-
   /**
    * @param aSender The {@link Sender} for which the config will be loaded.
    * @return The Sender specific config.

src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java

@@ -1,6 +1,8 @@
 package de.mediathekview.mserver.base.utils;
 
 
+import java.util.concurrent.ForkJoinPool;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -14,16 +16,19 @@
 
 public class CheckUrlAvailability {
   private static final Logger LOG = LogManager.getLogger(CheckUrlAvailability.class);
+  private final FileSizeDeterminer fsd;
+  private int numberOfThreads = 10;
   private Long minFileSize = 2048L;
   private int removedCounter = 0;
   private long timeoutInMS = 1*60*1000L;
   private boolean timeout = false;
   private long start = 0;
-  private FileSizeDeterminer fsd = new FileSizeDeterminer();
-
-  public CheckUrlAvailability(long minFileSize, long timeoutInSec) {
+
+  public CheckUrlAvailability(final long minFileSize, final long timeoutInSec, final int numberOfThreads) {
     this.minFileSize = minFileSize;
     this.timeoutInMS = timeoutInSec*1000;
+    this.numberOfThreads = numberOfThreads;
+    fsd = new FileSizeDeterminer(30L, 30L, numberOfThreads);
   }
 
   public Filmlist getAvaiableFilmlist(final Filmlist importList) {
@@ -32,7 +37,14 @@ public Filmlist getAvaiableFilmlist(final Filmlist importList) {
     Filmlist filteredFilmlist = new Filmlist();
     filteredFilmlist.setCreationDate(importList.getCreationDate());
     filteredFilmlist.setListId(importList.getListId());
-    importList.getFilms().values().stream().parallel().filter(this::isAvailable).forEach(filteredFilmlist::add);
+    //
+    ForkJoinPool customThreadPool = new ForkJoinPool(numberOfThreads);
+    customThreadPool.submit(() -> importList.getFilms().values().parallelStream()
+            .filter(this::isAvailable)
+            .forEach(filteredFilmlist::add))
+            .join();
+    customThreadPool.shutdown();
+    //
     LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter, ((System.currentTimeMillis()-start)/1000), timeout);
     return filteredFilmlist;
   }
@@ -49,7 +61,7 @@ private boolean isAvailable(Film pFilm) {
     if (pFilm.getThema().equalsIgnoreCase("Livestream")) {
       // do not remove livestreams
       return true;
-    } else if (ri.getCode() == 404) {
+    } else if (!(ri.getCode() >= 200 && ri.getCode() < 300)) {
       LOG.debug("Film response ({}): {} # {} # {} # {} ", ri.getCode(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
       removedCounter++;
       return false;
@@ -66,11 +78,6 @@ private boolean isAvailable(Film pFilm) {
       removedCounter++;
       return false;
     }
-    // just for debugging
-    if (ri.getCode() != 200) {
-      LOG.debug("Film not removed but status!=200 ({}): {} # {} # {} # {} ", ri.getCode(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-    }
-
     return true;
   }
 

src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java

@@ -57,8 +57,8 @@ public class CrawlerManager extends AbstractManager {
   private static final String FILMLIST_JSON_COMPRESSED_DEFAULT_NAME =
       FILMLIST_JSON_DEFAULT_NAME + ".xz";
   private static final Logger LOG = LogManager.getLogger(CrawlerManager.class);
-  private static CrawlerManager instance;
   private final MServerConfigDTO config;
+  private final MServerConfigManager rootConfig;
   private final ForkJoinPool forkJoinPool;
   private final Filmlist filmlist;
   private final IgnoreFilmFilter ingoreFilmFilter;
@@ -70,9 +70,9 @@ public class CrawlerManager extends AbstractManager {
   private final Collection<ProgressListener> copyProgressListeners;
   private final Filmlist differenceList;
 
-  private CrawlerManager() {
+  public CrawlerManager(MServerConfigManager aMServerConfigManager) {
     super();
-    final MServerConfigManager rootConfig = new MServerConfigManager();
+    rootConfig = aMServerConfigManager;
     config = rootConfig.getConfig();
     ingoreFilmFilter = new IgnoreFilmFilter(config.getIgnoreFilmslistPath());
     executorService = Executors.newFixedThreadPool(config.getMaximumCpuThreads());
@@ -86,11 +86,8 @@ private CrawlerManager() {
     initializeCrawler(rootConfig);
   }
 
-  public static CrawlerManager getInstance() {
-    if (instance == null) {
-      instance = new CrawlerManager();
-    }
-    return instance;
+  public MServerConfigManager getConfigManager() {
+    return rootConfig;
   }
 
   public void copyFilmlist() {
@@ -166,6 +163,7 @@ public void importLivestreamFilmlist() {
   }
 
   public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) {
+    LOG.debug("importLivestreamFilmlist {}", aFilmlistLocation);
     try {
       final Optional<Filmlist> importedFilmlist;
       if (aFilmlistLocation.startsWith(HTTP)) {
@@ -194,7 +192,13 @@ public void importFilmlist(final ImportFilmlistConfiguration importFilmlistConfi
       }
       //
       if (importFilmlistConfiguration.isCheckImportListUrl() && importedFilmlist.isPresent() ) {
-        importedFilmlist = Optional.of(new CheckUrlAvailability(config.getCheckImportListUrlMinSize() ,config.getCheckImportListUrlTimeoutInSec()).getAvaiableFilmlist(importedFilmlist.get()));
+        importedFilmlist = Optional.of(
+            new CheckUrlAvailability(
+                config.getCheckImportListUrlMinSize(),
+                config.getCheckImportListUrlTimeoutInSec(),
+                config.getMaximumCpuThreads())
+            .getAvaiableFilmlist(importedFilmlist.get())
+        );
       }
       //
       final Filmlist difflist = new Filmlist(UUID.randomUUID(), LocalDateTime.now());

src/main/java/de/mediathekview/mserver/crawler/basic/IgnoreFilmFilter.java

@@ -31,7 +31,7 @@ public IgnoreFilmFilter(String configFileNameAndPath) {
       } else {
         ignoreFilmTitles = read(configFileNameAndPath);
       }
-      LOG.debug("ignoreFilmList setup with {} entries", size());
+      LOG.debug("ignoreFilmList {} setup with {} entries", configFileNameAndPath, size());
     } catch (IOException e) {
       LOG.error("Could not read ignorefilmlist from {} ",configFileNameAndPath, e);
     }

src/main/java/de/mediathekview/mserver/crawler/dw/DWTaskBase.java

@@ -5,7 +5,6 @@
 import com.google.gson.GsonBuilder;
 
 import de.mediathekview.mlib.daten.Sender;
-import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
 import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -25,9 +24,7 @@
 public abstract class DWTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
   private static final Logger LOG = LogManager.getLogger(DWTaskBase.class);
 
-  private static final RateLimiter limiter =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
+  private static RateLimiter limiter = null;
 
   private final transient GsonBuilder gsonBuilder;
 
@@ -81,6 +78,9 @@ private Response executeRequest(final WebTarget aTarget) {
           request.header(
               ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
     }
+    if (limiter == null) {
+      limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
+    }
     limiter.acquire();
     return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
   }

src/main/java/de/mediathekview/mserver/crawler/funk/json/FunkVideoDeserializer.java

@@ -73,9 +73,7 @@ protected FilmInfoDto mapToElement(final JsonObject jsonObject) {
   }
 
   private MServerConfigDTO getRuntimeConfig() {
-    return crawler
-        .map(AbstractCrawler::getRuntimeConfig)
-        .orElseGet(() -> new MServerConfigManager().getConfig());
+    return crawler.get().getRuntimeConfig();
   }
 
   private String createNexxCloudUrl(final String entityId) {

src/main/java/de/mediathekview/mserver/crawler/sr/tasks/SrFilmDetailTask.java

@@ -18,7 +18,6 @@
 import org.jsoup.nodes.Node;
 import org.jsoup.select.Elements;
 
-import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;

src/main/java/de/mediathekview/mserver/crawler/sr/tasks/SrRateLimitedDocumentTask.java

@@ -14,9 +14,7 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>
 
   private static final long serialVersionUID = -4077182368484515410L;
 
-  private static final RateLimiter LIMITER =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
+  private static RateLimiter LIMITER = null;
 
   SrRateLimitedDocumentTask(
       final AbstractCrawler crawler,
@@ -26,6 +24,9 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>
 
   @Override
   protected void processElement(final D urlDTO) {
+    if (LIMITER== null) {
+      LIMITER = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
+    }
     LIMITER.acquire();
     super.processElement(urlDTO);
   }

src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfTaskBase.java

@@ -4,7 +4,6 @@
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import de.mediathekview.mlib.daten.Sender;
-import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
 import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -22,9 +21,7 @@
 public abstract class ZdfTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
   private static final Logger LOG = LogManager.getLogger(ZdfTaskBase.class);
 
-  private static final RateLimiter limiter =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
+  private static RateLimiter limiter = null;
 
   private final GsonBuilder gsonBuilder;
 
@@ -76,6 +73,10 @@ private Response executeRequest(final WebTarget aTarget) {
           request.header(
               ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
     }
+    if (limiter == null) {
+      limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
+    }
+
     limiter.acquire();
     return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
   }