core: redo search cache from scratch. resolves Jackett#10382 (Jackett…

…#10447)

In simple words, when you make a request in Jackett, the results are saved in memory (cache). The next request will return results form the cache improving response time and making fewer requests to the sites.
* We assume all indexers/sites are stateless, the same request return the same response. If you change the search term, categories or something in the query Jackett has to make a live request to the indexer.
* There are some situations when we don't want to use the cache:
** When we are testing the indexers => if query.IsTest results are not cached
** When the user updates the configuration of one indexer => We call CleanIndexerCache to remove cached results before testing the configuration
** When there is some error/exception in the indexer => The results are not cached so we can retry in the next request
* We want to limit the memory usage, so we try to remove elements from cache ASAP:
** Each indexer can have a maximum number of results in memory. If the limit is exceeded we remove old results
** Cached results expire after some time
* Users can configure the cache or even disable it

README.md

@@ -676,7 +676,7 @@ Jackett is available as a beta package from [SynoCommunity](https://synocommunit
 
 
 ## Running Jackett behind a reverse proxy
-When running jackett behind a reverse proxy make sure that the original hostname of the request is passed to Jackett. If HTTPS is used also set the X-Forwarded-Proto header to "https". Don't forget to adjust the "Base Path Override" Jackett option accordingly.
+When running jackett behind a reverse proxy make sure that the original hostname of the request is passed to Jackett. If HTTPS is used also set the X-Forwarded-Proto header to "https". Don't forget to adjust the "Base path override" Jackett option accordingly.
 
 Example config for apache:
 ```

src/Jackett.Common/Content/custom.css

@@ -134,8 +134,8 @@ hr {
     }
 
 .input-header {
-    font-size: 18px;
-    width: 160px;
+    font-size: 16px;
+    width: 250px;
     display: inline-block;
 }
 
@@ -165,7 +165,7 @@ hr {
     text-align: center;
 }
 
-#jackett-allowext, #jackett-allowupdate, #jackett-logging, #jackett-prerelease {
+#jackett-allowext, #jackett-allowupdate, #jackett-logging, #jackett-prerelease, #jackett-cache-enabled {
     width: 25px;
 }
 

src/Jackett.Common/Content/custom.js

@@ -98,6 +98,14 @@ function loadJackettSettings() {
         $("#jackett-allowupdate").attr('checked', data.updatedisabled);
         $("#jackett-prerelease").attr('checked', data.prerelease);
         $("#jackett-logging").attr('checked', data.logging);
+
+        $("#jackett-cache-enabled").attr('checked', data.cache_enabled);
+        $("#jackett-cache-ttl").val(data.cache_ttl);
+        $("#jackett-cache-max-results-per-indexer").val(data.cache_max_results_per_indexer);
+        if (!data.cache_enabled) {
+            $("#jackett-show-releases").attr("disabled", true);
+        }
+
         $("#jackett-omdbkey").val(data.omdbkey);
         $("#jackett-omdburl").val(data.omdburl);
         var password = data.password;
@@ -1126,6 +1134,9 @@ function bindUIButtons() {
         var jackett_update = $("#jackett-allowupdate").is(':checked');
         var jackett_prerelease = $("#jackett-prerelease").is(':checked');
         var jackett_logging = $("#jackett-logging").is(':checked');
+        var jackett_cache_enabled = $("#jackett-cache-enabled").is(':checked');
+        var jackett_cache_ttl = $("#jackett-cache-ttl").val();
+        var jackett_cache_max_results_per_indexer = $("#jackett-cache-max-results-per-indexer").val();
         var jackett_omdb_key = $("#jackett-omdbkey").val();
         var jackett_omdb_url = $("#jackett-omdburl").val();
 
@@ -1143,6 +1154,10 @@ function bindUIButtons() {
             blackholedir: $("#jackett-savedir").val(),
             logging: jackett_logging,
             basepathoverride: jackett_basepathoverride,
+            logging: jackett_logging,
+            cache_enabled: jackett_cache_enabled,
+            cache_ttl: jackett_cache_ttl,
+            cache_max_results_per_indexer: jackett_cache_max_results_per_indexer,
             omdbkey: jackett_omdb_key,
             omdburl: jackett_omdb_url,
             proxy_type: jackett_proxy_type,

src/Jackett.Common/Content/custom_mobile.css

@@ -126,8 +126,8 @@ hr {
     }
 
 .input-header {
-    font-size: 18px;
-    width: 160px;
+    font-size: 16px;
+    width: 250px;
     display: inline-block;
 }
 
@@ -157,7 +157,7 @@ hr {
     text-align: center;
 }
 
-#jackett-allowext, #jackett-allowupdate, #jackett-logging, #jackett-prerelease {
+#jackett-allowext, #jackett-allowupdate, #jackett-logging, #jackett-prerelease, #jackett-cache-enabled {
     width: 25px;
 }
 

src/Jackett.Common/Content/index.html

@@ -33,8 +33,8 @@
 
     <link rel="stylesheet" type="text/css" href="../bootstrap/bootstrap.min.css?changed=2017083001">
     <link rel="stylesheet" type="text/css" href="../animate.css?changed=2017083001">
-    <link rel="stylesheet" type="text/css" href="../custom.css?changed=20200223" media="only screen and (min-device-width: 480px)">
-    <link rel="stylesheet" type="text/css" href="../custom_mobile.css?changed=20200223" media="only screen and (max-device-width: 480px)">
+    <link rel="stylesheet" type="text/css" href="../custom.css?changed=20201208" media="only screen and (min-device-width: 480px)">
+    <link rel="stylesheet" type="text/css" href="../custom_mobile.css?changed=20201208" media="only screen and (max-device-width: 480px)">
     <link rel="stylesheet" type="text/css" href="../css/jquery.dataTables.min.css?changed=2017083001">
     <link rel="stylesheet" type="text/css" href="../css/bootstrap-multiselect.css?changed=2017083001" />
     <link rel="stylesheet" type="text/css" href="../css/font-awesome.min.css?changed=2017083001">
@@ -127,15 +127,15 @@ <h3>Jackett Configuration</h3>
             </a>
         </div>
         <div class="input-area">
-            <span class="input-header">Base Path Override: </span>
+            <span class="input-header">Base path override: </span>
             <input id="jackett-basepathoverride" class="form-control input-right" type="text" value="" placeholder="/jackett">
         </div>
         <div class="input-area">
             <span class="input-header">Server port: </span>
             <input id="jackett-port" class="form-control input-right" type="text" value="" placeholder="9117">
         </div>
         <div class="input-area">
-            <span class="input-header">Manual download blackhole directory: </span>
+            <span class="input-header">Blackhole directory: </span>
             <input id="jackett-savedir" class="form-control input-right" type="text" value="" placeholder="c:\torrents\">
         </div>
 
@@ -188,6 +188,18 @@ <h3>Jackett Configuration</h3>
             <span class="input-header">Enhanced logging: </span>
             <input id="jackett-logging" class="form-control input-right" type="checkbox" />
         </div>
+        <div class="input-area">
+            <span class="input-header">Cache enabled (recommended): </span>
+            <input id="jackett-cache-enabled" class="form-control input-right" type="checkbox" />
+        </div>
+        <div class="input-area">
+            <span class="input-header">Cache TTL (seconds): </span>
+            <input id="jackett-cache-ttl" class="form-control input-right" type="text" value="" placeholder="">
+        </div>
+        <div class="input-area">
+            <span class="input-header">Cache max results per indexer: </span>
+            <input id="jackett-cache-max-results-per-indexer" class="form-control input-right" type="text" value="" placeholder="">
+        </div>
         <div class="input-area">
             <span class="input-header">OMDB API key: </span>
             <input id="jackett-omdbkey" class="form-control input-right" type="text" value="" placeholder="">
@@ -680,6 +692,6 @@ <h4>Capabilities</h4>
     </script>
 
     <script type="text/javascript" src="../libs/api.js?changed=2017083001"></script>
-    <script type="text/javascript" src="../custom.js?changed=20201107"></script>
+    <script type="text/javascript" src="../custom.js?changed=20201208"></script>
 </body>
 </html>

src/Jackett.Common/Indexers/Abnormal.cs

@@ -49,7 +49,8 @@ private ConfigurationDataAbnormal ConfigData
             set => configData = value;
         }
 
-        public Abnormal(IIndexerConfigurationService configService, WebClient w, Logger l, IProtectionService ps)
+        public Abnormal(IIndexerConfigurationService configService, WebClient w, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "abnormal",
                    name: "Abnormal",
                    description: "General French Private Tracker",
@@ -68,6 +69,7 @@ public Abnormal(IIndexerConfigurationService configService, WebClient w, Logger
                    client: w,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    downloadBase: "https://abnormal.ws/torrents.php?action=download&id=",
                    configData: new ConfigurationDataAbnormal())
         {

src/Jackett.Common/Indexers/Abstract/AvistazTracker.cs

@@ -106,7 +106,7 @@ var res when _hdResolutions.Contains(res) => TorznabCatType.TVHD.ID,
 
         protected AvistazTracker(string link, string id, string name, string description,
                                  IIndexerConfigurationService configService, WebClient client, Logger logger,
-                                 IProtectionService p, TorznabCapabilities caps)
+                                 IProtectionService p, ICacheService cs, TorznabCapabilities caps)
             : base(id: id,
                    name: name,
                    description: description,
@@ -116,6 +116,7 @@ protected AvistazTracker(string link, string id, string name, string description
                    client: client,
                    logger: logger,
                    p: p,
+                   cacheService: cs,
                    configData: new ConfigurationDataBasicLoginWithPID(@"You have to check 'Enable RSS Feed' in 'My Account',
 without this configuration the torrent download does not work.<br/>You can find the PID in 'My profile'."))
         {

src/Jackett.Common/Indexers/Abstract/CouchPotatoTracker.cs

@@ -27,8 +27,8 @@ public abstract class CouchPotatoTracker : BaseWebIndexer
 
         protected CouchPotatoTracker(string link, string id, string name, string description,
                                      IIndexerConfigurationService configService, WebClient client, Logger logger,
-                                     IProtectionService p, TorznabCapabilities caps, ConfigurationData configData,
-                                     string endpoint)
+                                     IProtectionService p, ICacheService cs, TorznabCapabilities caps,
+                                     ConfigurationData configData, string endpoint)
             : base(id: id,
                    name: name,
                    description: description,
@@ -38,6 +38,7 @@ protected CouchPotatoTracker(string link, string id, string name, string descrip
                    client: client,
                    logger: logger,
                    p: p,
+                   cacheService: cs,
                    configData: configData)
         {
             this.endpoint = endpoint;

src/Jackett.Common/Indexers/Abstract/GazelleTracker.cs

@@ -37,9 +37,9 @@ public abstract class GazelleTracker : BaseWebIndexer
 
         protected GazelleTracker(string link, string id, string name, string description,
                                  IIndexerConfigurationService configService, WebClient client, Logger logger,
-                                 IProtectionService p, TorznabCapabilities caps, bool supportsFreeleechTokens,
-                                 bool imdbInTags = false, bool has2Fa = false, bool useApiKey = false,
-                                 string instructionMessageOptional = null)
+                                 IProtectionService p, ICacheService cs, TorznabCapabilities caps,
+                                 bool supportsFreeleechTokens, bool imdbInTags = false, bool has2Fa = false,
+                                 bool useApiKey = false, string instructionMessageOptional = null)
             : base(id: id,
                    name: name,
                    description: description,
@@ -49,6 +49,7 @@ protected GazelleTracker(string link, string id, string name, string description
                    client: client,
                    logger: logger,
                    p: p,
+                   cacheService: cs,
                    configData: new ConfigurationDataGazelleTracker(
                        has2Fa, supportsFreeleechTokens, useApiKey, instructionMessageOptional))
         {

src/Jackett.Common/Indexers/AlphaRatio.cs

@@ -12,7 +12,8 @@ namespace Jackett.Common.Indexers
     [ExcludeFromCodeCoverage]
     public class AlphaRatio : GazelleTracker
     {
-        public AlphaRatio(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
+        public AlphaRatio(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "alpharatio",
                    name: "AlphaRatio",
                    description: "AlphaRatio (AR) is a Private Torrent Tracker for 0DAY / GENERAL",
@@ -32,6 +33,7 @@ public AlphaRatio(IIndexerConfigurationService configService, WebClient wc, Logg
                    client: wc,
                    logger: l,
                    p: ps,
+                   cs: cs,
                    supportsFreeleechTokens: true,
                    imdbInTags: true)
         {

src/Jackett.Common/Indexers/AniDUB.cs

@@ -27,7 +27,8 @@ internal class AniDUB : BaseWebIndexer
         private static readonly Regex SeasonInfoRegex = new Regex(@"(?:(?:TV-)|(?:ТВ-))(\d+)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
         private static readonly Lazy<Regex> StripRussianTitleRegex = new Lazy<Regex>(() => new Regex(@"^.*?\/\s*", RegexOptions.Compiled));
 
-        public AniDUB(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
+        public AniDUB(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "anidub",
                    name: "AniDUB",
                    description: "AniDUB Tracker is a semi-private russian tracker and release group for anime",
@@ -51,6 +52,7 @@ public AniDUB(IIndexerConfigurationService configService, WebClient wc, Logger l
                    client: wc,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationDataAniDub())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/AniLibria.cs

@@ -18,7 +18,7 @@ namespace Jackett.Common.Indexers
     [ExcludeFromCodeCoverage]
     public class AniLibria : BaseWebIndexer
     {
-        public AniLibria(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l, IProtectionService ps)
+        public AniLibria(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l, IProtectionService ps, ICacheService cs)
             : base(id: "AniLibria",
                    name: "AniLibria",
                    description: "AniLibria is a Public torrent tracker for anime, voiced on russian by AniLibria team",
@@ -34,6 +34,7 @@ public AniLibria(IIndexerConfigurationService configService, Utils.Clients.WebCl
                    client: wc,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationDataAniLibria())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/Anidex.cs

@@ -23,7 +23,8 @@ namespace Jackett.Common.Indexers
     [ExcludeFromCodeCoverage]
     public class Anidex : BaseWebIndexer
     {
-        public Anidex(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l, IProtectionService ps)
+        public Anidex(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l,
+            IProtectionService ps, ICacheService cs)
             : base(id: "anidex",
                    name: "Anidex",
                    description: "Anidex is a Public torrent tracker and indexer, primarily for English fansub groups of anime",
@@ -39,6 +40,7 @@ public Anidex(IIndexerConfigurationService configService, Utils.Clients.WebClien
                    client: wc,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationData())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/AnimeBytes.cs

@@ -32,7 +32,8 @@ public class AnimeBytes : BaseCachingWebIndexer
 
         private ConfigurationDataAnimeBytes ConfigData => (ConfigurationDataAnimeBytes)configData;
 
-        public AnimeBytes(IIndexerConfigurationService configService, WebClient client, Logger l, IProtectionService ps)
+        public AnimeBytes(IIndexerConfigurationService configService, WebClient client, Logger l,
+            IProtectionService ps, ICacheService cs)
             : base(id: "animebytes",
                    name: "AnimeBytes",
                    description: "Powered by Tentacles",
@@ -59,6 +60,7 @@ public AnimeBytes(IIndexerConfigurationService configService, WebClient client,
                    },
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationDataAnimeBytes("Note: Go to AnimeBytes site and open your account settings. Go to 'Account' tab, move cursor over black part near 'Passkey' and copy its value. Your username is case sensitive."))
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/AnimeTorrents.cs

@@ -32,7 +32,8 @@ public class AnimeTorrents : BaseWebIndexer
             set => base.configData = value;
         }
 
-        public AnimeTorrents(IIndexerConfigurationService configService, WebClient c, Logger l, IProtectionService ps)
+        public AnimeTorrents(IIndexerConfigurationService configService, WebClient c, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "animetorrents",
                    name: "AnimeTorrents",
                    description: "Definitive source for anime and manga",
@@ -51,6 +52,7 @@ public AnimeTorrents(IIndexerConfigurationService configService, WebClient c, Lo
                    client: c,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationDataBasicLogin())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/Animedia.cs

@@ -27,7 +27,8 @@ internal class Animedia : BaseWebIndexer
         private static readonly Regex SizeInfoQueryRegex = new Regex(@"Размер:(.*)\n", RegexOptions.Compiled | RegexOptions.IgnoreCase);
         private static readonly Regex ReleaseDateInfoQueryRegex = new Regex(@"Добавлен:(.*)\n", RegexOptions.Compiled | RegexOptions.IgnoreCase);
 
-        public Animedia(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
+        public Animedia(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "Animedia",
                    name: "Animedia",
                    description: "Animedia is a public russian tracker and release group for anime.",
@@ -43,6 +44,7 @@ public Animedia(IIndexerConfigurationService configService, WebClient wc, Logger
                    client: wc,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationData())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/Anthelion.cs

@@ -30,7 +30,8 @@ public class Anthelion : BaseWebIndexer
             "https://tehconnection.me/"
         };
 
-        public Anthelion(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
+        public Anthelion(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "anthelion",
                    name: "Anthelion", // old name: TehConnection.me
                    description: "A movies tracker",
@@ -49,6 +50,7 @@ public Anthelion(IIndexerConfigurationService configService, WebClient wc, Logge
                    client: wc,
                    logger: l,
                    p: ps,
+                   cacheService: cs,
                    configData: new ConfigurationDataBasicLogin())
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/AvistaZ.cs

@@ -11,7 +11,8 @@ namespace Jackett.Common.Indexers
     [ExcludeFromCodeCoverage]
     public class AvistaZ : AvistazTracker
     {
-        public AvistaZ(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
+        public AvistaZ(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
             : base(id: "avistaz",
                    name: "AvistaZ",
                    description: "Aka AsiaTorrents",
@@ -30,7 +31,9 @@ public AvistaZ(IIndexerConfigurationService configService, WebClient wc, Logger
                    configService: configService,
                    client: wc,
                    logger: l,
-                   p: ps)
+                   p: ps,
+                   cs: cs
+                   )
         {
             AddCategoryMapping(1, TorznabCatType.Movies);
             AddCategoryMapping(1, TorznabCatType.MoviesUHD);