Merge branch 'master' into feature/cloudproxy

src/Jackett.Common/Definitions/frozenlayer.yml

@@ -73,7 +73,7 @@ search:
         - name: re_replace
           args: ["- Episodio ([0-9]*)", "$1"]
         - name: append
-          args: " [spanish]"
+          args: " Spanish"
     details:
       selector: td.tit a
       attribute: href

src/Jackett.Common/Definitions/hdcity.yml

@@ -172,7 +172,7 @@ search:
       optional: true
       filters:
         - name: append
-          args: " [English]"
+          args: " English"
         - name: re_replace
           args: ["(?i)T[\\s-_]?(\\d{1,2})\\b", " S$1 "]
         - name: re_replace
@@ -184,7 +184,7 @@ search:
       optional: true
       filters:
         - name: append
-          args: " [Spanish]"
+          args: " Spanish"
         - name: re_replace
           args: ["(?i)T[\\s-_]?(\\d{1,2})\\b", " S$1 "]
         - name: re_replace

src/Jackett.Common/Definitions/hdspain.yml

@@ -83,7 +83,7 @@ search:
         - name: prepend
           args: "{{ .Result.extras }} "
         - name: append
-          args: " [Spanish]"
+          args: " Spanish"
         - name: re_replace
           args: ["(?i)T(\\d{1,2})\\b", "S$1"]
     details:

src/Jackett.Common/Definitions/nyaasi.yml

@@ -142,7 +142,7 @@ search:
       optional: true
       filters:
         - name: append
-          args: " [Spanish]"
+          args: " Spanish"
     details:
       selector: td:nth-child(2) a:last-of-type
       attribute: href

src/Jackett.Common/Indexers/BitCityReloaded.cs

@@ -58,15 +58,12 @@ public BitCityReloaded(IIndexerConfigurationService configService, WebClient wc,
                    client: wc,
                    logger: l,
                    p: ps,
-                   configData: new ConfigurationDataBasicLoginWithRSSAndDisplay())
+                   configData: new ConfigurationDataBasicLoginWithRSSAndDisplay("Only the results from the first search result page are shown, adjust your profile settings to show a reasonable amount (it looks like there's no maximum)."))
         {
             Encoding = Encoding.GetEncoding("iso-8859-1");
             Language = "de-de";
             Type = "private";
 
-            configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show a reasonable amount (it looks like there's no maximum).";
-            configData.DisplayText.Name = "Notice";
-
             AddCategoryMapping(1, TorznabCatType.Other, "Other/Anderes");
             AddCategoryMapping(2, TorznabCatType.TVAnime, "TV/Anime");
             AddCategoryMapping(34, TorznabCatType.PC, "Appz/Linux");

src/Jackett.Common/Indexers/Cinecalidad.cs

@@ -128,10 +128,16 @@ public override async Task<byte[]> Download(Uri link)
             {
                 var parser = new HtmlParser();
                 var dom = parser.ParseDocument(results.ContentString);
-                var preotectedLink = dom.QuerySelector("a[service=BitTorrent]").GetAttribute("href");
-                preotectedLink = SiteLink + preotectedLink.TrimStart('/');
+                var protectedLink = dom.QuerySelector("a[service=BitTorrent]").GetAttribute("href");
+                if (protectedLink.Contains("/ouo.io/"))
+                {
+                    // protected link =>
+                    // https://ouo.io/qs/qsW6rCh4?s=https://www.cinecalidad.is/protect/v2.php?i=A8--9InL&title=High+Life+%282018%29
+                    var linkParts = protectedLink.Split('=');
+                    protectedLink = protectedLink.Replace(linkParts[0] + "=", "");
+                }
 
-                results = await RequestWithCookiesAsync(preotectedLink);
+                results = await RequestWithCookiesAsync(protectedLink);
                 dom = parser.ParseDocument(results.ContentString);
                 var magnetUrl = dom.QuerySelector("a[href^=magnet]").GetAttribute("href");
                 return await base.Download(new Uri(magnetUrl));

src/Jackett.Common/Indexers/Magnetico.cs

@@ -51,7 +51,9 @@ public Magnetico(IIndexerConfigurationService configService, WebClient wc, Logge
                    client: wc,
                    logger: l,
                    p: ps,
-                   configData: new ConfigurationDataBasicLogin("Configure the URL, username and password from your local magneticow.<br>Default credentials are: username=username, password=password."))
+                   configData: new ConfigurationDataBasicLogin("Configure the URL, username and password from your local magneticow.<br>" +
+                               "Default credentials are: username=username, password=password.<br>" +
+                               "If you have many torrents, it is recommended to use PostgreSQL database to make queries faster. With SQLite, timeouts may occur."))
 
         {
             Encoding = Encoding.UTF8;

src/Jackett.Common/Indexers/PirateTheNet.cs

@@ -47,15 +47,12 @@ public PirateTheNet(IIndexerConfigurationService configService, WebClient w, Log
                    client: w,
                    logger: l,
                    p: ps,
-                   configData: new ConfigurationDataBasicLoginWithRSSAndDisplay())
+                   configData: new ConfigurationDataBasicLoginWithRSSAndDisplay("Only the results from the first search result page are shown, adjust your profile settings to show the maximum."))
         {
             Encoding = Encoding.UTF8;
             Language = "en-us";
             Type = "private";
 
-            configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show the maximum.";
-            configData.DisplayText.Name = "Notice";
-
             AddCategoryMapping("1080P", TorznabCatType.MoviesHD, "1080P");
             AddCategoryMapping("2160P", TorznabCatType.MoviesHD, "2160P");
             AddCategoryMapping("720P", TorznabCatType.MoviesHD, "720P");

src/Jackett.Common/Indexers/TorrentSeeds.cs

@@ -12,6 +12,7 @@
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
 using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 
@@ -54,7 +55,7 @@ public TorrentSeeds(IIndexerConfigurationService configService, Utils.Clients.We
                  client: wc,
                  logger: l,
                  p: ps,
-                 configData: new ConfigurationDataBasicLoginWithRSSAndDisplay())
+                 configData: new ConfigurationDataBasicLoginWithRSSAndDisplay("For best results, change the <b>Torrents per page:</b> setting to <b>100</b> on your account profile."))
         {
             Encoding = Encoding.UTF8;
             Language = "en-us";
@@ -139,8 +140,6 @@ await ConfigureIfOK(
 
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
-            var releases = new List<ReleaseInfo>();
-
             // remove operator characters
             var cleanSearchString = Regex.Replace(query.GetQueryString().Trim(), "[ _.+-]+", " ", RegexOptions.Compiled);
 
@@ -156,16 +155,87 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
                 queryCollection.Add("query", cleanSearchString);
             foreach (var cat in MapTorznabCapsToTrackers(query))
                 queryCollection.Add($"cat[{cat}]", "1");
+
             searchUrl += "?" + queryCollection.GetQueryString();
             var response = await RequestWithCookiesAndRetryAsync(searchUrl);
+
+            // handle cookie expiration
             var results = response.ContentString;
-            if (!results.Contains("/logout.php?"))
+            if ((response.IsRedirect && response.RedirectingTo.Contains("/login.php?")) ||
+                (!response.IsRedirect && !results.Contains("/logout.php?")))
             {
-                await ApplyConfiguration(null);
+                await ApplyConfiguration(null); // re-login
                 response = await RequestWithCookiesAndRetryAsync(searchUrl);
-                results = response.ContentString;
             }
 
+            // handle single entries
+            if (response.IsRedirect)
+            {
+                var detailsLink = new Uri(response.RedirectingTo);
+                await FollowIfRedirect(response, accumulateCookies: true);
+                return ParseSingleResult(response, detailsLink);
+            }
+
+            return ParseMultiResult(response);
+        }
+
+        private List<ReleaseInfo> ParseSingleResult(WebResult response, Uri detailsLink)
+        {
+            var releases = new List<ReleaseInfo>();
+            var results = response.ContentString;
+
+            try
+            {
+                var parser = new HtmlParser();
+                var dom = parser.ParseDocument(results);
+                var content = dom.QuerySelector("tbody:has(script)");
+                var release = new ReleaseInfo();
+                release.MinimumRatio = 1;
+                release.MinimumSeedTime = 72 * 60 * 60;
+                var catStr = content.QuerySelector("tr:has(td:contains(\"Type\"))").Children[1].TextContent;
+                release.Category = MapTrackerCatDescToNewznab(catStr);
+                var qLink = content.QuerySelector("tr:has(td:contains(\"Download\"))")
+                                   .QuerySelector("a[href*=\"download.php?torrent=\"]");
+                release.Link = new Uri(SiteLink + qLink.GetAttribute("href"));
+                release.Title = dom.QuerySelector("h1").TextContent.Trim();
+                release.Details = detailsLink;
+                release.Guid = detailsLink;
+                var qSize = content.QuerySelector("tr:has(td.heading:contains(\"Size\"))").Children[1].TextContent
+                                   .Split('(')[0].Trim();
+                release.Size = ReleaseInfo.GetBytes(qSize);
+                var peerStats = content.QuerySelector("tr:has(td:has(a[href^=\"./peerlist_xbt.php?id=\"]))").Children[1]
+                                       .TextContent.Split(',');
+                var qSeeders = peerStats[0].Replace(" seeder(s)", "").Trim();
+                var qLeechers = peerStats[1].Split('=')[0].Replace(" leecher(s) ", "").Trim();
+                release.Seeders = ParseUtil.CoerceInt(qSeeders);
+                release.Peers = ParseUtil.CoerceInt(qLeechers) + release.Seeders;
+                var rawDateStr = content.QuerySelector("tr:has(td:contains(\"Added\"))").Children[1].TextContent;
+                var dateUpped = DateTimeUtil.FromUnknown(rawDateStr.Replace(",", string.Empty));
+
+                // Mar 4 2020, 05:47 AM
+                release.PublishDate = dateUpped.ToLocalTime();
+                var qGrabs = content.QuerySelector("tr:has(td:contains(\"Snatched\"))").Children[1];
+                release.Grabs = ParseUtil.CoerceInt(qGrabs.TextContent.Replace(" time(s)", ""));
+                var qFiles = content.QuerySelector("tr:has(td:has(a[href^=\"./filelist.php?id=\"]))").Children[1];
+                release.Files = ParseUtil.CoerceInt(qFiles.TextContent.Replace(" files", ""));
+                var qRatio = content.QuerySelector("tr:has(td:contains(\"Ratio After Download\"))").Children[1];
+                release.DownloadVolumeFactor = qRatio.QuerySelector("del") != null ? 0 : 1;
+                release.UploadVolumeFactor = 1;
+                releases.Add(release);
+            }
+            catch (Exception ex)
+            {
+                OnParseError(results, ex);
+            }
+
+            return releases;
+        }
+
+        private List<ReleaseInfo> ParseMultiResult(WebResult response)
+        {
+            var releases = new List<ReleaseInfo>();
+            var results = response.ContentString;
+
             try
             {
                 var parser = new HtmlParser();
@@ -215,5 +285,6 @@ protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuer
 
             return releases;
         }
+
     }
 }

src/Jackett.Common/Models/IndexerConfig/ConfigurationDataBasicLoginWithRSSAndDisplay.cs

@@ -5,14 +5,14 @@ public class ConfigurationDataBasicLoginWithRSSAndDisplay : ConfigurationData
         public StringItem Username { get; private set; }
         public StringItem Password { get; private set; }
         public HiddenItem RSSKey { get; private set; }
-        public DisplayItem DisplayText { get; private set; }
+        public DisplayItem Instructions { get; private set; }
 
-        public ConfigurationDataBasicLoginWithRSSAndDisplay()
+        public ConfigurationDataBasicLoginWithRSSAndDisplay(string instructionMessageOptional = null)
         {
             Username = new StringItem { Name = "Username" };
             Password = new StringItem { Name = "Password" };
             RSSKey = new HiddenItem { Name = "RSSKey" };
-            DisplayText = new DisplayItem("") { Name = "" };
+            Instructions = new DisplayItem(instructionMessageOptional) { Name = "" };
         }
     }
 }

src/Jackett.Common/Utils/Clients/WebClient.cs

@@ -168,14 +168,24 @@ protected virtual void PrepareRequest(WebRequest request)
 
         public virtual async Task<WebResult> GetResultAsync(WebRequest request)
         {
-            logger.Debug($"WebClient({ClientType}).GetResultAsync(Method: {request.Type} Url: {request.Url})");
+            if (logger.IsDebugEnabled) // performance optimization
+            {
+                var postData = "";
+                if (request.Type == RequestType.POST)
+                {
+                    var lines = request.PostData.Select(kvp => kvp.Key + "=" + kvp.Value);
+                    postData = $" PostData: {{{string.Join(", ", lines)}}} RawBody: {request.RawBody}";
+                }
+                logger.Debug($"WebClient({ClientType}).GetResultAsync(Method: {request.Type} Url: {request.Url}{postData})");
+            }
+
             PrepareRequest(request);
             await DelayRequest(request);
             var result = await Run(request);
             lastRequest = DateTime.Now;
             result.Request = request;
 
-            if (logger.IsDebugEnabled) // optimization to compute result.ContentString in debug mode only
+            if (logger.IsDebugEnabled) // performance optimization to compute result.ContentString in debug mode only
             {
                 var body = "";
                 var bodySize = 0;