From 19f19af2c65f94b337b85143ebcde590d9514e3f Mon Sep 17 00:00:00 2001 From: Yas Date: Sat, 7 Sep 2019 10:17:06 +1200 Subject: [PATCH 01/14] update HtmlWeb.cs so as not to parse content encoding when overriding encoding Although HtmlWeb class allows to ignore resp.ContentEncoding by OverrideEncoding property, it cannot help in the case where resp.ContentEncoding has encoding name which is not supported by Encoding class. This is because Encoding.GetEncoding method throws ArgumentException in that case. When OverrideEncoding is provided, the result from resp.ContentEncoding has not been used, so that simply skipping Encoding.GetEncoding method call can make it work. EncodingNotSupportedException is added in order to notify that such not supported content encoding is received in the response, with the information of the received content encoding name. This is more helpful than ArgumentException. --- .../EncodingNotSupportedException.cs | 39 +++++++++++++++++++ .../HtmlAgilityPack.Shared.projitems | 1 + src/HtmlAgilityPack.Shared/HtmlWeb.cs | 21 +++++++--- 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs diff --git a/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs b/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs new file mode 100644 index 00000000..ff5fe462 --- /dev/null +++ b/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs @@ -0,0 +1,39 @@ +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + +using System; +using System.Text; + +namespace HtmlAgilityPack +{ + public class EncodingNotSupportedException : Exception + { + #region Fields + + private string _encoding; + + #endregion + + #region Constructors + + internal EncodingNotSupportedException(string encoding) + { + _encoding = encoding; + } + + #endregion + + #region Properties + + internal string Encoding + { + get { return _encoding; } + } + + #endregion + } +} \ No newline at end of file diff --git a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems index fa59426f..eb937581 100644 --- a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems +++ b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems @@ -18,6 +18,7 @@ + diff --git a/src/HtmlAgilityPack.Shared/HtmlWeb.cs b/src/HtmlAgilityPack.Shared/HtmlWeb.cs index 0132bee0..09a1166b 100644 --- a/src/HtmlAgilityPack.Shared/HtmlWeb.cs +++ b/src/HtmlAgilityPack.Shared/HtmlWeb.cs @@ -1677,11 +1677,22 @@ private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc bool html = IsHtmlContent(resp.ContentType); bool isUnknown = string.IsNullOrEmpty(resp.ContentType); - Encoding respenc = !string.IsNullOrEmpty(resp.ContentEncoding) - ? Encoding.GetEncoding(resp.ContentEncoding) - : null; - if (OverrideEncoding != null) - respenc = OverrideEncoding; + Encoding respenc = OverrideEncoding; + if (respenc == null && !string.IsNullOrEmpty(resp.ContentEncoding)) + { + try + { + Encoding.GetEncoding(resp.ContentEncoding); + } + catch (ArgumentException ex) + { + if (ex.ParamName == "name") + { + throw new EncodingNotSupportedException(resp.ContentEncoding); + } + throw ex; + } + } if (CaptureRedirect) { From beb34cd08167695eef67c3495a72708edfcf2467 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sat, 7 Sep 2019 11:48:31 +1200 Subject: [PATCH 02/14] update EncodingNotSupportedException to make Encoding property accessible in public modify access modifier of Encoding property of EncodingNotSupportedException from internal to public. --- src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs | 2 +- src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs b/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs index ff5fe462..6da00d49 100644 --- a/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs +++ b/src/HtmlAgilityPack.Shared/EncodingNotSupportedException.cs @@ -29,7 +29,7 @@ internal EncodingNotSupportedException(string encoding) #region Properties - internal string Encoding + public string Encoding { get { return _encoding; } } diff --git a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems index eb937581..005bb1b2 100644 --- a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems +++ b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems @@ -17,8 +17,8 @@ - + From 73060a694590b24bcab283855752fdd16e88b807 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Fri, 15 Nov 2019 21:12:34 +1300 Subject: [PATCH 03/14] make HtmlWeb class testable wrap HttpRequest and HttpResponse with interfaces, and make HtmlWeb class testable. --- .../HtmlAgilityPack.Shared.projitems | 6 +++ src/HtmlAgilityPack.Shared/HtmlWeb.cs | 31 ++++++++++---- .../HttpWebRequestFactory.cs | 17 ++++++++ .../HttpWebRequestWrapper.cs | 30 ++++++++++++++ .../HttpWebResponseWrapper.cs | 41 +++++++++++++++++++ src/HtmlAgilityPack.Shared/IHttpWebRequest.cs | 20 +++++++++ .../IHttpWebRequestFactory.cs | 9 ++++ .../IHttpWebResponse.cs | 20 +++++++++ 8 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs create mode 100644 src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs create mode 100644 src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs create mode 100644 src/HtmlAgilityPack.Shared/IHttpWebRequest.cs create mode 100644 src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs create mode 100644 src/HtmlAgilityPack.Shared/IHttpWebResponse.cs diff --git a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems index 005bb1b2..8b76dfdf 100644 --- a/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems +++ b/src/HtmlAgilityPack.Shared/HtmlAgilityPack.Shared.projitems @@ -58,5 +58,11 @@ + + + + + + \ No newline at end of file diff --git a/src/HtmlAgilityPack.Shared/HtmlWeb.cs b/src/HtmlAgilityPack.Shared/HtmlWeb.cs index 09a1166b..22d9d484 100644 --- a/src/HtmlAgilityPack.Shared/HtmlWeb.cs +++ b/src/HtmlAgilityPack.Shared/HtmlWeb.cs @@ -83,6 +83,8 @@ public partial class HtmlWeb #region Fields + private IHttpWebRequestFactory _requestFactory; + private bool _autoDetectEncoding = true; private bool _cacheOnly; @@ -928,6 +930,19 @@ public bool UsingCache #endregion + #region Constructors + + public HtmlWeb() : this(new HttpWebRequestFactory()) + { + } + + internal HtmlWeb(IHttpWebRequestFactory requestFactory) + { + _requestFactory = requestFactory; + } + + #endregion + #region Public Methods #if !(NETSTANDARD1_3 || NETSTANDARD1_6) @@ -1547,10 +1562,10 @@ private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc ICredentials creds) { string cachePath = null; - HttpWebRequest req; + IHttpWebRequest req; bool oldFile = false; - req = WebRequest.Create(uri) as HttpWebRequest; + req = _requestFactory.Create(uri); req.Method = method; req.UserAgent = UserAgent; if (CaptureRedirect) @@ -1617,7 +1632,7 @@ private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc if (PreRequest != null) { // allow our user to change the request at will - if (!PreRequest(req)) + if (!PreRequest(req.Request)) { return HttpStatusCode.ResetContent; } @@ -1632,16 +1647,16 @@ private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc // } } - HttpWebResponse resp; + IHttpWebResponse resp; try { - resp = req.GetResponse() as HttpWebResponse; + resp = req.GetResponse(); } catch (WebException we) { _requestDuration = Environment.TickCount - tc; - resp = (HttpWebResponse) we.Response; + resp = we.Response == null ? null : new HttpWebResponseWrapper((HttpWebResponse) we.Response); if (resp == null) { if (oldFile) @@ -1668,7 +1683,7 @@ private HttpStatusCode Get(Uri uri, string method, string path, HtmlDocument doc // allow our user to get some info from the response if (PostResponse != null) { - PostResponse(req, resp); + PostResponse(req.Request, resp.Response); } _requestDuration = Environment.TickCount - tc; @@ -2103,7 +2118,7 @@ private HtmlDocument LoadUrl(Uri uri, string method, IWebProxy proxy, ICredentia } #endif #if !(NETSTANDARD1_3 || NETSTANDARD1_6) - private void SaveCacheHeaders(Uri requestUri, HttpWebResponse resp) + private void SaveCacheHeaders(Uri requestUri, IHttpWebResponse resp) { // we cache the original headers aside the cached document. string file = GetCacheHeadersPath(requestUri); diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs new file mode 100644 index 00000000..aec99922 --- /dev/null +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs @@ -0,0 +1,17 @@ +using System; +using System.Net; + +namespace HtmlAgilityPack +{ + internal class HttpWebRequestFactory : IHttpWebRequestFactory + { + public HttpWebRequestFactory() + { + } + + public IHttpWebRequest Create(Uri uri) + { + return new HttpWebRequestWrapper((HttpWebRequest)HttpWebRequest.Create(uri)); + } + } +} diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs new file mode 100644 index 00000000..7096cc5b --- /dev/null +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs @@ -0,0 +1,30 @@ +using System; +using System.Net; + +namespace HtmlAgilityPack +{ + internal class HttpWebRequestWrapper : IHttpWebRequest + { + HttpWebRequest _request; + public HttpWebRequest Request { get { return _request; } } + + public HttpWebRequestWrapper(HttpWebRequest request) + { + _request = request; + } + + public string Method { get { return _request.Method; } set { _request.Method = value; } } + public string UserAgent { get { return _request.UserAgent; } set { _request.UserAgent = value; } } + public bool AllowAutoRedirect { get { return _request.AllowAutoRedirect; } set { _request.AllowAutoRedirect = value; } } + public ICredentials Credentials { get { return _request.Credentials; } set { _request.Credentials = value; } } + public IWebProxy Proxy { get { return _request.Proxy; } set { _request.Proxy = value; } } + public Uri RequestUri { get { return _request.RequestUri; } } + public DateTime IfModifiedSince { get { return _request.IfModifiedSince; } set { _request.IfModifiedSince = value; } } + public CookieContainer CookieContainer { get { return _request.CookieContainer; } set { _request.CookieContainer = value; } } + + public IHttpWebResponse GetResponse() + { + return new HttpWebResponseWrapper((HttpWebResponse)_request.GetResponse()); + } + } +} diff --git a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs new file mode 100644 index 00000000..9e3d946e --- /dev/null +++ b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using System.Net; + +namespace HtmlAgilityPack +{ + internal class HttpWebResponseWrapper : IHttpWebResponse + { + HttpWebResponse _response; + public HttpWebResponse Response { get { return _response; } } + + public HttpWebResponseWrapper(HttpWebResponse response) + { + _response = response; + } + + public Uri ResponseUri { get { return _response.ResponseUri; } } + public HttpStatusCode StatusCode { get { return _response.StatusCode; } } + public string ContentType { get { return _response.ContentType; } } + public string ContentEncoding { get { return _response.ContentEncoding; } } + public WebHeaderCollection Headers { get { return _response.Headers; } } + public DateTime LastModified { get { return _response.LastModified; } } + + public void Close() + { + _response.Close(); + } + + public void Dispose() + { +#if NET20 || NET35 || NET40 + _response.Dispose(); +#endif + } + + public Stream GetResponseStream() + { + return _response.GetResponseStream(); + } + } +} diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs new file mode 100644 index 00000000..9f983e81 --- /dev/null +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs @@ -0,0 +1,20 @@ +using System; +using System.Net; + +namespace HtmlAgilityPack +{ + internal interface IHttpWebRequest + { + HttpWebRequest Request { get; } + string Method { get; set; } + string UserAgent { get; set; } + bool AllowAutoRedirect { get; set; } + ICredentials Credentials { get; set; } + IWebProxy Proxy { get; set; } + Uri RequestUri { get; } + DateTime IfModifiedSince { get; set; } + CookieContainer CookieContainer { get; set; } + + IHttpWebResponse GetResponse(); + } +} diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs new file mode 100644 index 00000000..13389352 --- /dev/null +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs @@ -0,0 +1,9 @@ +using System; + +namespace HtmlAgilityPack +{ + internal interface IHttpWebRequestFactory + { + IHttpWebRequest Create(Uri uri); + } +} diff --git a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs new file mode 100644 index 00000000..b778123f --- /dev/null +++ b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs @@ -0,0 +1,20 @@ +using System; +using System.IO; +using System.Net; + +namespace HtmlAgilityPack +{ + internal interface IHttpWebResponse : IDisposable + { + HttpWebResponse Response { get; } + Uri ResponseUri { get; } + HttpStatusCode StatusCode { get; } + string ContentType { get; } + string ContentEncoding { get; } + WebHeaderCollection Headers { get; } + DateTime LastModified { get; } + + Stream GetResponseStream(); + void Close(); + } +} From 494038dcf845eea58febf4c4305aa6dc1e20d1e6 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Fri, 15 Nov 2019 21:15:56 +1300 Subject: [PATCH 04/14] add a test project targeting multiple .NET versions add a test project targeting multiple .NET version. copy the existing tests in HtmlAgilityPack.Tests.Net45. --- .../Properties/AssemblyInfo.cs | 1 + .../Properties/AssemblyInfo.cs | 1 + .../AssemblyInfo.cs | 6 +- src/HtmlAgilityPack.sln | 23 + .../HtmlAgilityPack.Tests/AssemblyInfo.cs | 7 + .../HtmlAgilityPack.Tests.csproj | 70 ++ .../HtmlAgilityPack.Tests/HtmlAgilityPack.snk | Bin 0 -> 596 bytes .../HtmlDocumentTests.cs | 1040 +++++++++++++++++ .../HtmlAgilityPack.Tests/HtmlNode.Tests.cs | 104 ++ .../HtmlAgilityPack.Tests/files/mshome.htm | 582 +++++++++ .../HtmlAgilityPack.Tests/files/overflow.html | 94 ++ .../files/regression.html | 175 +++ .../HtmlAgilityPack.Tests/files/test.html | 12 + 13 files changed, 2114 insertions(+), 1 deletion(-) create mode 100644 src/Tests/HtmlAgilityPack.Tests/AssemblyInfo.cs create mode 100644 src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj create mode 100644 src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.snk create mode 100644 src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs create mode 100644 src/Tests/HtmlAgilityPack.Tests/HtmlNode.Tests.cs create mode 100644 src/Tests/HtmlAgilityPack.Tests/files/mshome.htm create mode 100644 src/Tests/HtmlAgilityPack.Tests/files/overflow.html create mode 100644 src/Tests/HtmlAgilityPack.Tests/files/regression.html create mode 100644 src/Tests/HtmlAgilityPack.Tests/files/test.html diff --git a/src/HtmlAgilityPack.Net40/Properties/AssemblyInfo.cs b/src/HtmlAgilityPack.Net40/Properties/AssemblyInfo.cs index d57cd451..e8bf2777 100644 --- a/src/HtmlAgilityPack.Net40/Properties/AssemblyInfo.cs +++ b/src/HtmlAgilityPack.Net40/Properties/AssemblyInfo.cs @@ -18,6 +18,7 @@ #endif #endif [assembly: InternalsVisibleTo("HtmlAgilityPack.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010027dc71d8e0b968c7324238e18a4cee4a367f1bf50c9d7a52d91ed46c6a1a584b9142c1d4234c4011d25437c909924079660c434eebe6d2c46412f30520a276e7ca8d8fa7075bb8b9e1c7502ef0e50423b32d469ba750012823fde16989ab42d8428ca5fdd0b06b801788a17239b78e0f75900012a50c5038ab93abbe2ac0d6ee")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("ZZZ Projects Inc.")] [assembly: AssemblyProduct("Html Agility Pack")] diff --git a/src/HtmlAgilityPack.Net45/Properties/AssemblyInfo.cs b/src/HtmlAgilityPack.Net45/Properties/AssemblyInfo.cs index d57cd451..e8bf2777 100644 --- a/src/HtmlAgilityPack.Net45/Properties/AssemblyInfo.cs +++ b/src/HtmlAgilityPack.Net45/Properties/AssemblyInfo.cs @@ -18,6 +18,7 @@ #endif #endif [assembly: InternalsVisibleTo("HtmlAgilityPack.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010027dc71d8e0b968c7324238e18a4cee4a367f1bf50c9d7a52d91ed46c6a1a584b9142c1d4234c4011d25437c909924079660c434eebe6d2c46412f30520a276e7ca8d8fa7075bb8b9e1c7502ef0e50423b32d469ba750012823fde16989ab42d8428ca5fdd0b06b801788a17239b78e0f75900012a50c5038ab93abbe2ac0d6ee")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("ZZZ Projects Inc.")] [assembly: AssemblyProduct("Html Agility Pack")] diff --git a/src/HtmlAgilityPack.NetStandard2_0/AssemblyInfo.cs b/src/HtmlAgilityPack.NetStandard2_0/AssemblyInfo.cs index 2bacd57c..7bc08237 100644 --- a/src/HtmlAgilityPack.NetStandard2_0/AssemblyInfo.cs +++ b/src/HtmlAgilityPack.NetStandard2_0/AssemblyInfo.cs @@ -1,3 +1,7 @@ using System; +using System.Runtime.CompilerServices; -[assembly: CLSCompliant(true)] \ No newline at end of file +[assembly: CLSCompliant(true)] + +[assembly: InternalsVisibleTo("HtmlAgilityPack.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010027dc71d8e0b968c7324238e18a4cee4a367f1bf50c9d7a52d91ed46c6a1a584b9142c1d4234c4011d25437c909924079660c434eebe6d2c46412f30520a276e7ca8d8fa7075bb8b9e1c7502ef0e50423b32d469ba750012823fde16989ab42d8428ca5fdd0b06b801788a17239b78e0f75900012a50c5038ab93abbe2ac0d6ee")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/HtmlAgilityPack.sln b/src/HtmlAgilityPack.sln index d862f345..bf2b51d9 100644 --- a/src/HtmlAgilityPack.sln +++ b/src/HtmlAgilityPack.sln @@ -17,6 +17,8 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "HtmlAgilityPack.Shared", "H EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HtmlAgilityPack.Tests.Net45", "Tests\HtmlAgilityPack.Tests.Net45\HtmlAgilityPack.Tests.Net45.csproj", "{F7D2CBBC-E23E-478E-865B-6BE445B55EC8}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HtmlAgilityPack.Tests", "Tests\HtmlAgilityPack.Tests\HtmlAgilityPack.Tests.csproj", "{D4614863-964F-4333-B592-A4725DACC385}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution HtmlAgilityPack.Shared\HtmlAgilityPack.Shared.projitems*{1028002a-bbe2-4ff2-94b7-a8368a8f6887}*SharedItemsImports = 4 @@ -116,6 +118,26 @@ Global {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x64.Build.0 = Release|Any CPU {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x86.ActiveCfg = Release|Any CPU {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x86.Build.0 = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|ARM.ActiveCfg = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|ARM.Build.0 = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|x64.ActiveCfg = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|x64.Build.0 = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|x86.ActiveCfg = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Debug|x86.Build.0 = Debug|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|Any CPU.Build.0 = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|ARM.ActiveCfg = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|ARM.Build.0 = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|x64.ActiveCfg = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|x64.Build.0 = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.ActiveCfg = Release|Any CPU + {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -126,6 +148,7 @@ Global {083D99CD-F083-4122-AA96-4E3A84F3EC01} = {4E506C9F-6514-4055-AAA8-DC89D85558FE} {5BA9A0D7-173F-4C4F-AFFA-3DD6DC2F9A79} = {4E506C9F-6514-4055-AAA8-DC89D85558FE} {F7D2CBBC-E23E-478E-865B-6BE445B55EC8} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} + {D4614863-964F-4333-B592-A4725DACC385} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5870022-9E57-469B-A27E-E840C436B2EF} diff --git a/src/Tests/HtmlAgilityPack.Tests/AssemblyInfo.cs b/src/Tests/HtmlAgilityPack.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..bc466ecb --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/AssemblyInfo.cs @@ -0,0 +1,7 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: ComVisible(false)] + +[assembly: Guid("d2276641-9ec1-4c80-a59b-4ba8e2a578aa")] \ No newline at end of file diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj new file mode 100644 index 00000000..9930263e --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj @@ -0,0 +1,70 @@ + + + + net40;net45;netcoreapp2.0 + HtmlAgilityPack.Tests + HtmlAgilityPack.Tests + 1.0.0 + ZZZ Projects Inc. + ZZZ Projects Inc. + Html Agility Pack Tests + Copyright © ZZZ Projects Inc. 2014 - 2017 + true + HtmlAgilityPack.snk + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + Always + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.snk b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.snk new file mode 100644 index 0000000000000000000000000000000000000000..c2d78f7bcb1057951f5fbca7acdaf650000840d3 GIT binary patch literal 596 zcmV-a0;~N80ssI2Bme+XQ$aES1ONa50096e+;Q08xoF2SLO9`yOzuiHe;f4-oqAH) z9@K1V8dyt_Lc!D{Oh6IRR5!^9l0bQ83`0)q=F-Gu67vNhqIT!XjgO}XTe!L5$51Zt zKCPyr|-{o!eet3ud9jHUh1uxo%9h@o;hw~h~WkN^^;3{W_$ldHZez}D`0 z<>hCFJaXf$jd}LiPo`)tb`h#R0&H>OsS*3c4p>43;jl!UzrkQ?I!VAF^Y(g|PLhWjz*MIQ&8a9u+%a;5)AXuM}?Cwy<$iy~1X~H`Tf#k8u zv!Gv6BDHPeu9VB3b0iqFip;W9MXJV!pG(%)wiUP@{NK*+cI;BBqt4W?De7&LI~S#! z3+7cB6r<%wRTrIk8d?=~MVkMDyd1-6c1|B1oQc%}zkQpFWwT6t_aod8a`2I**o~+i zWW1$?m(HMM9FEFbSuH}e%{7N$lLmSy&Ck(2#W#q?~b=#8H6Os(x`29?lLaXtmS}T%%kkPx$qgEk i%BCfV#W_bQWAU$zup%0cHi=C+Un1o-w791&j>87a<|Lf} literal 0 HcmV?d00001 diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs new file mode 100644 index 00000000..8a615591 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs @@ -0,0 +1,1040 @@ +using System.IO; +using System.Linq; +using System.Net; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Threading; + +namespace HtmlAgilityPack.Tests +{ + [TestFixture] + public class HtmlDocumentTests + { + private string _contentDirectory; + + + + [OneTimeSetUp] + public void Setup() + { + + + _contentDirectory = Path.GetDirectoryName(typeof(HtmlDocumentTests).Assembly.Location).ToString() + "\\files\\"; + // _contentDirectory = Path.Combine(@"C:\Users\Jonathan\Desktop\Z\zzzproject\HtmlAgilityPack\HtmlAgilityPack.Tests\bin\Debug\files\"); + } + + private HtmlDocument GetMshomeDocument() + { + var doc = new HtmlDocument(); + doc.Load(_contentDirectory + "mshome.htm"); + return doc; + } + + [Test] + public void HtmlAgilityPack_AttributeCollectionBug() + { + { + const string firstAttrName = "first"; + const string secondAttrName = "second"; + const string value = "value"; + + HtmlNode firstNode = HtmlNode.CreateNode("
"); + firstNode.Attributes.Add(firstAttrName, value); + + HtmlNode secondNode = HtmlNode.CreateNode("
"); + secondNode.Attributes.Add(secondAttrName, value); + + secondNode.Attributes[0] = firstNode.Attributes[0]; + + Assert.IsNotNull(secondNode.Attributes[0]); + Assert.AreEqual(firstAttrName, secondNode.Attributes[0].Name); + + Assert.IsNotNull(secondNode.Attributes[firstAttrName], $"'{firstAttrName}' should exist in the collection"); + Assert.AreEqual(firstAttrName, secondNode.Attributes[firstAttrName].Name); + + Assert.IsNull(secondNode.Attributes [secondAttrName], $"{secondAttrName} should not exist in the collection"); + } + + { + const string firstAttrName = "first"; + const string secondAttrName = "second"; + const string value = "value"; + + HtmlNode firstNode = HtmlNode.CreateNode("
"); + firstNode.Attributes.Add(firstAttrName, value); + + HtmlNode secondNode = HtmlNode.CreateNode("
"); + secondNode.Attributes.Add(secondAttrName, value); + var a = secondNode.Attributes[secondAttrName]; + secondNode.Attributes[secondAttrName] = firstNode.Attributes[firstAttrName]; + + Assert.IsNotNull(secondNode.Attributes[firstAttrName]); + Assert.AreEqual(firstAttrName, secondNode.Attributes[firstAttrName].Name); + + Assert.IsNotNull(secondNode.Attributes[0], $"'{firstAttrName}' should exist in the collection"); + Assert.AreEqual(firstAttrName, secondNode.Attributes[firstAttrName].Name); + + Assert.IsNull(secondNode.Attributes[secondAttrName], $"{secondAttrName} should not exist in the collection"); + } + } + + [Test] + public void TextInsideScriptTagShouldHaveCorrectStreamPosition() + { + { + var document = new HtmlDocument(); + document.LoadHtml(@"foo"); + var scraptText = document.DocumentNode.FirstChild.FirstChild; + Assert.AreEqual(8, scraptText.StreamPosition); + Assert.AreEqual(1, scraptText.Line); + Assert.AreEqual(9, scraptText.LinePosition); + } + { + var document = new HtmlDocument(); + document.LoadHtml(@""); + var scriptText = document.DocumentNode.FirstChild.FirstChild; + Assert.AreEqual(8, scriptText.StreamPosition); + Assert.AreEqual(1, scriptText.Line); + Assert.AreEqual(9, scriptText.LinePosition); + } + { + var document = new HtmlAgilityPack.HtmlDocument(); + document.LoadHtml(@" +foo"); + var scraptText = document.DocumentNode.LastChild.FirstChild; + // var aa = scraptText.FirstChild; + Assert.AreEqual(10, scraptText.StreamPosition); + Assert.AreEqual(2, scraptText.Line); + Assert.AreEqual(9, scraptText.LinePosition); + } + + + { + var document = new HtmlAgilityPack.HtmlDocument(); + document.LoadHtml(@" +"); + var scriptText = document.DocumentNode.LastChild.FirstChild; + Assert.AreEqual(10, scriptText.StreamPosition); + Assert.AreEqual(2, scriptText.Line); + Assert.AreEqual(9, scriptText.LinePosition); + } + } + + [Test] + public void CreateAttribute() + { + var doc = new HtmlDocument(); + var a = doc.CreateAttribute("href"); + Assert.AreEqual("href", a.Name); + } + + [Test] + public void CreateAttributeWithEncodedText() + { + var doc = new HtmlDocument(); + var a = doc.CreateAttribute("href", "http://something.com\"&<>"); + Assert.AreEqual("href", a.Name); + Assert.AreEqual("http://something.com\"&<>", a.Value); + } + + [Test] + public void CreateAttributeWithText() + { + var doc = new HtmlDocument(); + var a = doc.CreateAttribute("href", "http://something.com"); + Assert.AreEqual("href", a.Name); + Assert.AreEqual("http://something.com", a.Value); + } + + [Test] + public void testTEXTAREA() + { + { + var doc = new HtmlAgilityPack.HtmlDocument(); + doc.LoadHtml(@""); + HtmlNodeCollection divs = doc.DocumentNode.SelectNodes("//div"); + + Assert.IsNull(divs); + + HtmlNode ta = doc.DocumentNode.SelectSingleNode("//textarea"); + Assert.IsTrue(ta.InnerHtml.Contains("div")); + } + } + + //[Test] + //public void CreateComment() + //{ + // var doc = new HtmlDocument(); + // var a = doc.CreateComment(); + // Assert.AreEqual(HtmlNode.HtmlNodeTypeNameComment, a.Name); + // Assert.AreEqual(a.NodeType, HtmlNodeType.Comment); + //} + + //[Test] + //public void CreateCommentWithText() + //{ + // var doc = new HtmlDocument(); + // var a = doc.CreateComment("something"); + // Assert.AreEqual(HtmlNode.HtmlNodeTypeNameComment, a.Name); + // Assert.AreEqual("something", a.InnerText); + // Assert.AreEqual(a.NodeType, HtmlNodeType.Comment); + //} + + [Test] + public void CreateElement() + { + var doc = new HtmlDocument(); + var a = doc.CreateElement("a"); + Assert.AreEqual("a", a.Name); + Assert.AreEqual(a.NodeType, HtmlNodeType.Element); + } + + //[Test] + //public void CreateTextNode() + //{ + // var doc = new HtmlDocument(); + // var a = doc.CreateTextNode(); + // Assert.AreEqual(HtmlNode.HtmlNodeTypeNameText, a.Name); + // Assert.AreEqual(a.NodeType, HtmlNodeType.Text); + //} + + [Test] + public void CreateTextNodeWithText() + { + var doc = new HtmlDocument(); + var a = doc.CreateTextNode("something"); + Assert.AreEqual("something", a.InnerText); + Assert.AreEqual(a.NodeType, HtmlNodeType.Text); + } + + [Test] + public void HtmlEncode() + { + var result = HtmlDocument.HtmlEncode("http://something.com\"&<>"); + Assert.AreEqual("http://something.com"&<>", result); + } + + [Test] + public void TestParse() + { + var doc = GetMshomeDocument(); + Assert.IsTrue(doc.DocumentNode.Descendants().Count() > 0); + } + + //[Test] + //public void TestLimitDepthParse() + //{ + // HtmlAgilityPack.HtmlDocument.MaxDepthLevel = 10; + // var doc = GetMshomeDocument(); + // try + // { + // Assert.IsTrue(doc.DocumentNode.Descendants().Count() > 0); + // } + // catch (ArgumentException e) + // { + // Assert.IsTrue(e.Message == HtmlAgilityPack.HtmlNode.DepthLevelExceptionMessage); + // } + // HtmlAgilityPack.HtmlDocument.MaxDepthLevel = int.MaxValue; + //} + + [Test] + public void TestParseSaveParse() + { + var doc = GetMshomeDocument(); + var doc1desc = + doc.DocumentNode.Descendants().Where(x => !string.IsNullOrWhiteSpace(x.InnerText)).ToList(); + doc.Save(_contentDirectory + "testsaveparse.html"); + + var doc2 = new HtmlDocument(); + doc2.Load(_contentDirectory + "testsaveparse.html"); + var doc2desc = + doc2.DocumentNode.Descendants().Where(x => !string.IsNullOrWhiteSpace(x.InnerText)).ToList(); + Assert.AreEqual(doc1desc.Count, doc2desc.Count); + //for(var i=0; i< doc1desc.Count;i++) + //{ + // try + // { + // Assert.AreEqual(doc1desc[i].Name, doc2desc[i].Name); + // }catch(Exception e) + // { + // throw; + // } + //} + } + + [Test] + public void TestRemoveUpdatesPreviousSibling() + { + var doc = GetMshomeDocument(); + var docDesc = doc.DocumentNode.Descendants().ToList(); + var toRemove = docDesc[1200]; + var toRemovePrevSibling = toRemove.PreviousSibling; + var toRemoveNextSibling = toRemove.NextSibling; + toRemove.Remove(); + Assert.AreSame(toRemovePrevSibling, toRemoveNextSibling.PreviousSibling); + } + + [Test] + public void TestReplaceUpdatesSiblings() + { + var doc = GetMshomeDocument(); + var docDesc = doc.DocumentNode.Descendants().ToList(); + var toReplace = docDesc[1200]; + var toReplacePrevSibling = toReplace.PreviousSibling; + var toReplaceNextSibling = toReplace.NextSibling; + var newNode = doc.CreateElement("tr"); + toReplace.ParentNode.ReplaceChild(newNode, toReplace); + Assert.AreSame(toReplacePrevSibling, newNode.PreviousSibling); + Assert.AreSame(toReplaceNextSibling, newNode.NextSibling); + } + + [Test] + public void TestInsertUpdateSiblings() + { + var doc = GetMshomeDocument(); + var newNode = doc.CreateElement("td"); + var toReplace = doc.DocumentNode.ChildNodes[2]; + var toReplacePrevSibling = toReplace.PreviousSibling; + var toReplaceNextSibling = toReplace.NextSibling; + doc.DocumentNode.ChildNodes.Insert(2, newNode); + Assert.AreSame(newNode.NextSibling, toReplace); + Assert.AreSame(newNode.PreviousSibling, toReplacePrevSibling); + Assert.AreSame(toReplaceNextSibling, toReplace.NextSibling); + } + + [Test] + public void TestCopyFromNode() + { + var html = @"
"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var divNode = htmlDoc.DocumentNode.SelectSingleNode("//div"); + + var newNode = HtmlAgilityPack.HtmlNode.CreateNode(""); + newNode.CopyFrom(divNode); + + var attribute1 = divNode.Attributes[0]; + + var attribute2 = newNode.Attributes[0]; + + Assert.AreEqual(divNode.Attributes.Count, newNode.Attributes.Count); + Assert.AreEqual(attribute1.Value, attribute2.Value); + Assert.AreEqual(attribute1.QuoteType, attribute2.QuoteType); + } + + [Test] + public void TestCommentNode() + { + var html = + @" + + + + + +

This is bold headddding

+

This is underlinyed paragraph

+

This is italic heading

+ + "; + + var htmlDoc = new HtmlAgilityPack.HtmlDocument(); + htmlDoc.LoadHtml(html); + + var h1 = htmlDoc.DocumentNode.SelectNodes("//h1"); + var comments = htmlDoc.DocumentNode.SelectNodes("//comment()"); + + Assert.AreEqual(h1.Count, 1); + Assert.AreEqual(comments.Count, 4); + } + + [Test] + public void TestCloneNode() + { + var html = @"
"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var divNode = htmlDoc.DocumentNode.SelectSingleNode("//div"); + + var newNode = divNode.Clone(); + + var attribute1 = divNode.Attributes[0]; + + var attribute2 = newNode.Attributes[0]; + + Assert.AreEqual(divNode.Attributes.Count, newNode.Attributes.Count); + Assert.AreEqual(attribute1.Value, attribute2.Value); + Assert.AreEqual(attribute1.QuoteType, attribute2.QuoteType); + } + + [Test] + public void TestEmptyTag_Single() + { + var html = ""; + var doc = new HtmlAgilityPack.HtmlDocument(); + doc.LoadHtml(html); + + Assert.AreEqual(@"", doc.DocumentNode.OuterHtml); + } + + [Test] + public void TestEmptyTag_Many() + { + { + var html = ""; + var doc = new HtmlAgilityPack.HtmlDocument(); + doc.LoadHtml(html); + + Assert.AreEqual(@"", doc.DocumentNode.OuterHtml); + } + + { + var html = ""; + var doc = new HtmlAgilityPack.HtmlDocument(); + doc.LoadHtml(html); + + Assert.AreEqual(@"", doc.DocumentNode.OuterHtml); + } + } + + [Test] + public void TestAddClass() + { + var html = @"

This is new heading

"; + + string output = "

This is new heading

"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var h1Node = htmlDoc.DocumentNode.SelectSingleNode("//h1"); + + h1Node.AddClass("input"); + + Assert.AreEqual(h1Node.OuterHtml, output); + } + + [Test] + public void TestRemoveClass() + { + var output = @"

This is new heading

"; + + string html = "

This is new heading

"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var h1Node = htmlDoc.DocumentNode.SelectSingleNode("//h1"); + + h1Node.RemoveClass("input"); + + Assert.AreEqual(h1Node.OuterHtml, output); + } + + [Test] + public void TestReplaceClass() + { + var output = "

This is new heading

"; + + string html = "

This is new heading

"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var h1Node = htmlDoc.DocumentNode.SelectSingleNode("//h1"); + + h1Node.ReplaceClass("important", "input"); + + Assert.AreEqual(h1Node.OuterHtml, output); + } + + [Test] + + public void TestDetectEncoding() + { + string html = Path.Combine(_contentDirectory, "test.html"); + + var htmlDoc = new HtmlDocument(); + + var encoding = htmlDoc.DetectEncoding(html); + + Assert.AreEqual(System.Text.Encoding.UTF8, encoding); + } + + [Test] + public void TestLoadWithCache() + { + var dir = _contentDirectory + "cache"; + Directory.CreateDirectory(dir); + + var web = new HtmlAgilityPack.HtmlWeb() + { + CachePath = dir, + UsingCache = true + }; + + var url = "http://html-agility-pack.net/"; + var docCache = web.Load(url); + + var docLoad = new HtmlAgilityPack.HtmlWeb().Load(url); + Assert.AreEqual(docLoad.DocumentNode.OuterHtml, docCache.DocumentNode.OuterHtml); + } + + [Test] + public void OuterHtmlHasBeenCalled_RemoveCalled_SubsequentOuterHtmlCallsAreBroken() + { + var doc = new HtmlDocument(); + doc.LoadHtml("
SOme text here
some boldedtext
"); + var resultList = doc.DocumentNode.SelectNodes("//div"); + Assert.AreEqual(2, resultList.Count); + resultList.First().Remove(); + Assert.AreEqual("
some boldedtext
", doc.DocumentNode.OuterHtml); + var resultList2 = doc.DocumentNode.SelectNodes("//div"); + Assert.AreEqual(1, resultList2.Count); + resultList2.First().Remove(); + //
some boldedtext
should have been removed + Assert.AreEqual("", doc.DocumentNode.OuterHtml); + } + + [Test] + public void TestAttributeDeEntitizeValue() + { + var html = + "

This is underlined paragraph

"; + + string output = "\"Hello\""; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var body = htmlDoc.DocumentNode.SelectSingleNode("//body"); + + var val = body.Attributes["data-foo"].DeEntitizeValue; + + Assert.AreEqual(output, val); + } + + [Test] + public void TestAttributeDeEntitizeValue2() + { + var html = + "

This is underlined paragraph

"; + + string output = "\"\"Hello\"\""; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var body = htmlDoc.DocumentNode.SelectSingleNode("//body"); + + var val = body.Attributes["data-foo"].DeEntitizeValue; + + Assert.AreEqual(output, val); + } + + [Test] + public void TestAttributeDeEntitizeValue3() + { + var doc = new HtmlDocument(); + var a = doc.CreateAttribute("href", "\"bad_href\""); + Assert.AreEqual("href", a.Name); + Assert.AreEqual("\"bad_href\"", a.DeEntitizeValue); + } + + [Test] + public void TestAttributeDeEntitizeValue4() + { + var html = "

This is underlined paragraph

"; + + string output = "\"Hello\""; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var body = htmlDoc.DocumentNode.SelectSingleNode("//body"); + + var val = body.Attributes["data-foo"].DeEntitizeValue; + + Assert.AreEqual(output, val); + } + + [Test] + public void TestAttributeValue() + { + var html = @"

This is underlined paragraph

"; + + string output = "http://example.com/path?productId=9788762505032&title=something"; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + var body = htmlDoc.DocumentNode.SelectSingleNode("//body"); + + var val = body.Attributes["data-foo"].Value; + + Assert.AreEqual(output, val); + } + + [Test] + + public void TestSingleNodesEmptyCollection() + { + var html = + @" + + +

This is bold heading

+

This is underlined paragraph

+

This is italic heading

+

This is new heading

+ + "; + + var output = 0; + + var htmlDoc = new HtmlDocument(); + htmlDoc.LoadHtml(html); + + htmlDoc.OptionEmptyCollection = true; + + var divNodes = htmlDoc.DocumentNode.SelectNodes("//div"); + + Assert.AreEqual(output, divNodes.Count); + } + + [Test] + public void TestCreateNode() + { + string output = "

text

"; + HtmlNode node1 = HtmlNode.CreateNode(@" +

text

+ "); + + HtmlNode node2 = HtmlNode.CreateNode(@" + +

text

+ + "); + + HtmlNode node3 = HtmlNode.CreateNode(@"

text

"); + + Assert.AreEqual(output, node1.OuterHtml); + Assert.AreEqual(output, node2.OuterHtml); + Assert.AreEqual(output, node3.OuterHtml); + + try + { + HtmlNode node4 = HtmlNode.CreateNode("

a

b

"); + } + catch (Exception e) + { + Assert.AreEqual("Multiple node elments can't be created.", e.Message); + } + + HtmlNode node5 = HtmlNode.CreateNode(@"/r/n"); + Assert.AreEqual("/r/n", node5.OuterHtml); + } + + [Test] + public void TestParseListInList() + { + var html = "
    1. x
"; + + var doc = new HtmlDocument(); + doc.OptionFixNestedTags = true; + doc.Load(new StringReader(html)); + + Assert.AreEqual(doc.DocumentNode.OuterHtml, html); + } + + [Test] + public void TestLoadWithUri() + { + //string adress = "http://www.filmweb.pl/film/Piraci+z+Karaib%C3%B3w%3A+Zemsta+Salazara-2017-606542"; + //Uri uri = new Uri(adress, true); + //var web = new HtmlWeb(); + //HtmlAgilityPack.HtmlDocument document = web.Load(uri); + //Assert.AreNotEqual(string.Empty, document.DocumentNode.OuterHtml); + } + + [Test] + public void TestFormTag() + { + var html = @"
"; + var document = new HtmlDocument(); + document.LoadHtml(html); + var result = document.DocumentNode.Descendants().Select(dn => new {dn.NodeType, dn.Name, dn.OuterHtml}).ToArray(); + Assert.AreEqual(html, document.DocumentNode.OuterHtml); + Assert.AreEqual(1, result.Count()); + } + + [Test] + public void TestNumericTag() + { + var html = @"
<1
"; + var document = new HtmlDocument(); + document.LoadHtml(html); + var result = document.DocumentNode.Descendants().Select(dn => new {dn.NodeType, dn.Name, dn.OuterHtml}).ToArray(); + Assert.AreEqual(html, document.DocumentNode.OuterHtml); + } + + [Test] + public void ImplicitTag() + { + { + var html = @"
a
b
c
a
b
c"; + var document = new HtmlDocument(); + document.LoadHtml(html); + var result = document.DocumentNode.Descendants().Select(dn => new {dn.NodeType, dn.Name, dn.OuterHtml}).ToArray(); + + // TODO: Fix issue with last "dd" + Assert.AreEqual(html + "
", document.DocumentNode.OuterHtml); + } + + + { + var html = @"
a
b
c
a
b
c
"; + var document = new HtmlDocument(); + document.LoadHtml(html); + var result = document.DocumentNode.Descendants().Select(dn => new {dn.NodeType, dn.Name, dn.OuterHtml}).ToArray(); + + Assert.AreEqual(html, document.DocumentNode.OuterHtml); + } + } + + [Test] + public void DeEntitize() + { + var html = @"mouse's house"; + + Assert.AreEqual("mouse's house", HtmlEntity.DeEntitize("mouse's house")); + } + + [Test] + public void InnerText_Comment() + { + var document = new HtmlDocument(); + document.LoadHtml("

Expected value

"); + + Assert.AreEqual("Expected value", document.DocumentNode.FirstChild.InnerText); + } + + [Test] + public void TestFixNestedAnchors() + { + var inHtml = " Here's a great link! Here's another one!Here's some unrelated text."; + var expectedHtml = " Here's a great link! Here's another one!Here's some unrelated text."; + + var doc = new HtmlDocument(); + doc.LoadHtml(inHtml); + + Assert.AreEqual(expectedHtml, doc.DocumentNode.OuterHtml); + } + + [Test] + public void TestHandleNestedAnchors() + { + var inHtml = ""; + var expectedHtml = ""; + var doc = new HtmlDocument(); + doc.LoadHtml(inHtml); + Assert.AreEqual(expectedHtml, doc.DocumentNode.OuterHtml); + } + + [Test] + public void TestInnerText() + { + var inHtml = @" + + + InnerText bug Demo + + + +
+ This demonstration should show that the HAP currently parses div tags incorrectly, parsing carriage returns, new lines and tabular indents as text. +
+ +"; + var expectedHtml = "InnerText bug DemoThis demonstration should show that the HAP currently parses div tags incorrectly, parsing carriage returns, new lines and tabular indents as text."; + + var doc = new HtmlDocument() {BackwardCompatibility = false}; + doc.LoadHtml(inHtml); + + Assert.AreEqual(expectedHtml, doc.DocumentNode.InnerText); + } + + [Test] + public void TestOptionTag() + { + var html = ""; + + string output = ""; + var document = new HtmlDocument(); + document.LoadHtml(html); + Assert.AreEqual(output, document.DocumentNode.OuterHtml); + } + + [Test] + public void VerifyChildDivParent() + { + var doc = new HtmlDocument(); + doc.LoadHtml(""); + + var div = HtmlNode.CreateNode("
"); + var div2 = HtmlNode.CreateNode("
"); + + doc.DocumentNode.ChildNodes.Add(div); + div.ChildNodes.Add(div2); + + Assert.AreEqual(div.Name, div2.ParentNode.Name); + + } + + + [Test] + public void ChildIsRemovedFromParent() + { + var doc = new HtmlDocument(); + doc.LoadHtml(""); + + var div = HtmlNode.CreateNode("
"); + var div2 = HtmlNode.CreateNode("
"); + + div.ChildNodes.Add(div2); + doc.DocumentNode.ChildNodes.Add(div); + + + div.FirstChild.Remove(); + + Assert.AreEqual(0, div.ChildNodes.Count); + + } + + + [Test] + public void GetEncapsulatedData() + { + HtmlWeb stackoverflowSite = new HtmlWeb(); + HtmlDocument htmlDocument = stackoverflowSite.Load("https://stackoverflow.com/"); + StackOverflowPage stackOverflowPage = htmlDocument.DocumentNode.GetEncapsulatedData(); + IEnumerable filtered = stackOverflowPage.Questions.OrderByDescending(new Func(x => x.Statistics.Votes)); + + Assert.IsTrue(filtered.Count() > 5); + Assert.IsTrue(filtered.ElementAt(0).Statistics.Votes > 0); + + } + + [Test] + public void CompareLowerCulture() + { + + string html = File.ReadAllText(_contentDirectory + "regression.html"); + HtmlNode node1 = null; + // Test 1 + CultureInfo cul1 = CultureInfo.CreateSpecificCulture("en-US"); + Thread.CurrentThread.CurrentCulture = cul1; + HtmlAgilityPack.HtmlDocument doc1 = new HtmlAgilityPack.HtmlDocument(); + doc1.LoadHtml(html); + + node1 = doc1.DocumentNode.SelectSingleNode("//div[@id='mainContents']/h2"); + + CultureInfo cul2 = CultureInfo.CreateSpecificCulture("tr-TR"); + Thread.CurrentThread.CurrentCulture = cul2; + HtmlAgilityPack.HtmlDocument doc2 = new HtmlAgilityPack.HtmlDocument(); + doc2.LoadHtml(html); + var s = doc2.DocumentNode.OuterHtml; + + HtmlNode node2 = doc2.DocumentNode.SelectSingleNode("//div[@id='mainContents']/h2"); + if (node1?.InnerHtml == node2?.InnerHtml) + + + Assert.AreEqual(node1?.InnerHtml, node2?.InnerHtml); + Assert.AreEqual(0, doc2.DocumentNode.OwnerDocument.ParseErrors.Count()); + } + + + [Test] + public void OverFlowNotEndTag() + { + + string html = File.ReadAllText(_contentDirectory + "overflow.html"); + HtmlNode node1 = null; + // Test 1 + + HtmlAgilityPack.HtmlDocument doc1 = new HtmlAgilityPack.HtmlDocument(); + doc1.LoadHtml(html); + + Assert.AreEqual(15, doc1.DocumentNode.ChildNodes[4].ChildNodes.Count); + + Assert.AreEqual(0, doc1.DocumentNode.OwnerDocument.ParseErrors.Count()); + } + + [Test] + public void SanitizeXmlElementNameWithColon() + { + var input = @" + +"; + var htmlDoc = new HtmlAgilityPack.HtmlDocument(); + htmlDoc.LoadHtml(input); + htmlDoc.OptionDefaultStreamEncoding = System.Text.Encoding.UTF8; + htmlDoc.OptionOutputAsXml = true; + htmlDoc.OptionOutputOriginalCase = true; + var xmlDoc = htmlDoc.DocumentNode.WriteTo(); + + var expected = @"" + + @" + <_value3a_element> +"; + + Assert.AreEqual(expected, xmlDoc); + } + + [Test] + public void DoesNotSanitizeXmlElementNameWithColonWhenConfiguredToPreserveXmlNamespaces() + { + var input = @" + +"; + var htmlDoc = new HtmlAgilityPack.HtmlDocument(); + htmlDoc.LoadHtml(input); + htmlDoc.OptionDefaultStreamEncoding = System.Text.Encoding.UTF8; + htmlDoc.OptionOutputAsXml = true; + htmlDoc.OptionOutputOriginalCase = true; + htmlDoc.OptionPreserveXmlNamespaces = true; + var xmlDoc = htmlDoc.DocumentNode.WriteTo(); + + var expected = @"" + + @" + +"; + + Assert.AreEqual(expected, xmlDoc); + } + + [Test] + public void HasClass_WhereClassWithWhitespacePassed_ShouldReturnTrue() + { + var input = @""; + var htmlDoc = new HtmlDocument(); + + htmlDoc.LoadHtml(input); + + var aTag = htmlDoc.DocumentNode.SelectSingleNode("//a"); + Assert.True(aTag.HasClass("disabled")); + } + + [Test] + public void GetClasses_WhereClassWithWhitespacePassed_ShouldNotBeEmpty() + { + var input = @""; + var htmlDoc = new HtmlDocument(); + + htmlDoc.LoadHtml(input); + + var aTag = htmlDoc.DocumentNode.SelectSingleNode("//a"); + Assert.IsNotEmpty(aTag.GetClasses()); + } + + [HasXPath] + public class StackOverflowPage + { + [XPath("//*[@id='question-mini-list']/div/div")] + public IEnumerable Questions { get; set; } + + [XPath("//*[@id='hot-network-questions']/ul//li")] + public IEnumerable GetHotNetworkQuestions { get; set; } + + } + + [HasXPath] + [DebuggerDisplay("StackOverflowQuestion : {Question.QuestionTitle}")] + public class StackOverflowQuestion + { + [XPath("/div[@class='cp']")] + public StatisticsBox Statistics { get; set; } + + + [XPath("/div[@class='summary']")] + public QuestionBox Question { get; set; } + + + [XPath("/div[@class='summary']/div[@class='started']")] + public UserBox User { get; set; } + + } + + [HasXPath] + [DebuggerDisplay("Votes={Votes} , Answers={Answers} , Views={Views}")] + public class StatisticsBox + { + [XPath("/div[1]/div/span")] + public int Votes { get; set; } + + [XPath("/div[2]/div/span")] + public int Answers { get; set; } + + [XPath("/div[3]/div/span")] + public string Views { get; set; } + + + } + + [HasXPath] + [DebuggerDisplay("QuestionTitle={QuestionTitle}")] + public class QuestionBox + { + [XPath("/h3/a")] + public string QuestionTitle { get; set; } + + [XPath("/h3/a", "href")] + public string QuestionLink { get; set; } + + [XPath("/div[starts-with(@class,'tags')]//a")] + public IEnumerable Tags { get; set; } + } + + [HasXPath] + [DebuggerDisplay("UserID={UserID} , ReputationScore={ReputationScore}")] + public class UserBox + { + [XPath("/a[1]/span", "title")] + public DateTime ExactTime { get; set; } + + [XPath("/a[1]/span")] + public string RelativeTime { get; set; } + + [XPath("/a[2]")] + public string UserID { get; set; } + + [XPath("a[2]", "href")] + public string UserLink { get; set; } + + [XPath("/span[@class='reputation-score']")] + public string ReputationScore { get; set; } + } + + [HasXPath] + [DebuggerDisplay("Question Title={QuestionTitle}")] + public class HotNetworkQuestion + { + [XPath("/div", "title")] + public string QuestionCategory { get; set; } + + [XPath("/a")] + public string QuestionTitle { get; set; } + + [XPath("/a", "href")] + public string QuestionLink { get; set; } + } + } +} \ No newline at end of file diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlNode.Tests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlNode.Tests.cs new file mode 100644 index 00000000..f48faea8 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlNode.Tests.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace HtmlAgilityPack.Tests +{ + [TestFixture] + public class HtmlNode2 + { + //[Test(Description = + // "Attributes should maintain their original character casing if OptionOutputOriginalCase is true")] + //public void EnsureAttributeOriginalCaseIsPreserved() + //{ + // var html = "
"; + // var doc = new HtmlDocument + // { + // OptionOutputOriginalCase = true + // }; + // doc.LoadHtml(html); + // var div = doc.DocumentNode.Descendants("div").FirstOrDefault(); + // var writer = new StringWriter(); + // div.WriteAttributes(writer, false); + // var result = writer.GetStringBuilder().ToString(); + // Assert.AreEqual(" AttributeIsThis=\"val\"", result); + //} + + [Test] + public void ReadNotCloseTag() + { + var document = new HtmlDocument(); + document.LoadHtml("
  • item
"); + var span = document.DocumentNode.SelectSingleNode("//span"); + if (span == null) throw new Exception("Failed to find span element"); + var OuterHtml = span.OuterHtml; + var InnerHtml = span.InnerHtml; + var InnerText = span.InnerText; + + Assert.IsNotNull(OuterHtml); + Assert.IsNotNull(InnerHtml); + Assert.IsNotNull(InnerText); + } + + + [Test] + public void checkAttributForTextComment() + { + var doc = new HtmlAgilityPack.HtmlDocument(); + doc.LoadHtml(@"
some text
"); + var div = doc.GetElementbyId("foo"); + int count = 0; + Exception exception = null; + foreach (var textNode in div.ChildNodes) + { + try + { + textNode.Id = "1"; + count++; + } + catch (Exception e) + { + exception = e; + + } + } + + Assert.AreEqual(count, 1); + Assert.IsNotNull(exception); + } + + + [Test] + public void Prepend_CheckOrder() + { + HtmlNode source = HtmlNode.CreateNode(@" +
    +
  • Alpha
    • +
  • Bravo
    • +
  • Charlie
    • +
+"); + HtmlNode destination = HtmlNode.CreateNode(@" +
    +
  • Delta
    • +
  • Echo
    • +
  • Foxtrot
    • +
+"); + + destination.PrependChildren(source.ChildNodes); + + Assert.AreEqual(destination.WriteTo() + , @"
    +
  • Alpha
    • +
  • Bravo
    • +
  • Charlie
    • + +
  • Delta
    • +
  • Echo
    • +
  • Foxtrot
    • +
"); + } + } +} \ No newline at end of file diff --git a/src/Tests/HtmlAgilityPack.Tests/files/mshome.htm b/src/Tests/HtmlAgilityPack.Tests/files/mshome.htm new file mode 100644 index 00000000..a7115f57 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/files/mshome.htm @@ -0,0 +1,582 @@ + + + + +Microsoft Corporation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Microsoft Home + +  +  +All Products  +  +| +  +  +Support  +  +| +  +  +Search  +  +| +  +  +microsoft.com Home  +  + +
 
+  +  +Microsoft Home  +  +| +  +  +MSN Home  +  +| +  +  +Subscribe  +  +| +  +  +Manage Your Profile +
+ + + + + + + + + + +
+ + + + + + + + +
Search for
+ + +
Advanced Search
+ + +
+ + + + + + + + + + + + + + + + + + +
Product Families
+ +Windows +
+ +Office +
+ +Business Solutions +
+ +Servers +
+ +Developer Tools +
+ +Games and Xbox +
+ +MSN Services +
+ +All Products +
+
+ + +
+ + + + + + + + + + + + + + + + +
Resources
+ +Support +
+ +Downloads +
+ +Windows Update +
+ +Office Update +
+ +Learning Tools +
+ +Communities +
+ +Security +
+
+ + +
+ + + + + + + + + + + + + + + + +
Information For
+ +Home Users +
+ +IT Professionals (TechNet) +
+ +Developers (MSDN) +
+ +Microsoft Partners +
+ +Business Professionals +
+ +Educational Institutions +
+ +Journalists +
+
+ + +
+ + + + + + + + + + +
About Microsoft
+ +Corporate Information +
+ +Investor Relations +
+ +Careers +
+ +About this Site +
+
+ + +
+ + + + +
Worldwide
+ +Microsoft Worldwide +
+
+		 + + + + + + +
+ + +
Shoot for the stars
+ + + + + + + + + + + + + + + + + +
home & entertainment + + + + + + + + + + + + + +
home & entertainment
Download new transition elements for Movie Maker 2
Windows Movie Maker 2 for beginners and experts
Free stationery templates from Office
+
technical resources + + + + + + + + + + +
technical resources
Use C# and XML to display news feeds on your PC
Become an Exchange expert with a little help from your peers
Help improve security in a Windows 2000 Server environment
+
business agility + + + + + + + + + + + + + +
business agility
New for small businesses: Security Update e-mail alert
Download a calendar to track employee summer vacations
Discover Windows Server 2003
+
+ + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Get to know .NET from A to Z

Download the .NET Alerts SDK

Write mobile games using the .NET Compact Framework

+		 + + + + + + + + + + + + + +
Download the Digital Photography Fun Pack

The Windows XP Expert Zone Community is for everyone

Download MOM Management Packs

+		 + + + + + + + + + + + + + +
Translate your Word documents into different languages

Top 10 reasons to use Visio 2003

Use animation to add life to PowerPoint presentations

+		 +
More .NET ... +More Windows ... +More Office ... +

+ + +
Last Updated: Friday, May 30, 2003 - 3:05 p.m. Pacific Time
+

 + + + + + + + + + + + + + + + + + + + + + + + + + + + +
today's news
AOL Time Warner, Microsoft agree to collaborate

Microsoft announces Real-Time Communications Server 2003

Additional benefits coming for customers with Software Assurance

Do amazing things with our free Windows XP DVD (U.S. only)


More News ...
+
+ + + + + + + + + + + + + + + + + + + + + + +
downloads
PowerToys Fun Pack

DirectX 9.0a

Windows Server 2003 Evaluation Kit


More Downloads ...
+
+ + + + + + + + + + + + + + + + + + + + + +
support
Support for Office XP, 2000, and 97

Product Support Centers (FAQs)

365 Windows XP tips, just one click away


More Support ...
+
+
+ + + +
+ + Contact Us +| + Microsoft This Week! Newsletter +| + Legal + +
+ + ©2003 Microsoft Corporation. All rights reserved. + Terms of Use | +Privacy Statement | +Accessibility + +
+ + +.NET +Windows +Office + + + + diff --git a/src/Tests/HtmlAgilityPack.Tests/files/overflow.html b/src/Tests/HtmlAgilityPack.Tests/files/overflow.html new file mode 100644 index 00000000..29cf0db1 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/files/overflow.html @@ -0,0 +1,94 @@ +User list - XCC Forum + + + + + + + + + + + + + + + + +
X +
zzz
X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X + X + X + X + X +
X + X +X
X + X + X +
X + X + X + X + X + X +
diff --git a/src/Tests/HtmlAgilityPack.Tests/files/regression.html b/src/Tests/HtmlAgilityPack.Tests/files/regression.html new file mode 100644 index 00000000..2a198396 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/files/regression.html @@ -0,0 +1,175 @@ + + + +Steam Community :: Steam Web API Key + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+

Register Steam Web API Key

+
+
+ + +
+ +
+ +

Your Steam Web API Key

+

Key: Redacted

+

Domain Name: Redacted

+
+ + +
+ + +
+ + +
+
+
+
+
+
+ +
+ + + + + + + +
+ + diff --git a/src/Tests/HtmlAgilityPack.Tests/files/test.html b/src/Tests/HtmlAgilityPack.Tests/files/test.html new file mode 100644 index 00000000..d4924105 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/files/test.html @@ -0,0 +1,12 @@ + + + + + + +

This is bold heading

+

This is underlined paragraph

+

This is italic heading

+

This is new heading

+ + From 8d05430d441a1a4d6d42bb3de14cccbd75e5106e Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Fri, 15 Nov 2019 21:16:40 +1300 Subject: [PATCH 05/14] add test code generator add test code generator --- src/HtmlAgilityPack.sln | 25 ++ .../Generator/HtmlWebTestGenerator.cs | 223 ++++++++++++++++++ src/Tools/TestCodeGen/Program.cs | 79 +++++++ src/Tools/TestCodeGen/TestCode.cs | 10 + src/Tools/TestCodeGen/TestCodeGen.csproj | 15 ++ src/Tools/TestCodeGen/TestCodeGenerator.cs | 89 +++++++ 6 files changed, 441 insertions(+) create mode 100644 src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs create mode 100644 src/Tools/TestCodeGen/Program.cs create mode 100644 src/Tools/TestCodeGen/TestCode.cs create mode 100644 src/Tools/TestCodeGen/TestCodeGen.csproj create mode 100644 src/Tools/TestCodeGen/TestCodeGenerator.cs diff --git a/src/HtmlAgilityPack.sln b/src/HtmlAgilityPack.sln index bf2b51d9..da3c75c6 100644 --- a/src/HtmlAgilityPack.sln +++ b/src/HtmlAgilityPack.sln @@ -19,6 +19,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HtmlAgilityPack.Tests.Net45 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HtmlAgilityPack.Tests", "Tests\HtmlAgilityPack.Tests\HtmlAgilityPack.Tests.csproj", "{D4614863-964F-4333-B592-A4725DACC385}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{6459B875-8AB1-4E99-B088-472CA496F9B5}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestCodeGen", "Tools\TestCodeGen\TestCodeGen.csproj", "{EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution HtmlAgilityPack.Shared\HtmlAgilityPack.Shared.projitems*{1028002a-bbe2-4ff2-94b7-a8368a8f6887}*SharedItemsImports = 4 @@ -138,6 +142,26 @@ Global {D4614863-964F-4333-B592-A4725DACC385}.Release|x64.Build.0 = Release|Any CPU {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.ActiveCfg = Release|Any CPU {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.Build.0 = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|ARM.ActiveCfg = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|ARM.Build.0 = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x64.Build.0 = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x86.ActiveCfg = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x86.Build.0 = Debug|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Any CPU.Build.0 = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|ARM.ActiveCfg = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|ARM.Build.0 = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x64.ActiveCfg = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x64.Build.0 = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x86.ActiveCfg = Release|Any CPU + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -149,6 +173,7 @@ Global {5BA9A0D7-173F-4C4F-AFFA-3DD6DC2F9A79} = {4E506C9F-6514-4055-AAA8-DC89D85558FE} {F7D2CBBC-E23E-478E-865B-6BE445B55EC8} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} {D4614863-964F-4333-B592-A4725DACC385} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} + {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC} = {6459B875-8AB1-4E99-B088-472CA496F9B5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5870022-9E57-469B-A27E-E840C436B2EF} diff --git a/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs b/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs new file mode 100644 index 00000000..ff5b1db9 --- /dev/null +++ b/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using HtmlAgilityPack; + +namespace TestCodeGen.Generator +{ + public class HtmlWebTestGenerator : TestCodeGenerator + { + public override string MenuItemName => "Generate a test for HtmlWeb class mocking the web request."; + + #region Inner Classes + + class DummyException : Exception + { + } + + #endregion + + #region Parameter Loaders + + class UrlParamLoader : ParamLoader + { + public override string Message + => "Enter a URL"; + + public UrlParamLoader(TestCodeGenerator generator) : base(generator) { } + + public override void SetUp(string input) + { + try + { + ((HtmlWebTestGenerator)Generator).Url = new Uri(input); + } + catch (UriFormatException e) + { + throw new Exception("The entered URL was invalid.", e); + } + } + } + + class OutputContentParamLoader : ParamLoader + { + public override string Message + => "Does the test need response content? [Yes|No] (default: Yes)"; + + public OutputContentParamLoader(TestCodeGenerator generator) : base(generator) { } + + public override void SetUp(string input) + { + switch (input.Trim().ToLower()) + { + case "no": + case "n": + ((HtmlWebTestGenerator)Generator).OutputContent = false; + return; + } + ((HtmlWebTestGenerator)Generator).OutputContent = true; + } + } + + #endregion + + public Uri Url { get; private set; } + public bool OutputContent { get; private set; } + + public HtmlWebTestGenerator() + { + _paramLoaders.Add(new OutputContentParamLoader(this)); + _paramLoaders.Add(new UrlParamLoader(this)); + } + + public override IEnumerable Generate() + { + var code4StatusCode = ""; + var code4ContentType = ""; + var code4ContentEncoding = ""; + var code4Headers = ""; + var code4Content = ""; + string content = null; + + var htmlWeb = new HtmlWeb(); + htmlWeb.PostResponse += (HttpWebRequest req, HttpWebResponse res) => + { + // generate code for mocking IHttpWebResponse.StatusCode property + code4StatusCode = $@"{Indent(6)}resMock.Setup(x => x.StatusCode).Returns({typeof(HttpStatusCode).Name}.{res.StatusCode.ToString()});"; + + // generate code for mocking IHttpWebResponse.ContentType property + code4ContentType = $@"{Indent(6)}resMock.Setup(x => x.ContentType).Returns(""{res.ContentType}"");"; + + // generate code for mocking IHttpWebResponse.ContentEncoding property + code4ContentEncoding = $@"{Indent(6)}resMock.Setup(x => x.ContentEncoding).Returns(""{res.ContentEncoding}"");"; + + // generate code for mocking IHttpWebResponse.Headers property + var headers = res.Headers; + code4Headers = headers.AllKeys + .Select(k => $"{Indent(7)}headers.Add(\"" + k + "\", \"" + headers[k].Replace("\"", "\\\"") + "\");") + .DefaultIfEmpty() + .Aggregate((a, b) => $"{a}{Environment.NewLine}{b}"); + + code4Headers = $@"{Indent(6)}resMock.Setup(x => x.Headers).Returns(() => +{Indent(6)}{{ +{Indent(7)}var headers = new WebHeaderCollection(); +{code4Headers} +{Indent(7)}return headers; +{Indent(6)}}});"; + + // generate code for mocking IHttpWebResponse.GetResponseStream() method + if (OutputContent) + { + using (var s = res.GetResponseStream()) + { + Encoding encoding; + try + { + encoding = Encoding.GetEncoding(res.ContentEncoding); + } + catch (ArgumentException) + { + encoding = Encoding.UTF8; + } + var r = new StreamReader(s, encoding); + content = r.ReadToEnd(); + } + + code4Content = $@"{Indent(6)}resMock.Setup(x => x.GetResponseStream()) +{Indent(7)}.Returns(() => new FileStream( +{Indent(8)}Path.Combine(_contentDir, ""{TestName}.html""), +{Indent(8)}FileMode.Open, +{Indent(8)}FileAccess.Read));"; + } + else + { + code4Content = $@"{Indent(6)}resMock.Setup(x => x.GetResponseStream()).Returns( +{Indent(7)}() => new MemoryStream());"; + } + + throw new DummyException(); + }; + + try + { + htmlWeb.Load(Url); + } + catch (DummyException) + { + } + + var regexNewLine = new Regex("(\n\r?)|(\r\n)"); + yield return new TestCode + { + File = new FileInfo(Path.Combine(OutDir.ToString(), TestName + ".cs")), + Content = regexNewLine.Replace($@"using System; +using System.IO; +using System.Net; +using Moq; +using NUnit.Framework; + +namespace HtmlAgilityPack.Tests +{{ +{Indent(1)}[TestFixture] +{Indent(1)}class {TestName} +{Indent(1)}{{ +{Indent(2)}private string _contentDir; + +{Indent(2)}[OneTimeSetUp] +{Indent(2)}public void Setup() +{Indent(2)}{{ +{Indent(3)}_contentDir = Path.Combine( +{Indent(4)}Path.GetDirectoryName(typeof(HtmlDocumentTests).Assembly.Location), +{Indent(4)}""files""); +{Indent(2)}}} + +{Indent(2)}[Test] +{Indent(2)}public void Test() +{Indent(2)}{{ +{Indent(3)}var factoryMock = new Mock(); +{Indent(3)}factoryMock.Setup(x => x.Create(It.IsAny())) +{Indent(4)}.Returns(u => +{Indent(4)}{{ +{Indent(5)}var reqMock = new Mock(); +{Indent(5)}reqMock.Setup(x => x.Request).Returns(null as HttpWebRequest); +{Indent(5)}reqMock.Setup(x => x.GetResponse()).Returns(() => +{Indent(5)}{{ +{Indent(6)}var resMock = new Mock(); +{Indent(6)}resMock.Setup(x => x.ResponseUri).Returns(u); +{code4StatusCode} +{code4ContentType} +{code4ContentEncoding} +{code4Headers} +{code4Content} +{Indent(6)}resMock.Setup(x => x.LastModified).Returns(DateTime.UtcNow); +{Indent(6)}return resMock.Object; +{Indent(5)}}}); +{Indent(5)}return reqMock.Object; +{Indent(4)}}}); + +{Indent(3)}var htmlWeb = new HtmlWeb(factoryMock.Object); +{Indent(3)}htmlWeb.Load(new Uri(""{Url.OriginalString}"")); +{Indent(2)}}} +{Indent(1)}}} +}} +", Environment.NewLine), + }; + + if (OutputContent) + yield return new TestCode + { + File = new FileInfo(Path.Combine(OutDir.ToString(), "files", TestName + ".html")), + Content = content, + }; + } + + public string Indent(int count) + { + return new string(' ', 4 * count); + } + } +} diff --git a/src/Tools/TestCodeGen/Program.cs b/src/Tools/TestCodeGen/Program.cs new file mode 100644 index 00000000..ea1b9827 --- /dev/null +++ b/src/Tools/TestCodeGen/Program.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using System.Linq; +using TestCodeGen.Generator; + +namespace TestCodeGen +{ + class Program + { + static void Main(string[] args) + { + DirectoryInfo outDir = null; + Action paramAction = null; + foreach (var arg in args) + { + switch (arg) + { + case "-t": + paramAction = a => + { + outDir = new DirectoryInfo(a); + }; + break; + default: + paramAction?.Invoke(arg); + paramAction = null; + break; + } + } + + outDir = outDir ?? new DirectoryInfo(Directory.GetCurrentDirectory()); + + var generators = new TestCodeGenerator[] + { + new HtmlWebTestGenerator { OutDir = outDir }, + }; + + Console.WriteLine("Html Agile Pack Test Code Generator"); + Console.WriteLine(""); + for (var i = 0; i < generators.Length; i++) + { + Console.WriteLine($"{i}: {generators[i].MenuItemName}"); + } + Console.WriteLine(""); + Console.Write("Which do you want? Select a menu item by number : "); + var itemIndex = Console.ReadLine(); + if (!int.TryParse(itemIndex, out var index) + || index >= generators.Length) + { + Console.WriteLine("[ERROR] Your choice was invalid."); + return; + } + + try + { + foreach (var paramLoader in generators[index].ParamLoaders) + { + Console.Write($"{paramLoader.Message} : "); + var input = Console.ReadLine(); + paramLoader.SetUp(input); + } + } + catch (Exception e) + { + Console.WriteLine($"[ERROR] {e.Message}"); + return; + } + + var files = generators[index].GenerateAndSaveToFile().ToList(); + + Console.WriteLine("generating test code..."); + Console.WriteLine($"Test code was successfully saved to:"); + foreach (var file in files) + { + Console.WriteLine(" " + file.FullName); + } + } + } +} diff --git a/src/Tools/TestCodeGen/TestCode.cs b/src/Tools/TestCodeGen/TestCode.cs new file mode 100644 index 00000000..0196ff07 --- /dev/null +++ b/src/Tools/TestCodeGen/TestCode.cs @@ -0,0 +1,10 @@ +using System.IO; + +namespace TestCodeGen +{ + public class TestCode + { + public FileInfo File { get; set; } + public string Content { get; set; } + } +} \ No newline at end of file diff --git a/src/Tools/TestCodeGen/TestCodeGen.csproj b/src/Tools/TestCodeGen/TestCodeGen.csproj new file mode 100644 index 00000000..af3ef696 --- /dev/null +++ b/src/Tools/TestCodeGen/TestCodeGen.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp3.0 + + + + + + + + + + diff --git a/src/Tools/TestCodeGen/TestCodeGenerator.cs b/src/Tools/TestCodeGen/TestCodeGenerator.cs new file mode 100644 index 00000000..f6ad2a58 --- /dev/null +++ b/src/Tools/TestCodeGen/TestCodeGenerator.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace TestCodeGen +{ + public abstract class TestCodeGenerator + { + #region Inner Classes + + public abstract class ParamLoader + { + protected TestCodeGenerator Generator { get; private set; } + + public ParamLoader(TestCodeGenerator generator) + { + Generator = generator; + } + public abstract string Message { get; } + public abstract void SetUp(string input); + } + + #endregion + + #region Parameter Loaders + + class OutDirParamLoader : ParamLoader + { + public override string Message + => $"Where do you save the test code? (default: {Generator.OutDir.FullName})"; + + public OutDirParamLoader(TestCodeGenerator generator) : base(generator) { } + + public override void SetUp(string input) + { + if (!string.IsNullOrEmpty(input)) + Generator.OutDir = new DirectoryInfo(input); + } + } + + class TestNameParamLoader : ParamLoader + { + public override string Message + => "Enter test name"; + + public TestNameParamLoader(TestCodeGenerator generator) : base(generator) { } + + public override void SetUp(string input) + { + if (string.IsNullOrEmpty(input)) + throw new ArgumentException("Cannot proceed without a test name."); + Generator.TestName = input; + } + } + + #endregion + + public abstract string MenuItemName { get; } + protected List _paramLoaders { get; } + public IReadOnlyList ParamLoaders => _paramLoaders.AsReadOnly(); + public DirectoryInfo OutDir { get; set; } + public string TestName { get; private set; } + + public TestCodeGenerator() + { + _paramLoaders = new List(); + _paramLoaders.Add(new OutDirParamLoader(this)); + _paramLoaders.Add(new TestNameParamLoader(this)); + } + public abstract IEnumerable Generate(); + + public virtual IEnumerable GenerateAndSaveToFile() + { + var testCodes = Generate(); + + foreach (var testCode in testCodes) + { + if (!testCode.File.Directory.Exists) + testCode.File.Directory.Create(); + using (var s = new FileStream(testCode.File.ToString(), FileMode.Create, FileAccess.Write)) + using (var w = new StreamWriter(s)) + { + w.WriteLine(testCode.Content); + } + yield return testCode.File; + } + } + } +} From 10fcb6f1f1b21305b60f501e2d6b5cf6f3fb1579 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Fri, 15 Nov 2019 21:20:57 +1300 Subject: [PATCH 06/14] add test code for HtmlWeb.Load method add test code for HtmlWeb.Load method --- .../HtmlAgilityPack.Tests/HtmlWebTests.cs | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs new file mode 100644 index 00000000..ea8a6e57 --- /dev/null +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs @@ -0,0 +1,137 @@ +using System; +using System.IO; +using System.Net; +using Moq; +using NUnit.Framework; + +namespace HtmlAgilityPack.Tests +{ + [TestFixture] + class HtmlWebTests + { + + [Test] + public void TestLoad() + { + var factoryMock = new Mock(); + factoryMock.Setup(x => x.Create(It.IsAny())) + .Returns(u => + { + var reqMock = new Mock(); + reqMock.Setup(x => x.Request).Returns(null as HttpWebRequest); + reqMock.Setup(x => x.GetResponse()).Returns(() => + { + var resMock = new Mock(); + resMock.Setup(x => x.ResponseUri).Returns(u); + resMock.Setup(x => x.StatusCode).Returns(HttpStatusCode.OK); + resMock.Setup(x => x.ContentType).Returns("text/html; charset=utf-8"); + resMock.Setup(x => x.ContentEncoding).Returns(""); + resMock.Setup(x => x.Headers).Returns(() => + { + var headers = new WebHeaderCollection(); + headers.Add("Cache-Control", "no-store, no-transform, no-cache"); + headers.Add("Pragma", "no-cache"); + headers.Add("X-Activity-Id", "e9a8a0b4-7354-4136-a6fc-befe822aa602"); + headers.Add("MS-CV", "SkBSzuT59U+wjAq7.0"); + headers.Add("X-AppVersion", "1.0.7257.410"); + headers.Add("X-Az", "{did:92e7dc58ca2143cfb2c818b047cc5cd1, rid: OneDeployContainer, sn: marketingsites-prod-odeastasia, dt: 2018-05-03T20:14:23.4188992Z, bt: 2019-11-14T08:13:40.0000000Z}"); + headers.Add("ms-operation-id", "48df488ad0464c48a48b7c897ca7bc22"); + headers.Add("P3P", "CP=\"CAO CONi OTR OUR DEM ONL\""); + headers.Add("X-UA-Compatible", "IE=Edge;chrome=1"); + headers.Add("X-Content-Type-Options", "nosniff"); + headers.Add("X-Frame-Options", "SAMEORIGIN"); + headers.Add("Access-Control-Allow-Methods", "HEAD,GET,POST,PATCH,PUT,OPTIONS"); + headers.Add("X-XSS-Protection", "1; mode=block"); + headers.Add("X-EdgeConnect-MidMile-RTT", "43"); + headers.Add("X-EdgeConnect-Origin-MEX-Latency", "210"); + headers.Add("Date", "Fri, 15 Nov 2019 08:05:57 GMT"); + headers.Add("Transfer-Encoding", "chunked"); + headers.Add("Connection", "keep-alive, Transfer-Encoding"); + headers.Add("Set-Cookie", "isFirstSession=1; path=/; secure; HttpOnly, MUID=3390FF8D527B6A5D0528F19853B36BE3; domain=.microsoft.com; expires=Mon, 15-Nov-2021 08:05:57 GMT; path=/;SameSite=None; secure, X-FD-FEATURES=ids=1690c%2csfwaab%2catperf680c2%2c1786t1%2c969t1%2c882c%2c936t1a%2ctasmigration010%2ccartemberpl&imp=e9a8a0b4-7354-4136-a6fc-befe822aa602; expires=Sun, 15-Nov-2020 08:05:57 GMT; path=/; secure; HttpOnly, X-FD-Time=1; expires=Fri, 15-Nov-2019 08:10:57 GMT; path=/; secure; HttpOnly, akacd_OneRF=1581581157~rv=58~id=033ad52aa4add00d9694cf6d729f8b05; path=/; Expires=Thu, 13 Feb 2020 08:05:57 GMT, akacd_OneRF=1581581157~rv=58~id=033ad52aa4add00d9694cf6d729f8b05; path=/; Expires=Thu, 13 Feb 2020 08:05:57 GMT"); + headers.Add("TLS_version", "tls1.2"); + headers.Add("Strict-Transport-Security", "max-age=31536000"); + headers.Add("X-RTag", "RT"); + headers.Add("Content-Type", "text/html; charset=utf-8"); + headers.Add("Expires", "-1"); + return headers; + }); + resMock.Setup(x => x.GetResponseStream()).Returns( + () => new MemoryStream()); + resMock.Setup(x => x.LastModified).Returns(DateTime.UtcNow); + return resMock.Object; + }); + return reqMock.Object; + }); + + var htmlWeb = new HtmlWeb(factoryMock.Object); + htmlWeb.Load(new Uri("https://www.microsoft.com/en-nz")); + } + + [Test] + public void TestLoadThrowsEncodingNotSupportedException() + { + var factoryMock = new Mock(); + factoryMock.Setup(x => x.Create(It.IsAny())) + .Returns(u => + { + var reqMock = new Mock(); + reqMock.Setup(x => x.Request).Returns(null as HttpWebRequest); + reqMock.Setup(x => x.GetResponse()).Returns(() => + { + var resMock = new Mock(); + resMock.Setup(x => x.ResponseUri).Returns(u); + resMock.Setup(x => x.StatusCode).Returns(HttpStatusCode.OK); + resMock.Setup(x => x.ContentType).Returns("text/html; charset=UTF-8"); + resMock.Setup(x => x.ContentEncoding).Returns("identity"); + resMock.Setup(x => x.Headers).Returns(() => + { + var headers = new WebHeaderCollection(); + headers.Add("Transfer-Encoding", "chunked"); + headers.Add("Connection", "keep-alive"); + headers.Add("X-Cache-Hits", "4"); + headers.Add("Cache-Control", "no-store, must-revalidate, no-cache, max-age=0"); + headers.Add("X-Chef", "Gennaro"); + headers.Add("Pragma", "public"); + headers.Add("X-Cache-Keep", "3600.000"); + headers.Add("Vary", "Accept-Encoding"); + headers.Add("X-Country-Code", "NZ"); + headers.Add("X-Cache-Status", "MISS"); + headers.Add("X-Backend", "i_04400f0cd28e027e7_10_170_21_65"); + headers.Add("Date", "Fri, 15 Nov 2019 07:44:42 GMT"); + headers.Add("X-Cache-TTL-Remaining", "595.653"); + headers.Add("Server", "ZENEDGE"); + headers.Add("X-Zen-Fury", "efadaddb4a91c6f51faaa477a91c047faa84f52d"); + headers.Add("X-Cache", "FoodCache"); + headers.Add("Accept-Ranges", "bytes"); + headers.Add("Age", "3004"); + headers.Add("X-Cache-Age", "3004.347"); + headers.Add("X-Cdn", "Served-By-Zenedge"); + headers.Add("Content-Type", "text/html; charset=UTF-8"); + headers.Add("Last-Modified", "Thu, 01 Jan 1970 00:00:00 GMT"); + headers.Add("Expires", "Fri, 15 Nov 2019 08:44:41 GMT"); + headers.Add("Content-Encoding", "identity"); + return headers; + }); + resMock.Setup(x => x.GetResponseStream()).Returns( + () => new MemoryStream()); + resMock.Setup(x => x.LastModified).Returns(DateTime.UtcNow); + return resMock.Object; + }); + return reqMock.Object; + }); + + var htmlWeb = new HtmlWeb(factoryMock.Object); + Exception exception = null; + try + { + htmlWeb.Load(new Uri("https://www.jamieoliver.com/recipes/chicken-recipes/chicken-tofu-noodle-soup/")); + } + catch (Exception e) + { + exception = e; + } + Assert.That(exception, Is.InstanceOf(typeof(EncodingNotSupportedException))); + } + } +} + From b1d74b03b52cfa1a5820ca546d14fbf04e9d53fe Mon Sep 17 00:00:00 2001 From: Yas Date: Fri, 15 Nov 2019 21:44:02 +1300 Subject: [PATCH 07/14] refactor tests to run also in macOS --- src/HtmlAgilityPack.sln | 90 +++++++++---------- .../HtmlDocumentTests.cs | 14 +-- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/HtmlAgilityPack.sln b/src/HtmlAgilityPack.sln index da3c75c6..119a9b8f 100644 --- a/src/HtmlAgilityPack.sln +++ b/src/HtmlAgilityPack.sln @@ -17,11 +17,11 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "HtmlAgilityPack.Shared", "H EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HtmlAgilityPack.Tests.Net45", "Tests\HtmlAgilityPack.Tests.Net45\HtmlAgilityPack.Tests.Net45.csproj", "{F7D2CBBC-E23E-478E-865B-6BE445B55EC8}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HtmlAgilityPack.Tests", "Tests\HtmlAgilityPack.Tests\HtmlAgilityPack.Tests.csproj", "{D4614863-964F-4333-B592-A4725DACC385}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{6459B875-8AB1-4E99-B088-472CA496F9B5}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestCodeGen", "Tools\TestCodeGen\TestCodeGen.csproj", "{EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HtmlAgilityPack.Tests", "Tests\HtmlAgilityPack.Tests\HtmlAgilityPack.Tests.csproj", "{23944379-89E1-4D96-9B26-E29EC516D0CF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestCodeGen", "Tools\TestCodeGen\TestCodeGen.csproj", "{C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}" EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution @@ -122,46 +122,46 @@ Global {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x64.Build.0 = Release|Any CPU {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x86.ActiveCfg = Release|Any CPU {F7D2CBBC-E23E-478E-865B-6BE445B55EC8}.Release|x86.Build.0 = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|ARM.ActiveCfg = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|ARM.Build.0 = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|x64.ActiveCfg = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|x64.Build.0 = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|x86.ActiveCfg = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Debug|x86.Build.0 = Debug|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|Any CPU.Build.0 = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|ARM.ActiveCfg = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|ARM.Build.0 = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|x64.ActiveCfg = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|x64.Build.0 = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.ActiveCfg = Release|Any CPU - {D4614863-964F-4333-B592-A4725DACC385}.Release|x86.Build.0 = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|ARM.ActiveCfg = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|ARM.Build.0 = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x64.ActiveCfg = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x64.Build.0 = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x86.ActiveCfg = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Debug|x86.Build.0 = Debug|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Any CPU.Build.0 = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|ARM.ActiveCfg = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|ARM.Build.0 = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x64.ActiveCfg = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x64.Build.0 = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x86.ActiveCfg = Release|Any CPU - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC}.Release|x86.Build.0 = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|ARM.ActiveCfg = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|ARM.Build.0 = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|x64.ActiveCfg = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|x64.Build.0 = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|x86.ActiveCfg = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Debug|x86.Build.0 = Debug|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|Any CPU.Build.0 = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|ARM.ActiveCfg = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|ARM.Build.0 = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|x64.ActiveCfg = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|x64.Build.0 = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|x86.ActiveCfg = Release|Any CPU + {23944379-89E1-4D96-9B26-E29EC516D0CF}.Release|x86.Build.0 = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|ARM.ActiveCfg = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|ARM.Build.0 = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|x64.ActiveCfg = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|x64.Build.0 = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|x86.ActiveCfg = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Debug|x86.Build.0 = Debug|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|Any CPU.Build.0 = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|ARM.ActiveCfg = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|ARM.Build.0 = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|x64.ActiveCfg = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|x64.Build.0 = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|x86.ActiveCfg = Release|Any CPU + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -172,8 +172,8 @@ Global {083D99CD-F083-4122-AA96-4E3A84F3EC01} = {4E506C9F-6514-4055-AAA8-DC89D85558FE} {5BA9A0D7-173F-4C4F-AFFA-3DD6DC2F9A79} = {4E506C9F-6514-4055-AAA8-DC89D85558FE} {F7D2CBBC-E23E-478E-865B-6BE445B55EC8} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} - {D4614863-964F-4333-B592-A4725DACC385} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} - {EF2BE777-AF14-484A-AF7F-AD4DA99B48BC} = {6459B875-8AB1-4E99-B088-472CA496F9B5} + {23944379-89E1-4D96-9B26-E29EC516D0CF} = {FBD7AB4B-A0C8-4EE0-B1A9-6F26223097E9} + {C9C8CEA1-6C48-4A48-913D-8DB1A847FEDC} = {6459B875-8AB1-4E99-B088-472CA496F9B5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E5870022-9E57-469B-A27E-E840C436B2EF} diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs index 8a615591..3b751232 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlDocumentTests.cs @@ -22,14 +22,14 @@ public void Setup() { - _contentDirectory = Path.GetDirectoryName(typeof(HtmlDocumentTests).Assembly.Location).ToString() + "\\files\\"; + _contentDirectory = Path.Combine(Path.GetDirectoryName(typeof(HtmlDocumentTests).Assembly.Location).ToString(), "files"); // _contentDirectory = Path.Combine(@"C:\Users\Jonathan\Desktop\Z\zzzproject\HtmlAgilityPack\HtmlAgilityPack.Tests\bin\Debug\files\"); } private HtmlDocument GetMshomeDocument() { var doc = new HtmlDocument(); - doc.Load(_contentDirectory + "mshome.htm"); + doc.Load(Path.Combine(_contentDirectory, "mshome.htm")); return doc; } @@ -246,10 +246,10 @@ public void TestParseSaveParse() var doc = GetMshomeDocument(); var doc1desc = doc.DocumentNode.Descendants().Where(x => !string.IsNullOrWhiteSpace(x.InnerText)).ToList(); - doc.Save(_contentDirectory + "testsaveparse.html"); + doc.Save(Path.Combine(_contentDirectory, "testsaveparse.html")); var doc2 = new HtmlDocument(); - doc2.Load(_contentDirectory + "testsaveparse.html"); + doc2.Load(Path.Combine(_contentDirectory, "testsaveparse.html")); var doc2desc = doc2.DocumentNode.Descendants().Where(x => !string.IsNullOrWhiteSpace(x.InnerText)).ToList(); Assert.AreEqual(doc1desc.Count, doc2desc.Count); @@ -472,7 +472,7 @@ public void TestDetectEncoding() [Test] public void TestLoadWithCache() { - var dir = _contentDirectory + "cache"; + var dir = Path.Combine(_contentDirectory, "cache"); Directory.CreateDirectory(dir); var web = new HtmlAgilityPack.HtmlWeb() @@ -837,7 +837,7 @@ public void GetEncapsulatedData() public void CompareLowerCulture() { - string html = File.ReadAllText(_contentDirectory + "regression.html"); + string html = File.ReadAllText(Path.Combine(_contentDirectory, "regression.html")); HtmlNode node1 = null; // Test 1 CultureInfo cul1 = CultureInfo.CreateSpecificCulture("en-US"); @@ -866,7 +866,7 @@ public void CompareLowerCulture() public void OverFlowNotEndTag() { - string html = File.ReadAllText(_contentDirectory + "overflow.html"); + string html = File.ReadAllText(Path.Combine(_contentDirectory, "overflow.html")); HtmlNode node1 = null; // Test 1 From e37d1663195e012d353a3b3eceffddc8ba646ae3 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sat, 16 Nov 2019 09:57:57 +1300 Subject: [PATCH 08/14] modify test project target to core 2.1 It turned out NUnit tests are not running with core 2.0. Therefore, changed test project target to core 2.1 so that we can run tests over .NET Core. --- .../HtmlAgilityPack.Tests.csproj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj index 9930263e..c74b7a1e 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj @@ -1,7 +1,7 @@  - net40;net45;netcoreapp2.0 + net40;net45;netcoreapp2.1 HtmlAgilityPack.Tests HtmlAgilityPack.Tests 1.0.0 @@ -21,7 +21,7 @@ - + @@ -34,20 +34,21 @@ + - + - + - + From 66f70753b18f4d13880c98a3f72d872a5b642c47 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 07:40:09 +1300 Subject: [PATCH 09/14] add documentation comments --- src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs | 11 ++++++++++- src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs | 11 ++++++++++- src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs | 11 ++++++++++- src/HtmlAgilityPack.Shared/IHttpWebRequest.cs | 11 ++++++++++- src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs | 11 ++++++++++- src/HtmlAgilityPack.Shared/IHttpWebResponse.cs | 11 ++++++++++- .../HtmlAgilityPack.Tests.csproj | 2 -- src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs | 1 - 8 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs index aec99922..4797dd02 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs @@ -1,8 +1,17 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + using System.Net; namespace HtmlAgilityPack { + /// + /// Implement the initialization of HttpWebRequest. + /// internal class HttpWebRequestFactory : IHttpWebRequestFactory { public HttpWebRequestFactory() diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs index 7096cc5b..00656766 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs @@ -1,8 +1,17 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + using System.Net; namespace HtmlAgilityPack { + /// + /// Wraps HttpWebRequest. + /// internal class HttpWebRequestWrapper : IHttpWebRequest { HttpWebRequest _request; diff --git a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs index 9e3d946e..bc67b544 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs @@ -1,9 +1,18 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + using System.IO; using System.Net; namespace HtmlAgilityPack { + /// + /// Wraps HttpWebResponse. + /// internal class HttpWebResponseWrapper : IHttpWebResponse { HttpWebResponse _response; diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs index 9f983e81..0bb05b4e 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs @@ -1,9 +1,18 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + using System.Net; namespace HtmlAgilityPack { internal interface IHttpWebRequest + /// + /// Abstracts HttpWebRequest. + /// { HttpWebRequest Request { get; } string Method { get; set; } diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs index 13389352..50b58cba 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs @@ -1,7 +1,16 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + namespace HtmlAgilityPack { + /// + /// Abstracts the initialization of HttpWebRequest. + /// internal interface IHttpWebRequestFactory { IHttpWebRequest Create(Uri uri); diff --git a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs index b778123f..f62a4d72 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs @@ -1,9 +1,18 @@ -using System; +// Description: Html Agility Pack - HTML Parsers, selectors, traversors, manupulators. +// Website & Documentation: http://html-agility-pack.net +// Forum & Issues: https://github.com/zzzprojects/html-agility-pack +// License: https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE +// More projects: http://www.zzzprojects.com/ +// Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. + using System.IO; using System.Net; namespace HtmlAgilityPack { + /// + /// Abstracts HttpWebResponse. + /// internal interface IHttpWebResponse : IDisposable { HttpWebResponse Response { get; } diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj index c74b7a1e..c39d5c4e 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlAgilityPack.Tests.csproj @@ -38,12 +38,10 @@ - - diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs index ea8a6e57..6bd7a06b 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs @@ -9,7 +9,6 @@ namespace HtmlAgilityPack.Tests [TestFixture] class HtmlWebTests { - [Test] public void TestLoad() { From e183a2010f46d4943bde43ba1c9879b989b073b4 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 08:09:10 +1300 Subject: [PATCH 10/14] add lacking using statements --- src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs | 1 + src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs | 1 + src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs | 1 + src/HtmlAgilityPack.Shared/IHttpWebRequest.cs | 1 + src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs | 2 ++ src/HtmlAgilityPack.Shared/IHttpWebResponse.cs | 1 + 6 files changed, 7 insertions(+) diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs index 4797dd02..b4f99a66 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; using System.Net; namespace HtmlAgilityPack diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs index 00656766..7ff5df9f 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; using System.Net; namespace HtmlAgilityPack diff --git a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs index bc67b544..4f302ac2 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; using System.IO; using System.Net; diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs index 0bb05b4e..cc0846ef 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; using System.Net; namespace HtmlAgilityPack diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs index 50b58cba..ea3cb4c1 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs @@ -6,6 +6,8 @@ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; + namespace HtmlAgilityPack { /// diff --git a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs index f62a4d72..8ac59544 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +using System; using System.IO; using System.Net; From df6521142f4dd5c538384502e747d2cdaeff0af9 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 08:30:07 +1300 Subject: [PATCH 11/14] fix for build in HtmlAgilityPack.Master solution --- src/HtmlAgilityPack.Shared/HtmlWeb.cs | 4 ++++ src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs | 4 ++++ src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs | 4 ++++ src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs | 4 ++++ src/HtmlAgilityPack.Shared/IHttpWebRequest.cs | 4 ++++ src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs | 3 +++ src/HtmlAgilityPack.Shared/IHttpWebResponse.cs | 4 ++++ 7 files changed, 27 insertions(+) diff --git a/src/HtmlAgilityPack.Shared/HtmlWeb.cs b/src/HtmlAgilityPack.Shared/HtmlWeb.cs index 22d9d484..291e67ec 100644 --- a/src/HtmlAgilityPack.Shared/HtmlWeb.cs +++ b/src/HtmlAgilityPack.Shared/HtmlWeb.cs @@ -83,7 +83,9 @@ public partial class HtmlWeb #region Fields +#if !(NETSTANDARD1_3 || NETSTANDARD1_6) private IHttpWebRequestFactory _requestFactory; +#endif private bool _autoDetectEncoding = true; private bool _cacheOnly; @@ -932,6 +934,7 @@ public bool UsingCache #region Constructors +#if !(NETSTANDARD1_3 || NETSTANDARD1_6) public HtmlWeb() : this(new HttpWebRequestFactory()) { } @@ -940,6 +943,7 @@ internal HtmlWeb(IHttpWebRequestFactory requestFactory) { _requestFactory = requestFactory; } +#endif #endregion diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs index b4f99a66..db6b4cae 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestFactory.cs @@ -5,6 +5,8 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) + using System; using System.Net; @@ -25,3 +27,5 @@ public IHttpWebRequest Create(Uri uri) } } } + +#endif diff --git a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs index 7ff5df9f..cc8f5b82 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebRequestWrapper.cs @@ -5,6 +5,8 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) + using System; using System.Net; @@ -38,3 +40,5 @@ public IHttpWebResponse GetResponse() } } } + +#endif diff --git a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs index 4f302ac2..c3847056 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs @@ -5,6 +5,8 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) + using System; using System.IO; using System.Net; @@ -49,3 +51,5 @@ public Stream GetResponseStream() } } } + +#endif diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs index cc0846ef..3de4a47f 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequest.cs @@ -5,6 +5,8 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) + using System; using System.Net; @@ -28,3 +30,5 @@ internal interface IHttpWebRequest IHttpWebResponse GetResponse(); } } + +#endif diff --git a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs index ea3cb4c1..6f0690df 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebRequestFactory.cs @@ -5,6 +5,7 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) using System; @@ -18,3 +19,5 @@ internal interface IHttpWebRequestFactory IHttpWebRequest Create(Uri uri); } } + +#endif diff --git a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs index 8ac59544..0ae0f2e2 100644 --- a/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs +++ b/src/HtmlAgilityPack.Shared/IHttpWebResponse.cs @@ -5,6 +5,8 @@ // More projects: http://www.zzzprojects.com/ // Copyright © ZZZ Projects Inc. 2014 - 2017. All rights reserved. +#if !(NETSTANDARD1_3 || NETSTANDARD1_6 || METRO) + using System; using System.IO; using System.Net; @@ -28,3 +30,5 @@ internal interface IHttpWebResponse : IDisposable void Close(); } } + +#endif From ea6c29f1bddac2b98f47a05af4d7545790af600b Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 09:27:50 +1300 Subject: [PATCH 12/14] add mshome.html file load to mock in HtmlWebTests.TestLoad add mshome.html file load to mock in HtmlWebTests.TestLoad. modify how to get assembly location in generated test code in HtmlWebTestGenerator. --- .../HtmlAgilityPack.Tests/HtmlWebTests.cs | 36 +++++++++++++------ .../Generator/HtmlWebTestGenerator.cs | 7 ++-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs index 6bd7a06b..d53da7fc 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs @@ -1,6 +1,8 @@ using System; using System.IO; +using System.Linq; using System.Net; +using System.Reflection; using Moq; using NUnit.Framework; @@ -9,6 +11,16 @@ namespace HtmlAgilityPack.Tests [TestFixture] class HtmlWebTests { + private string _contentDir; + + [OneTimeSetUp] + public void Setup() + { + _contentDir = Path.Combine( + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + "files"); + } + [Test] public void TestLoad() { @@ -30,23 +42,23 @@ public void TestLoad() var headers = new WebHeaderCollection(); headers.Add("Cache-Control", "no-store, no-transform, no-cache"); headers.Add("Pragma", "no-cache"); - headers.Add("X-Activity-Id", "e9a8a0b4-7354-4136-a6fc-befe822aa602"); - headers.Add("MS-CV", "SkBSzuT59U+wjAq7.0"); + headers.Add("X-Activity-Id", "ec46db78-0cbf-40c4-97a2-a2420cd8e1ca"); + headers.Add("MS-CV", "wjoFJw0sEE+jo45O.0"); headers.Add("X-AppVersion", "1.0.7257.410"); headers.Add("X-Az", "{did:92e7dc58ca2143cfb2c818b047cc5cd1, rid: OneDeployContainer, sn: marketingsites-prod-odeastasia, dt: 2018-05-03T20:14:23.4188992Z, bt: 2019-11-14T08:13:40.0000000Z}"); - headers.Add("ms-operation-id", "48df488ad0464c48a48b7c897ca7bc22"); + headers.Add("ms-operation-id", "1be30902be694b4c86c5a401f0bb91fb"); headers.Add("P3P", "CP=\"CAO CONi OTR OUR DEM ONL\""); headers.Add("X-UA-Compatible", "IE=Edge;chrome=1"); headers.Add("X-Content-Type-Options", "nosniff"); headers.Add("X-Frame-Options", "SAMEORIGIN"); headers.Add("Access-Control-Allow-Methods", "HEAD,GET,POST,PATCH,PUT,OPTIONS"); headers.Add("X-XSS-Protection", "1; mode=block"); - headers.Add("X-EdgeConnect-MidMile-RTT", "43"); - headers.Add("X-EdgeConnect-Origin-MEX-Latency", "210"); - headers.Add("Date", "Fri, 15 Nov 2019 08:05:57 GMT"); + headers.Add("X-EdgeConnect-MidMile-RTT", "41"); + headers.Add("X-EdgeConnect-Origin-MEX-Latency", "282"); + headers.Add("Date", "Sat, 16 Nov 2019 20:09:49 GMT"); headers.Add("Transfer-Encoding", "chunked"); headers.Add("Connection", "keep-alive, Transfer-Encoding"); - headers.Add("Set-Cookie", "isFirstSession=1; path=/; secure; HttpOnly, MUID=3390FF8D527B6A5D0528F19853B36BE3; domain=.microsoft.com; expires=Mon, 15-Nov-2021 08:05:57 GMT; path=/;SameSite=None; secure, X-FD-FEATURES=ids=1690c%2csfwaab%2catperf680c2%2c1786t1%2c969t1%2c882c%2c936t1a%2ctasmigration010%2ccartemberpl&imp=e9a8a0b4-7354-4136-a6fc-befe822aa602; expires=Sun, 15-Nov-2020 08:05:57 GMT; path=/; secure; HttpOnly, X-FD-Time=1; expires=Fri, 15-Nov-2019 08:10:57 GMT; path=/; secure; HttpOnly, akacd_OneRF=1581581157~rv=58~id=033ad52aa4add00d9694cf6d729f8b05; path=/; Expires=Thu, 13 Feb 2020 08:05:57 GMT, akacd_OneRF=1581581157~rv=58~id=033ad52aa4add00d9694cf6d729f8b05; path=/; Expires=Thu, 13 Feb 2020 08:05:57 GMT"); + headers.Add("Set-Cookie", "isFirstSession=1; path=/; secure; HttpOnly, MUID=054E212C88D965E415702F3A89116462; domain=.microsoft.com; expires=Tue, 16-Nov-2021 20:09:48 GMT; path=/;SameSite=None; secure, X-FD-FEATURES=ids=1690t1%2csfwaab%2catperf680t2%2c1786t1a%2c969t1%2c882c%2c936ca%2ctasmigration010%2ccartemberpl&imp=ec46db78-0cbf-40c4-97a2-a2420cd8e1ca; expires=Mon, 16-Nov-2020 20:09:48 GMT; path=/; secure; HttpOnly, X-FD-Time=1; expires=Sat, 16-Nov-2019 20:14:48 GMT; path=/; secure; HttpOnly, akacd_OneRF=1581710989~rv=77~id=b1ce48559f2a627f15dc2e792b7d9740; path=/; Expires=Fri, 14 Feb 2020 20:09:49 GMT, akacd_OneRF=1581710989~rv=77~id=b1ce48559f2a627f15dc2e792b7d9740; path=/; Expires=Fri, 14 Feb 2020 20:09:49 GMT"); headers.Add("TLS_version", "tls1.2"); headers.Add("Strict-Transport-Security", "max-age=31536000"); headers.Add("X-RTag", "RT"); @@ -54,8 +66,11 @@ public void TestLoad() headers.Add("Expires", "-1"); return headers; }); - resMock.Setup(x => x.GetResponseStream()).Returns( - () => new MemoryStream()); + resMock.Setup(x => x.GetResponseStream()) + .Returns(() => new FileStream( + Path.Combine(_contentDir, "mshome.htm"), + FileMode.Open, + FileAccess.Read)); resMock.Setup(x => x.LastModified).Returns(DateTime.UtcNow); return resMock.Object; }); @@ -63,7 +78,8 @@ public void TestLoad() }); var htmlWeb = new HtmlWeb(factoryMock.Object); - htmlWeb.Load(new Uri("https://www.microsoft.com/en-nz")); + var doc = htmlWeb.Load(new Uri("https://www.microsoft.com/")); + Assert.IsTrue(doc.DocumentNode.Descendants().Count() > 0); } [Test] diff --git a/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs b/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs index ff5b1db9..ba1fd2b0 100644 --- a/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs +++ b/src/Tools/TestCodeGen/Generator/HtmlWebTestGenerator.cs @@ -98,7 +98,7 @@ public override IEnumerable Generate() // generate code for mocking IHttpWebResponse.Headers property var headers = res.Headers; code4Headers = headers.AllKeys - .Select(k => $"{Indent(7)}headers.Add(\"" + k + "\", \"" + headers[k].Replace("\"", "\\\"") + "\");") + .Select(k => $@"{Indent(7)}headers.Add(""{k}"", ""{headers[k].Replace("\"", "\\\"")}"");") .DefaultIfEmpty() .Aggregate((a, b) => $"{a}{Environment.NewLine}{b}"); @@ -157,6 +157,7 @@ public override IEnumerable Generate() Content = regexNewLine.Replace($@"using System; using System.IO; using System.Net; +using System.Reflection; using Moq; using NUnit.Framework; @@ -171,7 +172,7 @@ namespace HtmlAgilityPack.Tests {Indent(2)}public void Setup() {Indent(2)}{{ {Indent(3)}_contentDir = Path.Combine( -{Indent(4)}Path.GetDirectoryName(typeof(HtmlDocumentTests).Assembly.Location), +{Indent(4)}Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), {Indent(4)}""files""); {Indent(2)}}} @@ -200,7 +201,7 @@ namespace HtmlAgilityPack.Tests {Indent(4)}}}); {Indent(3)}var htmlWeb = new HtmlWeb(factoryMock.Object); -{Indent(3)}htmlWeb.Load(new Uri(""{Url.OriginalString}"")); +{Indent(3)}var doc = htmlWeb.Load(new Uri(""{Url.OriginalString}"")); {Indent(2)}}} {Indent(1)}}} }} From 7fbaf94dfb7ae710d8495664598a8d25206362f4 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 10:51:14 +1300 Subject: [PATCH 13/14] add test for HtmlWeb.OverrideEncoding --- .../HtmlAgilityPack.Tests/HtmlWebTests.cs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs index d53da7fc..adc0f722 100644 --- a/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs +++ b/src/Tests/HtmlAgilityPack.Tests/HtmlWebTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Net; using System.Reflection; +using System.Text; using Moq; using NUnit.Framework; @@ -147,6 +148,82 @@ public void TestLoadThrowsEncodingNotSupportedException() } Assert.That(exception, Is.InstanceOf(typeof(EncodingNotSupportedException))); } + + [Test] + public void TestLoadOverridingEncoding() + { + var factoryMock = new Mock(); + factoryMock.Setup(x => x.Create(It.IsAny())) + .Returns(u => + { + var reqMock = new Mock(); + reqMock.Setup(x => x.Request).Returns(null as HttpWebRequest); + reqMock.Setup(x => x.GetResponse()).Returns(() => + { + var resMock = new Mock(); + resMock.Setup(x => x.ResponseUri).Returns(u); + resMock.Setup(x => x.StatusCode).Returns(HttpStatusCode.OK); + resMock.Setup(x => x.ContentType).Returns("text/html; charset=UTF-8"); + resMock.Setup(x => x.ContentEncoding).Returns("identity"); + resMock.Setup(x => x.Headers).Returns(() => + { + var headers = new WebHeaderCollection(); + headers.Add("Transfer-Encoding", "chunked"); + headers.Add("Connection", "keep-alive"); + headers.Add("X-Cache-Hits", "4"); + headers.Add("Cache-Control", "no-store, must-revalidate, no-cache, max-age=0"); + headers.Add("X-Chef", "Gennaro"); + headers.Add("Pragma", "public"); + headers.Add("X-Cache-Keep", "3600.000"); + headers.Add("Vary", "Accept-Encoding"); + headers.Add("X-Country-Code", "NZ"); + headers.Add("X-Cache-Status", "MISS"); + headers.Add("X-Backend", "i_04400f0cd28e027e7_10_170_21_65"); + headers.Add("Date", "Fri, 15 Nov 2019 07:44:42 GMT"); + headers.Add("X-Cache-TTL-Remaining", "595.653"); + headers.Add("Server", "ZENEDGE"); + headers.Add("X-Zen-Fury", "efadaddb4a91c6f51faaa477a91c047faa84f52d"); + headers.Add("X-Cache", "FoodCache"); + headers.Add("Accept-Ranges", "bytes"); + headers.Add("Age", "3004"); + headers.Add("X-Cache-Age", "3004.347"); + headers.Add("X-Cdn", "Served-By-Zenedge"); + headers.Add("Content-Type", "text/html; charset=UTF-8"); + headers.Add("Last-Modified", "Thu, 01 Jan 1970 00:00:00 GMT"); + headers.Add("Expires", "Fri, 15 Nov 2019 08:44:41 GMT"); + headers.Add("Content-Encoding", "identity"); + return headers; + }); + resMock.Setup(x => x.GetResponseStream()).Returns( + () => new MemoryStream()); + resMock.Setup(x => x.LastModified).Returns(DateTime.UtcNow); + return resMock.Object; + }); + return reqMock.Object; + }); + + var url = new Uri("https://www.jamieoliver.com/recipes/chicken-recipes/chicken-tofu-noodle-soup/"); + var htmlWeb = new HtmlWeb(factoryMock.Object); + HtmlDocument doc = null; + try + { + doc = htmlWeb.Load(url); + Assert.Fail("Load should fail when receiving a response with 'Content-Encoding: identity' in the headers."); + } + catch (EncodingNotSupportedException) + { + try + { + htmlWeb.OverrideEncoding = Encoding.UTF8; + doc = htmlWeb.Load(url); + } + catch (Exception e) + { + Assert.Fail("Load failed even though the invalid encoding was adjusted by overriding it."); + } + } + Assert.That(doc, Is.Not.Null); + } } } From d9dad6423cb218b6b15bb598c596935cdac10422 Mon Sep 17 00:00:00 2001 From: Yas Ikeda Date: Sun, 17 Nov 2019 11:11:27 +1300 Subject: [PATCH 14/14] correct Preprocessor symbols used in HtmlWebResponseWrapper The code was never be active because of the wrong condition and wrong symbols in the processor directive. correct it so that the code is activated in net4.5 and newer. add a preprocessor symbol in two projects so as to build HtmlAgilityPack.Master solution also with Debug configuration. --- src/HtmlAgilityPack.Net35/HtmlAgilityPack.Net35.csproj | 2 +- .../HtmlAgilityPack.Net40-client.csproj | 2 +- src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/HtmlAgilityPack.Net35/HtmlAgilityPack.Net35.csproj b/src/HtmlAgilityPack.Net35/HtmlAgilityPack.Net35.csproj index eb8ebf1d..a69d929b 100644 --- a/src/HtmlAgilityPack.Net35/HtmlAgilityPack.Net35.csproj +++ b/src/HtmlAgilityPack.Net35/HtmlAgilityPack.Net35.csproj @@ -18,7 +18,7 @@ full false bin\Debug\ - DEBUG;TRACE + DEBUG;TRACE;FX35 prompt 4 diff --git a/src/HtmlAgilityPack.Net40-client/HtmlAgilityPack.Net40-client.csproj b/src/HtmlAgilityPack.Net40-client/HtmlAgilityPack.Net40-client.csproj index 91a90c33..502fcdde 100644 --- a/src/HtmlAgilityPack.Net40-client/HtmlAgilityPack.Net40-client.csproj +++ b/src/HtmlAgilityPack.Net40-client/HtmlAgilityPack.Net40-client.csproj @@ -44,7 +44,7 @@ full false bin\Debug\ - DEBUG;TRACE + DEBUG;TRACE;FX40 prompt 4 bin\Debug\HtmlAgilityPack.XML diff --git a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs index c3847056..0354a55e 100644 --- a/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs +++ b/src/HtmlAgilityPack.Shared/HttpWebResponseWrapper.cs @@ -40,7 +40,7 @@ public void Close() public void Dispose() { -#if NET20 || NET35 || NET40 +#if !(FX20 || FX35 || FX40) _response.Dispose(); #endif }