From 1a1c6ca37a60ada4550ab60b20037de595187904 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Fri, 23 Oct 2020 11:09:34 -0700 Subject: [PATCH 01/21] update markdig to 0.22.0 --- src/NuGetGallery/NuGetGallery.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGetGallery/NuGetGallery.csproj b/src/NuGetGallery/NuGetGallery.csproj index 66a656ca0b..30598958ba 100644 --- a/src/NuGetGallery/NuGetGallery.csproj +++ b/src/NuGetGallery/NuGetGallery.csproj @@ -2063,7 +2063,7 @@ 4.3.1 - 0.15.4 + 0.22.0 2.2.0 From 616471d4647235aa59765e12af67c3ab5972d734 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Fri, 23 Oct 2020 14:07:22 -0700 Subject: [PATCH 02/21] updated markdig.signed.dll in third party props --- sign.thirdparty.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sign.thirdparty.props b/sign.thirdparty.props index 7d468c6722..3d8cf9a42d 100644 --- a/sign.thirdparty.props +++ b/sign.thirdparty.props @@ -26,7 +26,7 @@ - + From 53186dd85d6b9eceba9bd46a9c661bbf2320ae0e Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Mon, 26 Oct 2020 17:18:11 -0700 Subject: [PATCH 03/21] test with newversion package with updated markdig version --- src/NuGetGallery.Core/NuGetGallery.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index d1c037fa2f..9ac504c2f5 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -248,7 +248,7 @@ 2.74.0 - 2.74.0 + 2.77.1 2.74.0 From ebb7557d66bc6ce374efed38cc6e5be06fdb272d Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Mon, 26 Oct 2020 17:27:24 -0700 Subject: [PATCH 04/21] version test --- src/NuGetGallery.Core/NuGetGallery.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index 9ac504c2f5..e0d7eefa10 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -248,7 +248,7 @@ 2.74.0 - 2.77.1 + 2.77.1-upgrade-markdig-4176507 2.74.0 From a2e8cae0b45b901ea4f5363ebcf8d7c8e3c8821b Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Tue, 27 Oct 2020 12:27:32 -0700 Subject: [PATCH 05/21] update the version of nuget.services --- src/DatabaseMigrationTools/DatabaseMigrationTools.csproj | 2 +- .../GitHubVulnerabilities2Db.csproj | 2 +- src/NuGetGallery.Core/NuGetGallery.Core.csproj | 8 ++++---- src/NuGetGallery.Services/NuGetGallery.Services.csproj | 4 ++-- src/NuGetGallery/NuGetGallery.csproj | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj b/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj index 0defba23ea..c4d9d11a46 100644 --- a/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj +++ b/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj @@ -65,7 +65,7 @@ all - 2.74.0 + 2.78.0 diff --git a/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj b/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj index 08859234c8..7285725568 100644 --- a/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj +++ b/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj @@ -89,7 +89,7 @@ 4.3.0-dev-3612825 - 2.74.0 + 2.78.0 diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index e0d7eefa10..87347d7108 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -245,16 +245,16 @@ - 2.74.0 + 2.78.0 - 2.77.1-upgrade-markdig-4176507 + 2.78.0 - 2.74.0 + 2.78.0 - 2.74.0 + 2.78.0 1.2.0 diff --git a/src/NuGetGallery.Services/NuGetGallery.Services.csproj b/src/NuGetGallery.Services/NuGetGallery.Services.csproj index ffd976fb36..1f635650df 100644 --- a/src/NuGetGallery.Services/NuGetGallery.Services.csproj +++ b/src/NuGetGallery.Services/NuGetGallery.Services.csproj @@ -305,10 +305,10 @@ 5.8.0-preview.3.6823 - 2.74.0 + 2.78.0 - 2.74.0 + 2.78.0 0.2.0 diff --git a/src/NuGetGallery/NuGetGallery.csproj b/src/NuGetGallery/NuGetGallery.csproj index 30598958ba..6cec8a9f58 100644 --- a/src/NuGetGallery/NuGetGallery.csproj +++ b/src/NuGetGallery/NuGetGallery.csproj @@ -2245,13 +2245,13 @@ 5.8.0-preview.3.6823 - 2.74.0 + 2.78.0 - 2.74.0 + 2.78.0 - 2.74.0 + 2.78.0 1.0.0 From 44c8a6b21c25c9baa5ba733ea15f6e057c97df08 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Tue, 27 Oct 2020 14:57:41 -0700 Subject: [PATCH 06/21] a --- src/NuGetGallery/Services/MarkdownService.cs | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index e8374fa936..37e616f8ad 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -7,6 +7,8 @@ using System.Web; using CommonMark; using CommonMark.Syntax; +using Markdig; + namespace NuGetGallery { @@ -109,6 +111,26 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); return output; } + } + + public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy) + { + var output = new RenderedMarkdownResult() + { + ImagesRewritten = false, + Content = "" + }; + + var readmeWithoutBom = markdownString.StartsWith("\ufeff") ? markdownString.Replace("\ufeff", "") : markdownString; + + // HTML encode markdown, except for block quotes, to block inline html. + var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); + + var builder = + + + + } } } \ No newline at end of file From 0e925e775abdd5a8767170c9689b61d54c95a627 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Wed, 28 Oct 2020 23:40:34 -0700 Subject: [PATCH 07/21] migrate to markdig --- src/NuGetGallery/Services/IMarkdownService.cs | 15 +++++ src/NuGetGallery/Services/MarkdownService.cs | 65 +++++++++++++++++-- .../Services/MarkdownServiceFacts.cs | 29 +++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/NuGetGallery/Services/IMarkdownService.cs b/src/NuGetGallery/Services/IMarkdownService.cs index e6e647900a..32e689ef07 100644 --- a/src/NuGetGallery/Services/IMarkdownService.cs +++ b/src/NuGetGallery/Services/IMarkdownService.cs @@ -19,5 +19,20 @@ public interface IMarkdownService /// headers can be incremented by this value, eg if 2 supplied then h1 will become h3 /// HTML data RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy); + + /// + /// Returns HTML from the supplied markdown using Markdig + /// + /// markdown input + /// HTML data + RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString); + + /// + /// Returns HTML from the supplied markdown using Markdig + /// + /// markdown input + /// headers can be incremented by this value, eg if 2 supplied then h1 will become h3 + /// HTML data + RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy); } } \ No newline at end of file diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index 37e616f8ad..adbb7929d1 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -3,12 +3,17 @@ using System; using System.IO; +using System.Linq; using System.Text.RegularExpressions; +using System.Timers; using System.Web; using CommonMark; using CommonMark.Syntax; using Markdig; - +using Markdig.Parsers; +using Markdig.Renderers; +using Markdig.Syntax; +using Markdig.Syntax.Inlines; namespace NuGetGallery { @@ -23,6 +28,11 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) return GetHtmlFromMarkdown(markdownString, 1); } + public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString) + { + return GetHtmlFromMarkdownMarkdig(markdownString, 1); + } + public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy) { var output = new RenderedMarkdownResult() @@ -122,15 +132,62 @@ public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, }; var readmeWithoutBom = markdownString.StartsWith("\ufeff") ? markdownString.Replace("\ufeff", "") : markdownString; - - // HTML encode markdown, except for block quotes, to block inline html. var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); - var builder = + var pipeline = new MarkdownPipelineBuilder() + .UseSoftlineBreakAsHardlineBreak() + .Build(); + var document = Markdown.Parse(encodedMarkdown, pipeline); + + foreach (var node in document.Descendants()) + { + if (node is Markdig.Syntax.Block) + { + if (node is HeadingBlock heading) + { + heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); + } + } + else if (node is Markdig.Syntax.Inlines.Inline) + { + if (node is LinkInline linkInline) + { + if (linkInline.IsImage) + { + if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true)) + { + linkInline.Url = string.Empty; + } + else + { + output.ImagesRewritten = output.ImagesRewritten || (linkInline.Url != readyUriString); + linkInline.Url = readyUriString; + } + } + else + { + if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString)) + { + linkInline.Url = string.Empty; + } + else + { + linkInline.Url = readyUriString; + } + } + } + } + } + StringWriter htmlWriter = new StringWriter(); + var renderer = new HtmlRenderer(htmlWriter); + renderer.Render(document); + htmlWriter.Flush(); + output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); + return output; } } } \ No newline at end of file diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index f5ea720723..4a707fd93d 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -46,6 +46,35 @@ public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); } + [Theory] + [InlineData("", "

<script>alert('test')</script>

")] + [InlineData("", "

<img src="javascript:alert('test');">

")] + [InlineData("", "

<a href="javascript:alert('test');">

")] + public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml) + { + Assert.Equal(expectedHtml, StripNewLines( + _markdownService.GetHtmlFromMarkdownMarkdig(originalMd).Content)); + } + + [Theory] + [InlineData("# Heading", "

Heading

", false)] + [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false)] + [InlineData("- List", "
  • List
", false)] + [InlineData("[text](http://www.test.com)", "

text

", false)] + [InlineData("[text](javascript:alert('hi'))", "

text

", false)] + [InlineData("> Blockquote", "

<text>Blockquote</text>

", false)] + [InlineData("[text](http://www.asp.net)", "

text

", false)] + [InlineData("[text](badurl://www.asp.net)", "

text

", false)] + [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true)] + [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false)] + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true)] + public void ConvertsMarkdownToHtmlMarkdig(string originalMd, string expectedHtml, bool imageRewriteExpected) + { + var readMeResult = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd); + Assert.Equal(expectedHtml, StripNewLines(readMeResult.Content)); + Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); + } + private static string StripNewLines(string text) { return text.Replace("\r\n", "").Replace("\n", ""); From b866e4b65b0be362cd3063d5405164deaa2ee7a0 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Thu, 29 Oct 2020 13:46:23 -0700 Subject: [PATCH 08/21] rm legacy to test markdown with markdig --- src/NuGetGallery/Services/MarkdownService.cs | 99 +------------------ .../Services/MarkdownServiceFacts.cs | 18 +++- 2 files changed, 17 insertions(+), 100 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index adbb7929d1..4f0dd16f18 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -28,11 +28,6 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) return GetHtmlFromMarkdown(markdownString, 1); } - public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString) - { - return GetHtmlFromMarkdownMarkdig(markdownString, 1); - } - public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy) { var output = new RenderedMarkdownResult() @@ -41,99 +36,11 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc Content = "" }; - var readmeWithoutBom = markdownString.StartsWith("\ufeff") ? markdownString.Replace("\ufeff", "") : markdownString; + var readmeWithoutBom = markdownString.TrimStart('\ufeff'); // HTML encode markdown, except for block quotes, to block inline html. var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); - var settings = CommonMarkSettings.Default.Clone(); - settings.RenderSoftLineBreaksAsLineBreaks = true; - - // Parse executes CommonMarkConverter's ProcessStage1 and ProcessStage2. - var document = CommonMarkConverter.Parse(encodedMarkdown, settings); - foreach (var node in document.AsEnumerable()) - { - if (node.IsOpening) - { - var block = node.Block; - if (block != null) - { - switch (block.Tag) - { - // Demote heading tags so they don't overpower expander headings. - case BlockTag.AtxHeading: - case BlockTag.SetextHeading: - var level = (byte)Math.Min(block.Heading.Level + incrementHeadersBy, 6); - block.Heading = new HeadingData(level); - break; - - // Decode preformatted blocks to prevent double encoding. - // Skip BlockTag.BlockQuote, which are partially decoded upfront. - case BlockTag.FencedCode: - case BlockTag.IndentedCode: - if (block.StringContent != null) - { - var content = block.StringContent.TakeFromStart(block.StringContent.Length); - var unencodedContent = HttpUtility.HtmlDecode(content); - block.StringContent.Replace(unencodedContent, 0, unencodedContent.Length); - } - break; - } - } - - var inline = node.Inline; - if (inline != null) - { - if (inline.Tag == InlineTag.Link) - { - // Allow only http or https links in markdown. Transform link to https for known domains. - if (!PackageHelper.TryPrepareUrlForRendering(inline.TargetUrl, out string readyUriString)) - { - inline.TargetUrl = string.Empty; - } - else - { - inline.TargetUrl = readyUriString; - } - } - - else if (inline.Tag == InlineTag.Image) - { - if (!PackageHelper.TryPrepareUrlForRendering(inline.TargetUrl, out string readyUriString, rewriteAllHttp: true)) - { - inline.TargetUrl = string.Empty; - } - else - { - output.ImagesRewritten = output.ImagesRewritten || (inline.TargetUrl != readyUriString); - inline.TargetUrl = readyUriString; - } - } - } - } - } - - // CommonMark.Net does not support link attributes, so manually inject nofollow. - using (var htmlWriter = new StringWriter()) - { - CommonMarkConverter.ProcessStage3(document, htmlWriter, settings); - - output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); - return output; - } - } - - public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy) - { - var output = new RenderedMarkdownResult() - { - ImagesRewritten = false, - Content = "" - }; - - var readmeWithoutBom = markdownString.StartsWith("\ufeff") ? markdownString.Replace("\ufeff", "") : markdownString; - var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); - var pipeline = new MarkdownPipelineBuilder() .UseSoftlineBreakAsHardlineBreak() .Build(); @@ -144,6 +51,7 @@ public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, { if (node is Markdig.Syntax.Block) { + // Demote heading tags so they don't overpower expander headings. if (node is HeadingBlock heading) { heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); @@ -156,6 +64,7 @@ public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, { if (linkInline.IsImage) { + // Allow only http or https links in markdown. Transform link to https for known domains. if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true)) { linkInline.Url = string.Empty; @@ -185,8 +94,8 @@ public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, var renderer = new HtmlRenderer(htmlWriter); renderer.Render(document); htmlWriter.Flush(); + //manually inject nofollow since markdig doesn't support inject nofollw in encode output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); - return output; } } diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 4a707fd93d..1d600f5744 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -52,17 +52,16 @@ public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool [InlineData("", "

<a href="javascript:alert('test');">

")] public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml) { - Assert.Equal(expectedHtml, StripNewLines( - _markdownService.GetHtmlFromMarkdownMarkdig(originalMd).Content)); + Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdownMarkdig(originalMd).Content); } [Theory] [InlineData("# Heading", "

Heading

", false)] [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false)] - [InlineData("- List", "
  • List
", false)] + [InlineData("- List", "
    \n
  • List
    • \n
", false)] [InlineData("[text](http://www.test.com)", "

text

", false)] [InlineData("[text](javascript:alert('hi'))", "

text

", false)] - [InlineData("> Blockquote", "

<text>Blockquote</text>

", false)] + [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false)] [InlineData("[text](http://www.asp.net)", "

text

", false)] [InlineData("[text](badurl://www.asp.net)", "

text

", false)] [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true)] @@ -71,10 +70,19 @@ public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml) public void ConvertsMarkdownToHtmlMarkdig(string originalMd, string expectedHtml, bool imageRewriteExpected) { var readMeResult = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd); - Assert.Equal(expectedHtml, StripNewLines(readMeResult.Content)); + Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); } + [Theory] + [InlineData("- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image resize in browser.")] + [InlineData("```Sample text here...```")] + public void compareNewAPIwithOldAPI(string originalMd) { + var readMeResultMarkdig = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd); + var expected = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(StripNewLines(expected.Content), StripNewLines(readMeResultMarkdig.Content)); + } + private static string StripNewLines(string text) { return text.Replace("\r\n", "").Replace("\n", ""); From 76109a012dd9ce203a42a5f1f067162cba3f8f89 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Thu, 29 Oct 2020 14:02:42 -0700 Subject: [PATCH 09/21] update function --- src/NuGetGallery/Services/IMarkdownService.cs | 15 ------ .../Services/MarkdownServiceFacts.cs | 49 ++----------------- 2 files changed, 3 insertions(+), 61 deletions(-) diff --git a/src/NuGetGallery/Services/IMarkdownService.cs b/src/NuGetGallery/Services/IMarkdownService.cs index 32e689ef07..e6e647900a 100644 --- a/src/NuGetGallery/Services/IMarkdownService.cs +++ b/src/NuGetGallery/Services/IMarkdownService.cs @@ -19,20 +19,5 @@ public interface IMarkdownService /// headers can be incremented by this value, eg if 2 supplied then h1 will become h3 /// HTML data RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy); - - /// - /// Returns HTML from the supplied markdown using Markdig - /// - /// markdown input - /// HTML data - RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString); - - /// - /// Returns HTML from the supplied markdown using Markdig - /// - /// markdown input - /// headers can be incremented by this value, eg if 2 supplied then h1 will become h3 - /// HTML data - RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy); } } \ No newline at end of file diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 1d600f5744..21edfc4f0b 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -23,36 +23,7 @@ public GetReadMeHtmlMethod() [InlineData("", "

<a href="javascript:alert('test');">

")] public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml) { - Assert.Equal(expectedHtml, StripNewLines( - _markdownService.GetHtmlFromMarkdown(originalMd).Content)); - } - - [Theory] - [InlineData("# Heading", "

Heading

", false)] - [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false)] - [InlineData("- List", "
  • List
", false)] - [InlineData("[text](http://www.test.com)", "

text

", false)] - [InlineData("[text](javascript:alert('hi'))", "

text

", false)] - [InlineData("> Blockquote", "

<text>Blockquote</text>

", false)] - [InlineData("[text](http://www.asp.net)", "

text

", false)] - [InlineData("[text](badurl://www.asp.net)", "

text

", false)] - [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true)] - [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false)] - [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true)] - public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected) - { - var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); - Assert.Equal(expectedHtml, StripNewLines(readMeResult.Content)); - Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); - } - - [Theory] - [InlineData("", "

<script>alert('test')</script>

")] - [InlineData("", "

<img src="javascript:alert('test');">

")] - [InlineData("", "

<a href="javascript:alert('test');">

")] - public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml) - { - Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdownMarkdig(originalMd).Content); + Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdown(originalMd).Content); } [Theory] @@ -67,26 +38,12 @@ public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml) [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true)] [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true)] - public void ConvertsMarkdownToHtmlMarkdig(string originalMd, string expectedHtml, bool imageRewriteExpected) + public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected) { - var readMeResult = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); } - - [Theory] - [InlineData("- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image resize in browser.")] - [InlineData("```Sample text here...```")] - public void compareNewAPIwithOldAPI(string originalMd) { - var readMeResultMarkdig = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd); - var expected = _markdownService.GetHtmlFromMarkdown(originalMd); - Assert.Equal(StripNewLines(expected.Content), StripNewLines(readMeResultMarkdig.Content)); - } - - private static string StripNewLines(string text) - { - return text.Replace("\r\n", "").Replace("\n", ""); - } } } } From c133a6a8ae0996fb9de71f68a0f663a556bf2e78 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Mon, 9 Nov 2020 14:40:14 -0800 Subject: [PATCH 10/21] add feature flag --- .../Configuration/FeatureFlagService.cs | 6 + .../Configuration/IFeatureFlagService.cs | 5 + src/NuGetGallery/Services/MarkdownService.cs | 123 +++++++++++++++++- .../Services/MarkdownServiceFacts.cs | 50 ++++--- 4 files changed, 165 insertions(+), 19 deletions(-) diff --git a/src/NuGetGallery.Services/Configuration/FeatureFlagService.cs b/src/NuGetGallery.Services/Configuration/FeatureFlagService.cs index 9069fc48fd..d080695717 100644 --- a/src/NuGetGallery.Services/Configuration/FeatureFlagService.cs +++ b/src/NuGetGallery.Services/Configuration/FeatureFlagService.cs @@ -40,6 +40,7 @@ public class FeatureFlagService : IFeatureFlagService private const string PackageRenamesFeatureName = GalleryPrefix + "PackageRenames"; private const string EmbeddedReadmeFlightName = GalleryPrefix + "EmbeddedReadmes"; private const string LicenseMdRenderingFlightName = GalleryPrefix + "LicenseMdRendering"; + private const string MarkdigMdRenderingFlightName = GalleryPrefix + "MarkdigMdRendering"; private const string ODataV1GetAllNonHijackedFeatureName = GalleryPrefix + "ODataV1GetAllNonHijacked"; private const string ODataV1GetAllCountNonHijackedFeatureName = GalleryPrefix + "ODataV1GetAllCountNonHijacked"; @@ -284,5 +285,10 @@ public bool IsODataV2SearchCountNonHijackedEnabled() { return _client.IsEnabled(ODataV2SearchCountNonHijackedFeatureName, defaultValue: true); } + + public bool IsMarkdigMdRenderingEnabled() + { + return _client.IsEnabled(MarkdigMdRenderingFlightName, defaultValue: false); + } } } \ No newline at end of file diff --git a/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs b/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs index b30d3bfac9..dcc883aa85 100644 --- a/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs +++ b/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs @@ -225,5 +225,10 @@ public interface IFeatureFlagService /// Whether the /Search()/$count endpoint is enabled for non-hijacked queries for the V2 OData API. /// bool IsODataV2SearchCountNonHijackedEnabled(); + + /// + /// Whether rending Markdown content to HTML using Markdig is enabaled + /// + bool IsMarkdigMdRenderingEnabled(); } } diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index 4f0dd16f18..f0c932104f 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -21,14 +21,131 @@ public class MarkdownService : IMarkdownService { private static readonly TimeSpan RegexTimeout = TimeSpan.FromMinutes(1); private static readonly Regex EncodedBlockQuotePattern = new Regex("^ {0,3}>", RegexOptions.Multiline, RegexTimeout); - private static readonly Regex CommonMarkLinkPattern = new Regex(" "); + + var settings = CommonMarkSettings.Default.Clone(); + settings.RenderSoftLineBreaksAsLineBreaks = true; + + // Parse executes CommonMarkConverter's ProcessStage1 and ProcessStage2. + var document = CommonMarkConverter.Parse(encodedMarkdown, settings); + foreach (var node in document.AsEnumerable()) + { + if (node.IsOpening) + { + var block = node.Block; + if (block != null) + { + switch (block.Tag) + { + // Demote heading tags so they don't overpower expander headings. + case BlockTag.AtxHeading: + case BlockTag.SetextHeading: + var level = (byte)Math.Min(block.Heading.Level + incrementHeadersBy, 6); + block.Heading = new HeadingData(level); + break; + + // Decode preformatted blocks to prevent double encoding. + // Skip BlockTag.BlockQuote, which are partially decoded upfront. + case BlockTag.FencedCode: + case BlockTag.IndentedCode: + if (block.StringContent != null) + { + var content = block.StringContent.TakeFromStart(block.StringContent.Length); + var unencodedContent = HttpUtility.HtmlDecode(content); + block.StringContent.Replace(unencodedContent, 0, unencodedContent.Length); + } + break; + } + } + + var inline = node.Inline; + if (inline != null) + { + if (inline.Tag == InlineTag.Link) + { + // Allow only http or https links in markdown. Transform link to https for known domains. + if (!PackageHelper.TryPrepareUrlForRendering(inline.TargetUrl, out string readyUriString)) + { + inline.TargetUrl = string.Empty; + } + else + { + inline.TargetUrl = readyUriString; + } + } + + else if (inline.Tag == InlineTag.Image) + { + if (!PackageHelper.TryPrepareUrlForRendering(inline.TargetUrl, out string readyUriString, rewriteAllHttp: true)) + { + inline.TargetUrl = string.Empty; + } + else + { + output.ImagesRewritten = output.ImagesRewritten || (inline.TargetUrl != readyUriString); + inline.TargetUrl = readyUriString; + } + } + } + } + } + + // CommonMark.Net does not support link attributes, so manually inject nofollow. + using (var htmlWriter = new StringWriter()) + { + CommonMarkConverter.ProcessStage3(document, htmlWriter, settings); + + output.Content = LinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); + return output; + } + } + + private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy) { var output = new RenderedMarkdownResult() { @@ -95,7 +212,7 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc renderer.Render(document); htmlWriter.Flush(); //manually inject nofollow since markdig doesn't support inject nofollw in encode - output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); + output.Content = LinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); return output; } } diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 21edfc4f0b..003f764edd 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Moq; using Xunit; namespace NuGetGallery @@ -11,34 +12,51 @@ public class MarkdownServiceFacts public class GetReadMeHtmlMethod { private readonly MarkdownService _markdownService; + private readonly Mock _featureFlagService; public GetReadMeHtmlMethod() { - _markdownService = new MarkdownService(); + _featureFlagService = new Mock(); + _markdownService = new MarkdownService(_featureFlagService.Object); } [Theory] - [InlineData("", "

<script>alert('test')</script>

")] - [InlineData("", "

<img src="javascript:alert('test');">

")] - [InlineData("", "

<a href="javascript:alert('test');">

")] - public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml) + [InlineData("", "

<script>alert('test')</script>

", true)] + [InlineData("", "

<script>alert('test')</script>

", false)] + [InlineData("", "

<img src="javascript:alert('test');">

", true)] + [InlineData("", "

<img src="javascript:alert('test');">

", false)] + [InlineData("", "

<a href="javascript:alert('test');">

", true)] + [InlineData("", "

<a href="javascript:alert('test');">

", false)] + public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool isMarkdigMdRenderingEnabled) { + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdown(originalMd).Content); } [Theory] - [InlineData("# Heading", "

Heading

", false)] - [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false)] - [InlineData("- List", "
    \n
  • List
    • \n
", false)] - [InlineData("[text](http://www.test.com)", "

text

", false)] - [InlineData("[text](javascript:alert('hi'))", "

text

", false)] - [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false)] - [InlineData("[text](http://www.asp.net)", "

text

", false)] - [InlineData("[text](badurl://www.asp.net)", "

text

", false)] - [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true)] - [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false)] + [InlineData("# Heading", "

Heading

", false, true)] + [InlineData("# Heading", "

Heading

", false, false)] + [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, true)] + [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, false)] + [InlineData("- List", "
    \n
  • List
    • \n
", false, true)] + [InlineData("- List", "
    \n
  • List
    • \n
", false, false)] + [InlineData("[text](http://www.test.com)", "

text

", false, true)] + [InlineData("[text](http://www.test.com)", "

text

", false, false)] + [InlineData("[text](javascript:alert('hi'))", "

text

", false, true)] + [InlineData("[text](javascript:alert('hi'))", "

text

", false, false)] + [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, true)] + [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, false)] + [InlineData("[text](http://www.asp.net)", "

text

", false, true)] + [InlineData("[text](http://www.asp.net)", "

text

", false, false)] + [InlineData("[text](badurl://www.asp.net)", "

text

", false, true)] + [InlineData("[text](badurl://www.asp.net)", "

text

", false, false)] + [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true, true)] + [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true, false)] + [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, true)] + [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, false)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true)] - public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected) + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false)] + public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected, bool isMarkdigMdRenderingEnabled) { var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); From 3c3fcb2cf045ef18d3477b04bbd8e564c44d91eb Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Wed, 11 Nov 2020 17:45:24 -0800 Subject: [PATCH 11/21] change unit test --- tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 003f764edd..65906b3504 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -38,14 +38,14 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("# Heading", "

Heading

", false, false)] [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, true)] [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, false)] - [InlineData("- List", "
    \n
  • List
    • \n
", false, true)] - [InlineData("- List", "
    \n
  • List
    • \n
", false, false)] + [InlineData("- List", "
    \r\n
  • List
    • \r\n
", false, true)] + [InlineData("- List", "
    \r\n
  • List
    • \r\n
", false, false)] [InlineData("[text](http://www.test.com)", "

text

", false, true)] [InlineData("[text](http://www.test.com)", "

text

", false, false)] [InlineData("[text](javascript:alert('hi'))", "

text

", false, true)] [InlineData("[text](javascript:alert('hi'))", "

text

", false, false)] - [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, true)] - [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, false)] + [InlineData("> Blockquote", "
\r\n

<text>Blockquote</text>

\r\n
", false, true)] + [InlineData("> Blockquote", "
\r\n

<text>Blockquote</text>

\r\n
", false, false)] [InlineData("[text](http://www.asp.net)", "

text

", false, true)] [InlineData("[text](http://www.asp.net)", "

text

", false, false)] [InlineData("[text](badurl://www.asp.net)", "

text

", false, true)] From 79678a11be391d0a462195c8515a813413e7d313 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Wed, 11 Nov 2020 18:04:14 -0800 Subject: [PATCH 12/21] unit test --- tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 65906b3504..d9e7e59057 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -54,8 +54,10 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("![image](http://www.asp.net/fake.jpg)", "

\"image\"

", true, false)] [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, true)] [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, false)] - [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true)] - [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false)] + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, true)] + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, false)] + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false, true)] + [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false, false)] public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected, bool isMarkdigMdRenderingEnabled) { var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); From fc338a95a45ec99707d2bbaa19067c06e015cb72 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Wed, 11 Nov 2020 18:24:15 -0800 Subject: [PATCH 13/21] unit test --- tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index d9e7e59057..15f368787a 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -56,8 +56,6 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, false)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, true)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, false)] - [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false, true)] - [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", false, false)] public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected, bool isMarkdigMdRenderingEnabled) { var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); From 7c4d9fe18a47161caddbb7c7ea7ed920b61aa71a Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Mon, 16 Nov 2020 10:54:17 -0800 Subject: [PATCH 14/21] add unit test --- src/NuGetGallery/Services/MarkdownService.cs | 30 ++++- .../Services/MarkdownServiceFacts.cs | 108 +++++++++++++++++- 2 files changed, 131 insertions(+), 7 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index f0c932104f..bc4e8d6dd2 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -159,9 +159,20 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); var pipeline = new MarkdownPipelineBuilder() + .UseGridTables() + .UsePipeTables() + .UseListExtras() + .UseTaskLists() .UseSoftlineBreakAsHardlineBreak() + .UseEmojiAndSmiley() + .UseReferralLinks("nofollow") + .UseAutoLinks() .Build(); + StringWriter htmlWriter = new StringWriter(); + var renderer = new HtmlRenderer(htmlWriter); + pipeline.Setup(renderer); + var document = Markdown.Parse(encodedMarkdown, pipeline); foreach (var node in document.Descendants()) @@ -174,6 +185,20 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); } + // Decode preformatted blocks to prevent double encoding. + // Skip BlockTag.BlockQuote, which are partially decoded upfront. + if (node is FencedCodeBlock || node is CodeBlock) + { + LeafBlock codeBlock = (LeafBlock)node; + var lines =codeBlock.Lines; + for (int i = 0; i < lines.Count; i++) + { + var content = lines.Lines[i].Slice.ToString(); + var unencodedContent = HttpUtility.HtmlDecode(content); + lines.Lines[i].Slice = new Markdig.Helpers.StringSlice(unencodedContent); + } + } + } else if (node is Markdig.Syntax.Inlines.Inline) { @@ -207,12 +232,9 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, } } - StringWriter htmlWriter = new StringWriter(); - var renderer = new HtmlRenderer(htmlWriter); renderer.Render(document); htmlWriter.Flush(); - //manually inject nofollow since markdig doesn't support inject nofollw in encode - output.Content = LinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim(); + output.Content = htmlWriter.ToString().Trim(); return output; } } diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 15f368787a..2e2fbf8b8d 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -2,17 +2,19 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Moq; +using System.Data.Entity.Core.Objects; using Xunit; namespace NuGetGallery { public class MarkdownServiceFacts { - + public class GetReadMeHtmlMethod { private readonly MarkdownService _markdownService; private readonly Mock _featureFlagService; + private bool isMarkdigMdRenderingEnabled = true; public GetReadMeHtmlMethod() { @@ -38,13 +40,13 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("# Heading", "

Heading

", false, false)] [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, true)] [InlineData("\ufeff# Heading with BOM", "

Heading with BOM

", false, false)] - [InlineData("- List", "
    \r\n
  • List
    • \r\n
", false, true)] + [InlineData("- List", "
    \n
  • List
    • \n
", false, true)] [InlineData("- List", "
    \r\n
  • List
    • \r\n
", false, false)] [InlineData("[text](http://www.test.com)", "

text

", false, true)] [InlineData("[text](http://www.test.com)", "

text

", false, false)] [InlineData("[text](javascript:alert('hi'))", "

text

", false, true)] [InlineData("[text](javascript:alert('hi'))", "

text

", false, false)] - [InlineData("> Blockquote", "
\r\n

<text>Blockquote</text>

\r\n
", false, true)] + [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, true)] [InlineData("> Blockquote", "
\r\n

<text>Blockquote</text>

\r\n
", false, false)] [InlineData("[text](http://www.asp.net)", "

text

", false, true)] [InlineData("[text](http://www.asp.net)", "

text

", false, false)] @@ -56,12 +58,112 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("![image](https://www.asp.net/fake.jpg)", "

\"image\"

", false, false)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, true)] [InlineData("![image](http://www.otherurl.net/fake.jpg)", "

\"image\"

", true, false)] + [InlineData("## License\n\tLicensed under the Apache License, Version 2.0 (the \"License\");", "

License

\n
Licensed under the Apache License, Version 2.0 (the "License");\n
", false, true)] + [InlineData("## License\n\tLicensed under the Apache License, Version 2.0 (the \"License\");", "

License

\n
Licensed under the Apache License, Version 2.0 (the "License");\n
", false, true)] public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool imageRewriteExpected, bool isMarkdigMdRenderingEnabled) { + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten); } + + [Fact] + public void TestToHtmlWithExtension() + { + var originalMd = "This is a paragraph\r\n with a break inside"; + var expectedHtml = "

This is a paragraph
\nwith a break inside

"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + [Fact] + public void TestToHtmlWithPipeTable() { + var originalMd = @"a | b +-- | - +0 | 1"; + + var expectedHtml = "\n\n\n\n\n\n\n\n\n\n\n\n\n
ab
01
"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + [Fact] + public void TestToHtmlWithGridTable() + { + var originalMd = @"+---+---+ +| a | b | ++===+===+ +| 1 | 2 | ++---+---+ +"; + + var expectedHtml = "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ab
12
"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + [Fact] + public void TestToHtmlWithEmojiAndSmiley() + { + var originalMd = "This is a test with a :) and a :angry: smiley"; + + var expectedHtml = "

This is a test with a 😃 and a 😠 smiley

"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + [Fact] + public void TestToHtmlWithTaskLists() + { + var originalMd = @"- [ ] Item1 +- [x] Item2 +- [ ] Item3 +- Item4"; + + var expectedHtml = "
    \n
  • Item1
    • \n
  • " + + " Item2
    • \n
  • " + + "Item3
    • \n
  • Item4
    • \n
"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + [Fact] + public void TestToHtmlWithAddtionalList() + { + var originalMd = @"1. First item + +Some text + +2. Second item"; + + var expectedHtml = "
    \n
  1. First item
    2. \n
\n

Some text

\n
    \n
  1. Second item
    2. \n
"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } + + public void TestToHtmlWithAutoLinks() + { + var originalMd = "This is a http://www.google.com URL and https://www.google.com"; + + var expectedHtml = "

This is a https://www.google.com URL and https://www.google.com"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } } } } From 40735cbb9cb75bd95bd517a5e0f95154b60f1aa4 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Sun, 22 Nov 2020 22:20:19 -0800 Subject: [PATCH 15/21] Resolve conflict --- .../DatabaseMigrationTools.csproj | 1 - .../GitHubVulnerabilities2Db.csproj | 1 - src/NuGetGallery.Core/NuGetGallery.Core.csproj | 11 ----------- .../NuGetGallery.Services.csproj | 4 ---- src/NuGetGallery/NuGetGallery.csproj | 10 ---------- 5 files changed, 27 deletions(-) diff --git a/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj b/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj index 6bf333d3e5..acffa19906 100644 --- a/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj +++ b/src/DatabaseMigrationTools/DatabaseMigrationTools.csproj @@ -65,7 +65,6 @@ all - 2.78.0 2.79.0 diff --git a/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj b/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj index 2d2cf5f588..02e4059e1a 100644 --- a/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj +++ b/src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj @@ -89,7 +89,6 @@ 4.3.0-dev-3612825 - 2.78.0 2.79.0 diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index 0768dfdb46..62d4981e83 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -39,7 +39,6 @@ 5.8.0-preview.3.6823 - 2.78.0 2.79.0 @@ -49,16 +48,6 @@ - 2.78.0 - - - 2.78.0 - - - 2.78.0 - - - 1.2.0 2.79.0 diff --git a/src/NuGetGallery.Services/NuGetGallery.Services.csproj b/src/NuGetGallery.Services/NuGetGallery.Services.csproj index 9b47b74f38..f3cf84c4ca 100644 --- a/src/NuGetGallery.Services/NuGetGallery.Services.csproj +++ b/src/NuGetGallery.Services/NuGetGallery.Services.csproj @@ -305,10 +305,6 @@ 5.8.0-preview.3.6823 - 2.78.0 - - - 2.78.0 2.79.0 diff --git a/src/NuGetGallery/NuGetGallery.csproj b/src/NuGetGallery/NuGetGallery.csproj index 118a642512..25d34cef9f 100644 --- a/src/NuGetGallery/NuGetGallery.csproj +++ b/src/NuGetGallery/NuGetGallery.csproj @@ -2063,9 +2063,6 @@ 4.3.1 - - 0.22.0 - 2.2.0 @@ -2240,13 +2237,6 @@ 5.8.0-preview.3.6823 - 2.78.0 - - - 2.78.0 - - - 2.78.0 2.79.0 From 8729da12ee0526094e818cb9e3b81b46fae38fe9 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Sun, 22 Nov 2020 23:34:34 -0800 Subject: [PATCH 16/21] style change --- src/NuGetGallery/Services/MarkdownService.cs | 93 ++++++++++---------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index bc4e8d6dd2..d7954fd49f 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -52,7 +52,6 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc { return GetHtmlFromMarkdownCommonMark(markdownString, incrementHeadersBy); } - } private RenderedMarkdownResult GetHtmlFromMarkdownCommonMark(string markdownString, int incrementHeadersBy) @@ -169,73 +168,73 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, .UseAutoLinks() .Build(); - StringWriter htmlWriter = new StringWriter(); - var renderer = new HtmlRenderer(htmlWriter); - pipeline.Setup(renderer); + using(StringWriter htmlWriter = new StringWriter()) + { + var renderer = new HtmlRenderer(htmlWriter); + pipeline.Setup(renderer); - var document = Markdown.Parse(encodedMarkdown, pipeline); + var document = Markdown.Parse(encodedMarkdown, pipeline); - foreach (var node in document.Descendants()) - { - if (node is Markdig.Syntax.Block) + foreach (var node in document.Descendants()) { - // Demote heading tags so they don't overpower expander headings. - if (node is HeadingBlock heading) + if (node is Markdig.Syntax.Block) { - heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); - } - - // Decode preformatted blocks to prevent double encoding. - // Skip BlockTag.BlockQuote, which are partially decoded upfront. - if (node is FencedCodeBlock || node is CodeBlock) - { - LeafBlock codeBlock = (LeafBlock)node; - var lines =codeBlock.Lines; - for (int i = 0; i < lines.Count; i++) + // Demote heading tags so they don't overpower expander headings. + if (node is HeadingBlock heading) { - var content = lines.Lines[i].Slice.ToString(); - var unencodedContent = HttpUtility.HtmlDecode(content); - lines.Lines[i].Slice = new Markdig.Helpers.StringSlice(unencodedContent); + heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); } - } - } - else if (node is Markdig.Syntax.Inlines.Inline) - { - if (node is LinkInline linkInline) - { - if (linkInline.IsImage) + // Decode preformatted blocks to prevent double encoding. + // Skip BlockTag.BlockQuote, which are partially decoded upfront. + if (node is FencedCodeBlock || node is CodeBlock) { - // Allow only http or https links in markdown. Transform link to https for known domains. - if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true)) - { - linkInline.Url = string.Empty; - } - else + LeafBlock codeBlock = (LeafBlock)node; + var lines =codeBlock.Lines; + for (int i = 0; i < lines.Count; i++) { - output.ImagesRewritten = output.ImagesRewritten || (linkInline.Url != readyUriString); - linkInline.Url = readyUriString; + var content = lines.Lines[i].Slice.ToString(); + var unencodedContent = HttpUtility.HtmlDecode(content); + lines.Lines[i].Slice = new Markdig.Helpers.StringSlice(unencodedContent); } } - else + } + else if (node is Markdig.Syntax.Inlines.Inline) + { + if (node is LinkInline linkInline) { - if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString)) + if (linkInline.IsImage) { - linkInline.Url = string.Empty; + // Allow only http or https links in markdown. Transform link to https for known domains. + if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true)) + { + linkInline.Url = string.Empty; + } + else + { + output.ImagesRewritten = output.ImagesRewritten || (linkInline.Url != readyUriString); + linkInline.Url = readyUriString; + } } else { - linkInline.Url = readyUriString; + if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString)) + { + linkInline.Url = string.Empty; + } + else + { + linkInline.Url = readyUriString; + } } } } } - } - renderer.Render(document); - htmlWriter.Flush(); - output.Content = htmlWriter.ToString().Trim(); - return output; + renderer.Render(document); + output.Content = htmlWriter.ToString().Trim(); + return output; + } } } } \ No newline at end of file From 01de1b63efba0274277a6aadb03997e7f99960af Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Tue, 1 Dec 2020 23:02:09 -0800 Subject: [PATCH 17/21] add disable html extension --- .../Configuration/IFeatureFlagService.cs | 2 +- src/NuGetGallery/Services/MarkdownService.cs | 22 +++---------------- .../Services/MarkdownServiceFacts.cs | 3 ++- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs b/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs index 8984cc450f..635a3d1c57 100644 --- a/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs +++ b/src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs @@ -227,7 +227,7 @@ public interface IFeatureFlagService bool IsODataV2SearchCountNonHijackedEnabled(); ///

- /// Whether rending Markdown content to HTML using Markdig is enabaled + /// Whether rendering Markdown content to HTML using Markdig is enabled /// bool IsMarkdigMdRenderingEnabled(); diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index d7954fd49f..83bdcdfd95 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -154,9 +154,6 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, var readmeWithoutBom = markdownString.TrimStart('\ufeff'); - // HTML encode markdown, except for block quotes, to block inline html. - var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> "); - var pipeline = new MarkdownPipelineBuilder() .UseGridTables() .UsePipeTables() @@ -166,14 +163,15 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, .UseEmojiAndSmiley() .UseReferralLinks("nofollow") .UseAutoLinks() + .DisableHtml() //block inline html .Build(); - using(StringWriter htmlWriter = new StringWriter()) + using(var htmlWriter = new StringWriter()) { var renderer = new HtmlRenderer(htmlWriter); pipeline.Setup(renderer); - var document = Markdown.Parse(encodedMarkdown, pipeline); + var document = Markdown.Parse(readmeWithoutBom, pipeline); foreach (var node in document.Descendants()) { @@ -184,20 +182,6 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, { heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); } - - // Decode preformatted blocks to prevent double encoding. - // Skip BlockTag.BlockQuote, which are partially decoded upfront. - if (node is FencedCodeBlock || node is CodeBlock) - { - LeafBlock codeBlock = (LeafBlock)node; - var lines =codeBlock.Lines; - for (int i = 0; i < lines.Count; i++) - { - var content = lines.Lines[i].Slice.ToString(); - var unencodedContent = HttpUtility.HtmlDecode(content); - lines.Lines[i].Slice = new Markdig.Helpers.StringSlice(unencodedContent); - } - } } else if (node is Markdig.Syntax.Inlines.Inline) { diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 2e2fbf8b8d..e8cfa92636 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -80,7 +80,8 @@ public void TestToHtmlWithExtension() } [Fact] - public void TestToHtmlWithPipeTable() { + public void TestToHtmlWithPipeTable() + { var originalMd = @"a | b -- | - 0 | 1"; From 28272b65d0995dab63af4900bafd8422550e28fb Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Mon, 7 Dec 2020 20:41:26 -0800 Subject: [PATCH 18/21] address comments --- src/NuGetGallery/Services/MarkdownService.cs | 16 +++++++++-- .../Services/MarkdownServiceFacts.cs | 28 ++++++------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index 83bdcdfd95..d8991fb5e8 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -32,6 +32,12 @@ public MarkdownService(IFeatureFlagService features) public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) { + if (_features == null) + { + throw new ArgumentNullException(nameof(_features)); + + } + if (_features.IsMarkdigMdRenderingEnabled()) { return GetHtmlFromMarkdownMarkdig(markdownString, 1); @@ -44,6 +50,12 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy) { + if (_features == null) + { + throw new ArgumentNullException(nameof(_features)); + + } + if (_features.IsMarkdigMdRenderingEnabled()) { return GetHtmlFromMarkdownMarkdig(markdownString, incrementHeadersBy); @@ -162,11 +174,10 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, .UseSoftlineBreakAsHardlineBreak() .UseEmojiAndSmiley() .UseReferralLinks("nofollow") - .UseAutoLinks() .DisableHtml() //block inline html .Build(); - using(var htmlWriter = new StringWriter()) + using (var htmlWriter = new StringWriter()) { var renderer = new HtmlRenderer(htmlWriter); pipeline.Setup(renderer); @@ -202,6 +213,7 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, } else { + // Allow only http or https links in markdown. Transform link to https for known domains. if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString)) { linkInline.Url = string.Empty; diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index e8cfa92636..84d4dd94f9 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -1,8 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using Moq; using System.Data.Entity.Core.Objects; +using Moq; using Xunit; namespace NuGetGallery @@ -14,7 +14,6 @@ public class GetReadMeHtmlMethod { private readonly MarkdownService _markdownService; private readonly Mock _featureFlagService; - private bool isMarkdigMdRenderingEnabled = true; public GetReadMeHtmlMethod() { @@ -48,6 +47,8 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i [InlineData("[text](javascript:alert('hi'))", "

text

", false, false)] [InlineData("> Blockquote", "
\n

<text>Blockquote</text>

\n
", false, true)] [InlineData("> Blockquote", "
\r\n

<text>Blockquote</text>

\r\n
", false, false)] + [InlineData("> > Blockquote", "
\n
\n

<text>Blockquote</text>

\n
\n
", false, true)] + [InlineData("> > Blockquote", "
\r\n

> <text>Blockquote</text>

\r\n
", false, false)] [InlineData("[text](http://www.asp.net)", "

text

", false, true)] [InlineData("[text](http://www.asp.net)", "

text

", false, false)] [InlineData("[text](badurl://www.asp.net)", "

text

", false, true)] @@ -73,7 +74,7 @@ public void TestToHtmlWithExtension() { var originalMd = "This is a paragraph\r\n with a break inside"; var expectedHtml = "

This is a paragraph
\nwith a break inside

"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); @@ -87,7 +88,7 @@ public void TestToHtmlWithPipeTable() 0 | 1"; var expectedHtml = "\n\n\n\n\n\n\n\n\n\n\n\n\n
ab
01
"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); @@ -104,7 +105,7 @@ public void TestToHtmlWithGridTable() "; var expectedHtml = "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ab
12
"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); @@ -116,7 +117,7 @@ public void TestToHtmlWithEmojiAndSmiley() var originalMd = "This is a test with a :) and a :angry: smiley"; var expectedHtml = "

This is a test with a 😃 and a 😠 smiley

"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); @@ -133,7 +134,7 @@ public void TestToHtmlWithTaskLists() var expectedHtml = "
    \n
  • Item1
    • \n
  • " + " Item2
    • \n
  • " + "Item3
    • \n
  • Item4
    • \n
"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); @@ -149,18 +150,7 @@ Some text 2. Second item"; var expectedHtml = "
    \n
  1. First item
    2. \n
\n

Some text

\n
    \n
  1. Second item
    2. \n
"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); - var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); - Assert.Equal(expectedHtml, readMeResult.Content); - Assert.Equal(false, readMeResult.ImagesRewritten); - } - - public void TestToHtmlWithAutoLinks() - { - var originalMd = "This is a http://www.google.com URL and https://www.google.com"; - - var expectedHtml = "

This is a https://www.google.com URL and https://www.google.com"; - _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); From bc7da900d78455b970ccf92f2d09537025c8cbf3 Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Thu, 10 Dec 2020 14:50:59 -0800 Subject: [PATCH 19/21] add tests and add validation for parameter --- src/NuGetGallery/Services/MarkdownService.cs | 17 +++++----- .../Services/MarkdownServiceFacts.cs | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index d8991fb5e8..764f69d466 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -4,6 +4,7 @@ using System; using System.IO; using System.Linq; +using System.Management; using System.Text.RegularExpressions; using System.Timers; using System.Web; @@ -32,10 +33,9 @@ public MarkdownService(IFeatureFlagService features) public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) { - if (_features == null) + if (markdownString == null) { - throw new ArgumentNullException(nameof(_features)); - + throw new ArgumentNullException(nameof(markdownString)); } if (_features.IsMarkdigMdRenderingEnabled()) @@ -50,10 +50,14 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString) public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy) { - if (_features == null) + if (markdownString == null) { - throw new ArgumentNullException(nameof(_features)); + throw new ArgumentNullException(nameof(markdownString)); + } + if (incrementHeadersBy < 0) + { + throw new ArgumentOutOfRangeException(nameof(incrementHeadersBy)); } if (_features.IsMarkdigMdRenderingEnabled()) @@ -191,7 +195,7 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, // Demote heading tags so they don't overpower expander headings. if (node is HeadingBlock heading) { - heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6); + heading.Level = Math.Min(heading.Level + incrementHeadersBy, 6); } } else if (node is Markdig.Syntax.Inlines.Inline) @@ -200,7 +204,6 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, { if (linkInline.IsImage) { - // Allow only http or https links in markdown. Transform link to https for known domains. if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true)) { linkInline.Url = string.Empty; diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 84d4dd94f9..49a90ef0ef 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Data.Entity.Core.Objects; using Moq; using Xunit; @@ -21,6 +22,19 @@ public GetReadMeHtmlMethod() _markdownService = new MarkdownService(_featureFlagService.Object); } + [Fact] + public void ThrowsArgumentOutOfRangeExceptionForNegativeParameterValue() + { + Assert.Throws(() => _markdownService.GetHtmlFromMarkdown("markdown file test", -1)); + } + + [Fact] + public void ThrowsArgumentNullExceptionForNullMarkdownString() + { + Assert.Throws(() => _markdownService.GetHtmlFromMarkdown(null, 0)); + Assert.Throws(() => _markdownService.GetHtmlFromMarkdown(null)); + } + [Theory] [InlineData("", "

<script>alert('test')</script>

", true)] [InlineData("", "

<script>alert('test')</script>

", false)] @@ -34,6 +48,23 @@ public void EncodesHtmlInMarkdown(string originalMd, string expectedHtml, bool i Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdown(originalMd).Content); } + [Theory] + [InlineData("# Heading", "

Heading

", true, 0)] + [InlineData("# Heading", "

Heading

", false, 0)] + [InlineData("# Heading", "

Heading

", true, 1)] + [InlineData("# Heading", "

Heading

", false, 1)] + [InlineData("# Heading", "
Heading
", true, 6)] + [InlineData("# Heading", "
Heading
", false, 6)] + [InlineData("# Heading", "
Heading
", true, 7)] + [InlineData("# Heading", "
Heading
", false, 7)] + [InlineData("# Heading", "
Heading
", true, 5)] + [InlineData("# Heading", "
Heading
", false, 5)] + public void EncodesHtmlInMarkdownWithAdaptiveHeader(string originalMd, string expectedHtml, bool isMarkdigMdRenderingEnabled, int incrementHeadersBy) + { + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(isMarkdigMdRenderingEnabled); + Assert.Equal(expectedHtml, _markdownService.GetHtmlFromMarkdown(originalMd, incrementHeadersBy).Content); + } + [Theory] [InlineData("# Heading", "

Heading

", false, true)] [InlineData("# Heading", "

Heading

", false, false)] From a0ef14ff4946f4f9b953f2e1a768563984ffc4ff Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Thu, 10 Dec 2020 16:10:44 -0800 Subject: [PATCH 20/21] add more test cases --- src/NuGetGallery/Services/MarkdownService.cs | 2 +- .../NuGetGallery.Facts/Services/MarkdownServiceFacts.cs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index 764f69d466..438f49a389 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -57,7 +57,7 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc if (incrementHeadersBy < 0) { - throw new ArgumentOutOfRangeException(nameof(incrementHeadersBy)); + throw new ArgumentOutOfRangeException(nameof(incrementHeadersBy), $"{nameof(incrementHeadersBy)} must be greater than or equal to 0"); } if (_features.IsMarkdigMdRenderingEnabled()) diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 49a90ef0ef..8eedae4c4b 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -22,10 +22,13 @@ public GetReadMeHtmlMethod() _markdownService = new MarkdownService(_featureFlagService.Object); } - [Fact] - public void ThrowsArgumentOutOfRangeExceptionForNegativeParameterValue() + [Theory] + [InlineData(-1)] + public void ThrowsArgumentOutOfRangeExceptionForNegativeIncrementHeadersBy(int incrementHeadersBy) { - Assert.Throws(() => _markdownService.GetHtmlFromMarkdown("markdown file test", -1)); + var exception = Assert.Throws(() => _markdownService.GetHtmlFromMarkdown("markdown file test", incrementHeadersBy)); + Assert.Equal(nameof(incrementHeadersBy), exception.ParamName); + Assert.Contains("must be greater than or equal to 0", exception.Message); } [Fact] From 22be88a4d3ccd791c36445a947ef50fdf83f833b Mon Sep 17 00:00:00 2001 From: Lynn Dai Date: Thu, 10 Dec 2020 18:28:25 -0800 Subject: [PATCH 21/21] add autolink and test --- src/NuGetGallery/Services/MarkdownService.cs | 1 + .../Services/MarkdownServiceFacts.cs | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/NuGetGallery/Services/MarkdownService.cs b/src/NuGetGallery/Services/MarkdownService.cs index 438f49a389..7cc54f5376 100644 --- a/src/NuGetGallery/Services/MarkdownService.cs +++ b/src/NuGetGallery/Services/MarkdownService.cs @@ -177,6 +177,7 @@ private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, .UseTaskLists() .UseSoftlineBreakAsHardlineBreak() .UseEmojiAndSmiley() + .UseAutoLinks() .UseReferralLinks("nofollow") .DisableHtml() //block inline html .Build(); diff --git a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs index 8eedae4c4b..d85c3b29be 100644 --- a/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs +++ b/tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs @@ -189,6 +189,18 @@ Some text Assert.Equal(expectedHtml, readMeResult.Content); Assert.Equal(false, readMeResult.ImagesRewritten); } + + [Fact] + public void TestToHtmlWithAutoLinks() + { + var originalMd = "This is a http://www.google.com URL and https://www.google.com"; + + var expectedHtml = "

This is a http://www.google.com URL and https://www.google.com

"; + _featureFlagService.Setup(x => x.IsMarkdigMdRenderingEnabled()).Returns(true); + var readMeResult = _markdownService.GetHtmlFromMarkdown(originalMd); + Assert.Equal(expectedHtml, readMeResult.Content); + Assert.Equal(false, readMeResult.ImagesRewritten); + } } } }