Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Markdig version update #8292

Merged
merged 22 commits into from
Dec 11, 2020
Merged

Markdig version update #8292

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1a1c6ca
update markdig to 0.22.0
Oct 23, 2020
616471d
updated markdig.signed.dll in third party props
Oct 23, 2020
53186dd
test with newversion package with updated markdig version
Oct 27, 2020
ebb7557
version test
Oct 27, 2020
a2e8cae
update the version of nuget.services
Oct 27, 2020
44c8a6b
a
Oct 27, 2020
0e925e7
migrate to markdig
Oct 29, 2020
b866e4b
rm legacy to test markdown with markdig
Oct 29, 2020
76109a0
update function
Oct 29, 2020
c133a6a
add feature flag
Nov 9, 2020
3c3fcb2
change unit test
Nov 12, 2020
79678a1
unit test
Nov 12, 2020
fc338a9
unit test
Nov 12, 2020
7c4d9fe
add unit test
Nov 16, 2020
d4584b4
merge
Nov 23, 2020
40735cb
Resolve conflict
Nov 23, 2020
8729da1
style change
Nov 23, 2020
01de1b6
add disable html extension
Dec 2, 2020
28272b6
address comments
Dec 8, 2020
bc7da90
add tests and add validation for parameter
Dec 10, 2020
a0ef14f
add more test cases
Dec 11, 2020
22be88a
add autolink and test
Dec 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sign.thirdparty.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Snowball.dll" />
<ThirdPartyBinaries Include="Lucene.Net.Contrib.SpellChecker.dll" />
<ThirdPartyBinaries Include="Lucene.Net.dll" />
<ThirdPartyBinaries Include="Markdig.dll" />
<ThirdPartyBinaries Include="Markdig.Signed.dll" />
<ThirdPartyBinaries Include="MarkdownSharp.dll" />
<ThirdPartyBinaries Include="Microsoft.ApplicationServer.Caching.Client.dll" />
<ThirdPartyBinaries Include="Microsoft.ApplicationServer.Caching.Core.dll" />
Expand Down
2 changes: 1 addition & 1 deletion src/DatabaseMigrationTools/DatabaseMigrationTools.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="NuGet.Services.Validation">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion src/GitHubVulnerabilities2Db/GitHubVulnerabilities2Db.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@
<Version>4.3.0-dev-3612825</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Cursor">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>
Expand Down
8 changes: 4 additions & 4 deletions src/NuGetGallery.Core/NuGetGallery.Core.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,16 +245,16 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="NuGet.Services.FeatureFlags">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Messaging.Email">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Validation">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Validation.Issues">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.StrongName.AnglicanGeek.MarkdownMailer">
<Version>1.2.0</Version>
Expand Down
4 changes: 2 additions & 2 deletions src/NuGetGallery.Services/NuGetGallery.Services.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,10 @@
<Version>5.8.0-preview.3.6823</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Configuration">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Logging">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.StrongName.WebBackgrounder">
<Version>0.2.0</Version>
Expand Down
8 changes: 4 additions & 4 deletions src/NuGetGallery/NuGetGallery.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,7 +2063,7 @@
<Version>4.3.1</Version>
</PackageReference>
<PackageReference Include="Markdig.Signed">
<Version>0.15.4</Version>
<Version>0.22.0</Version>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.DependencyInjection">
<Version>2.2.0</Version>
Expand Down Expand Up @@ -2245,13 +2245,13 @@
<Version>5.8.0-preview.3.6823</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Licenses">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Owin">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Sql">
<Version>2.74.0</Version>
<Version>2.78.0</Version>
</PackageReference>
<PackageReference Include="Owin">
<Version>1.0.0</Version>
Expand Down
15 changes: 15 additions & 0 deletions src/NuGetGallery/Services/IMarkdownService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,20 @@ public interface IMarkdownService
/// <param name="incrementHeadersBy">headers can be incremented by this value, eg if 2 supplied then h1 will become h3</param>
/// <returns>HTML data</returns>
RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy);

/// <summary>
/// Returns HTML from the supplied markdown using Markdig
/// </summary>
/// <param name="markdownString">markdown input</param>
/// <returns>HTML data</returns>
RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString);

/// <summary>
/// Returns HTML from the supplied markdown using Markdig
/// </summary>
/// <param name="markdownString">markdown input</param>
/// <param name="incrementHeadersBy">headers can be incremented by this value, eg if 2 supplied then h1 will become h3</param>
/// <returns>HTML data</returns>
RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy);
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
}
}
79 changes: 79 additions & 0 deletions src/NuGetGallery/Services/MarkdownService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,17 @@

using System;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Timers;
using System.Web;
using CommonMark;
using CommonMark.Syntax;
loic-sharma marked this conversation as resolved.
Show resolved Hide resolved
using Markdig;
using Markdig.Parsers;
using Markdig.Renderers;
using Markdig.Syntax;
using Markdig.Syntax.Inlines;

namespace NuGetGallery
{
Expand All @@ -21,6 +28,11 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString)
return GetHtmlFromMarkdown(markdownString, 1);
}

public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString)
{
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
return GetHtmlFromMarkdownMarkdig(markdownString, 1);
}

public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy)
{
var output = new RenderedMarkdownResult()
Expand Down Expand Up @@ -110,5 +122,72 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc
return output;
}
}

public RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method seems to be used only here. Is the parametrization of the "incrementHeadersBy" needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Current we support adaptive header, in order to continue support the same functionality, I believe we still need "incrementHeaderBy"

{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any argument validation needed?

var output = new RenderedMarkdownResult()
{
ImagesRewritten = false,
Content = ""
};

var readmeWithoutBom = markdownString.StartsWith("\ufeff") ? markdownString.Replace("\ufeff", "") : markdownString;
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
var encodedMarkdown = EncodedBlockQuotePattern.Replace(HttpUtility.HtmlEncode(readmeWithoutBom), "> ");
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved

var pipeline = new MarkdownPipelineBuilder()
.UseSoftlineBreakAsHardlineBreak()
.Build();
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved

var document = Markdown.Parse(encodedMarkdown, pipeline);

foreach (var node in document.Descendants())
{
if (node is Markdig.Syntax.Block)
{
if (node is HeadingBlock heading)
{
heading.Level = (byte)Math.Min(heading.Level + incrementHeadersBy, 6);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe add a comment why 6?

}

}
else if (node is Markdig.Syntax.Inlines.Inline)
{
if (node is LinkInline linkInline)
{
if (linkInline.IsImage)
{
if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true))
{
linkInline.Url = string.Empty;
}
else
{
output.ImagesRewritten = output.ImagesRewritten || (linkInline.Url != readyUriString);
linkInline.Url = readyUriString;
}
}
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
else
{
if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString))
{
linkInline.Url = string.Empty;
}
else
{
linkInline.Url = readyUriString;
}
}
}
}
}

StringWriter htmlWriter = new StringWriter();
var renderer = new HtmlRenderer(htmlWriter);
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
renderer.Render(document);
htmlWriter.Flush();
output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim();

return output;
}
}
}
29 changes: 29 additions & 0 deletions tests/NuGetGallery.Facts/Services/MarkdownServiceFacts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,35 @@ public void ConvertsMarkdownToHtml(string originalMd, string expectedHtml, bool
Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten);
}

[Theory]
[InlineData("<script>alert('test')</script>", "<p>&lt;script&gt;alert('test')&lt;/script&gt;</p>")]
[InlineData("<img src=\"javascript:alert('test');\">", "<p>&lt;img src=&quot;javascript:alert('test');&quot;&gt;</p>")]
[InlineData("<a href=\"javascript:alert('test');\">", "<p>&lt;a href=&quot;javascript:alert('test');&quot;&gt;</p>")]
public void EncodesHtmlInMarkdownMarkdig(string originalMd, string expectedHtml)
{
Assert.Equal(expectedHtml, StripNewLines(
_markdownService.GetHtmlFromMarkdownMarkdig(originalMd).Content));
}

[Theory]
[InlineData("# Heading", "<h2>Heading</h2>", false)]
[InlineData("\ufeff# Heading with BOM", "<h2>Heading with BOM</h2>", false)]
[InlineData("- List", "<ul><li>List</li></ul>", false)]
[InlineData("[text](http://www.test.com)", "<p><a href=\"http://www.test.com/\" rel=\"nofollow\">text</a></p>", false)]
[InlineData("[text](javascript:alert('hi'))", "<p><a href=\"\" rel=\"nofollow\">text</a></p>", false)]
[InlineData("> <text>Blockquote</text>", "<blockquote><p>&lt;text&gt;Blockquote&lt;/text&gt;</p></blockquote>", false)]
[InlineData("[text](http://www.asp.net)", "<p><a href=\"https://www.asp.net/\" rel=\"nofollow\">text</a></p>", false)]
[InlineData("[text](badurl://www.asp.net)", "<p><a href=\"\" rel=\"nofollow\">text</a></p>", false)]
[InlineData("![image](http://www.asp.net/fake.jpg)", "<p><img src=\"https://www.asp.net/fake.jpg\" alt=\"image\" /></p>", true)]
[InlineData("![image](https://www.asp.net/fake.jpg)", "<p><img src=\"https://www.asp.net/fake.jpg\" alt=\"image\" /></p>", false)]
[InlineData("![image](http://www.otherurl.net/fake.jpg)", "<p><img src=\"https://www.otherurl.net/fake.jpg\" alt=\"image\" /></p>", true)]
public void ConvertsMarkdownToHtmlMarkdig(string originalMd, string expectedHtml, bool imageRewriteExpected)
{
var readMeResult = _markdownService.GetHtmlFromMarkdownMarkdig(originalMd);
Assert.Equal(expectedHtml, StripNewLines(readMeResult.Content));
lyndaidaii marked this conversation as resolved.
Show resolved Hide resolved
Assert.Equal(imageRewriteExpected, readMeResult.ImagesRewritten);
}

private static string StripNewLines(string text)
{
return text.Replace("\r\n", "").Replace("\n", "");
Expand Down