From 1a3186aa252bb91017e3da3e23e749c9b70d45dc Mon Sep 17 00:00:00 2001 From: Avi Avni Date: Sun, 19 Jun 2016 06:16:17 +0300 Subject: [PATCH 001/108] add range to MarkdownParagraph and MarkdownSpan --- src/Common/StringParsing.fs | 106 +++++++++--- src/FSharp.CodeFormat/CommentFilter.fs | 10 +- src/FSharp.Literate/Document.fs | 2 +- src/FSharp.Literate/Evaluator.fs | 8 +- src/FSharp.Literate/Formatting.fs | 10 +- src/FSharp.Literate/Main.fs | 4 +- src/FSharp.Literate/ParseScript.fs | 24 +-- src/FSharp.Literate/Transformations.fs | 54 +++---- src/FSharp.Markdown/HtmlFormatting.fs | 64 ++++---- src/FSharp.Markdown/LatexFormatting.fs | 48 +++--- src/FSharp.Markdown/Main.fs | 10 +- src/FSharp.Markdown/Markdown.fs | 96 +++++------ src/FSharp.Markdown/MarkdownParser.fs | 195 ++++++++++++----------- src/FSharp.MetadataFormat/Main.fs | 14 +- tests/FSharp.Literate.Tests/EvalTests.fs | 16 +- tests/FSharp.Literate.Tests/Tests.fs | 26 +-- tests/FSharp.Markdown.Tests/Markdown.fs | 36 ++--- 17 files changed, 394 insertions(+), 329 deletions(-) diff --git a/src/Common/StringParsing.fs b/src/Common/StringParsing.fs index 0f376562a..8d2ce0d8c 100644 --- a/src/Common/StringParsing.fs +++ b/src/Common/StringParsing.fs @@ -14,44 +14,57 @@ open FSharp.Collections module String = /// Matches when a string is a whitespace or null - let (|WhiteSpace|_|) s = + let (|WhiteSpaceS|_|) (s) = + if String.IsNullOrWhiteSpace(s) then Some() else None + + /// Matches when a string is a whitespace or null + let (|WhiteSpace|_|) (s, n: int) = if String.IsNullOrWhiteSpace(s) then Some() else None /// Matches when a string does starts with non-whitespace - let (|Unindented|_|) (s:string) = + let (|Unindented|_|) (s:string, n:int) = if not (String.IsNullOrWhiteSpace(s)) && s.TrimStart() = s then Some() else None /// Returns a string trimmed from both start and end - let (|TrimBoth|) (text:string) = text.Trim() + let (|TrimBothS|) (text:string) = text.Trim() + /// Returns a string trimmed from both start and end + let (|TrimBoth|) (text:string, n:int) = (text.Trim(), n) /// Returns a string trimmed from the end - let (|TrimEnd|) (text:string) = text.TrimEnd() + let (|TrimEnd|) (text:string, n:int) = (text.TrimEnd(), n) /// Returns a string trimmed from the start - let (|TrimStart|) (text:string) = text.TrimStart() + let (|TrimStart|) (text:string, n:int) = (text.TrimStart(), n) /// Retrusn a string trimmed from the end using characters given as a parameter - let (|TrimEndUsing|) chars (text:string) = text.TrimEnd(Array.ofSeq chars) + let (|TrimEndUsing|) chars (text:string, n:int) = text.TrimEnd(Array.ofSeq chars) /// Returns a string trimmed from the start together with /// the number of skipped whitespace characters - let (|TrimStartAndCount|) (text:string) = + let (|TrimStartAndCount|) (text:string, n:int) = let trimmed = text.TrimStart([|' '; '\t'|]) let len = text.Length - trimmed.Length - len, text.Substring(0, len).Replace("\t", " ").Length, trimmed + len, text.Substring(0, len).Replace("\t", " ").Length, (trimmed, n) /// Matches when a string starts with any of the specified sub-strings - let (|StartsWithAny|_|) (starts:seq) (text:string) = + let (|StartsWithAny|_|) (starts:seq) (text:string, n:int) = if starts |> Seq.exists (text.StartsWith) then Some() else None /// Matches when a string starts with the specified sub-string - let (|StartsWith|_|) (start:string) (text:string) = + let (|StartsWithS|_|) (start:string) (text:string) = if text.StartsWith(start) then Some(text.Substring(start.Length)) else None /// Matches when a string starts with the specified sub-string + let (|StartsWith|_|) (start:string) (text:string, n:int) = + if text.StartsWith(start) then Some(text.Substring(start.Length), n) else None + /// Matches when a string starts with the specified sub-string /// The matched string is trimmed from all whitespace. - let (|StartsWithTrim|_|) (start:string) (text:string) = + let (|StartsWithTrimS|_|) (start:string) (text:string) = if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None + /// Matches when a string starts with the specified sub-string + /// The matched string is trimmed from all whitespace. + let (|StartsWithTrim|_|) (start:string) (text:string, n:int) = + if text.StartsWith(start) then Some(text.Substring(start.Length).Trim(), n) else None /// Matches when a string starts with the specified sub-string (ignoring whitespace at the start) /// The matched string is trimmed from all whitespace. - let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string) = + let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string, n:int) = if text.Contains(start) then let beforeStart = text.Substring(0, text.IndexOf(start)) if String.IsNullOrWhiteSpace (beforeStart) then @@ -67,12 +80,26 @@ module String = /// Matches when a string starts with the given value and ends /// with a given value (and returns the rest of it) - let (|StartsAndEndsWith|_|) (starts, ends) (s:string) = + let (|StartsAndEndsWithS|_|) (starts, ends) (s:string) = if s.StartsWith(starts) && s.EndsWith(ends) && s.Length >= starts.Length + ends.Length then Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length)) else None + /// Matches when a string starts with the given value and ends + /// with a given value (and returns the rest of it) + let (|StartsAndEndsWith|_|) (starts, ends) (s:string, n:int) = + if s.StartsWith(starts) && s.EndsWith(ends) && + s.Length >= starts.Length + ends.Length then + Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length), n) + else None + + /// Matches when a string starts with the given value and ends + /// with a given value (and returns trimmed body) + let (|StartsAndEndsWithTrimS|_|) args = function + | StartsAndEndsWithS args (TrimBothS res) -> Some res + | _ -> None + /// Matches when a string starts with the given value and ends /// with a given value (and returns trimmed body) let (|StartsAndEndsWithTrim|_|) args = function @@ -85,7 +112,7 @@ module String = /// /// let (StartsWithRepeated "/\" (2, " abc")) = "/\/\ abc" /// - let (|StartsWithRepeated|_|) (repeated:string) (text:string) = + let (|StartsWithRepeated|_|) (repeated:string) (text:string, ln:int) = let rec loop i = if i = text.Length then i elif text.[i] <> repeated.[i % repeated.Length] then i @@ -93,7 +120,7 @@ module String = let n = loop 0 if n = 0 || n % repeated.Length <> 0 then None - else Some(n/repeated.Length, text.Substring(n, text.Length - n)) + else Some(n/repeated.Length, (text.Substring(n, text.Length - n), ln)) /// Ignores everything until a end-line character is detected, returns the remaining string. let (|SkipSingleLine|) (text:string) = @@ -114,12 +141,11 @@ module String = FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text result - /// Matches when a string starts with a sub-string wrapped using the /// opening and closing sub-string specified in the parameter. /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped /// text together with the rest. - let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) = + let (|StartsWithWrappedS|_|) (starts:string, ends:string) (text:string) = if text.StartsWith(starts) then let id = text.IndexOf(ends, starts.Length) if id >= 0 then @@ -129,10 +155,24 @@ module String = else None else None + /// Matches when a string starts with a sub-string wrapped using the + /// opening and closing sub-string specified in the parameter. + /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped + /// text together with the rest. + let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string, n:int) = + if text.StartsWith(starts) then + let id = text.IndexOf(ends, starts.Length) + if id >= 0 then + let wrapped = text.Substring(starts.Length, id - starts.Length) + let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length) + Some(wrapped, (rest, n)) + else None + else None + /// Matches when a string consists of some number of /// complete repetitions of a specified sub-string. - let (|EqualsRepeated|_|) repeated = function - | StartsWithRepeated repeated (n, "") -> Some() + let (|EqualsRepeated|_|) (repeated, n:int) = function + | StartsWithRepeated repeated (n, ("", _)) -> Some() | _ -> None /// Given a list of lines indented with certan number of whitespace @@ -197,8 +237,8 @@ module Lines = /// Removes blank lines from the start and the end of a list let (|TrimBlank|) lines = lines - |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev - |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev + |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev + |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev /// Matches when there are some lines at the beginning that are /// either empty (or whitespace) or start with the specified string. @@ -213,7 +253,7 @@ module Lines = /// either empty (or whitespace) or start with at least 4 spaces (a tab counts as 4 spaces here). /// Returns all such lines from the beginning until a different line and /// the number of spaces the first line started with. - let (|TakeCodeBlock|_|) (input:string list) = + let (|TakeCodeBlock|_|) (input:(string * int) list) = let spaceNum = 4 //match input with //| h :: _ -> @@ -225,20 +265,20 @@ module Lines = let normalized = s.Replace("\t", " ") normalized.Length >= spaceNum && normalized.Substring(0, spaceNum) = System.String(' ', spaceNum) - match List.partitionWhile (fun s -> + match List.partitionWhile (fun (s, n) -> String.IsNullOrWhiteSpace s || startsWithSpaces s) input with | matching, rest when matching <> [] && spaceNum >= 4 -> Some(spaceNum, matching, rest) | _ -> None /// Removes whitespace lines from the beginning of the list - let (|TrimBlankStart|) = List.skipWhile (String.IsNullOrWhiteSpace) + let (|TrimBlankStart|) = List.skipWhile (fun (s:string, n:int) -> String.IsNullOrWhiteSpace s) /// Trims all lines of the current paragraph let (|TrimParagraphLines|) lines = lines // first remove all whitespace on the beginning of the line - |> List.map (fun (s:string) -> s.TrimStart()) + |> List.map (fun (s:string, n:int) -> s.TrimStart()) // Now remove all additional spaces at the end, but keep two spaces if existent |> List.map (fun s -> let endsWithTwoSpaces = s.EndsWith(" ") @@ -258,7 +298,21 @@ open System.Collections.Generic /// recognize `key1=value, key2=value` and also `key1:value, key2:value` /// The key of the command should be identifier with just /// characters in it - otherwise, the parsing fails. -let (|ParseCommands|_|) (str:string) = +let (|ParseCommandsS|_|) (str:string) = + let kvs = + [ for cmd in str.Split(',') do + let kv = cmd.Split([| '='; ':' |]) + if kv.Length = 2 then yield kv.[0].Trim(), kv.[1].Trim() + elif kv.Length = 1 then yield kv.[0].Trim(), "" ] + let allKeysValid = + kvs |> Seq.forall (fst >> Seq.forall (fun c -> Char.IsLetter c || c = '_' || c = '-')) + if allKeysValid && kvs <> [] then Some(dict kvs) else None + +/// Utility for parsing commands. Commands can be used in different places. We +/// recognize `key1=value, key2=value` and also `key1:value, key2:value` +/// The key of the command should be identifier with just +/// characters in it - otherwise, the parsing fails. +let (|ParseCommands|_|) (str:string, n:int) = let kvs = [ for cmd in str.Split(',') do let kv = cmd.Split([| '='; ':' |]) diff --git a/src/FSharp.CodeFormat/CommentFilter.fs b/src/FSharp.CodeFormat/CommentFilter.fs index 279f5e176..8364862e2 100644 --- a/src/FSharp.CodeFormat/CommentFilter.fs +++ b/src/FSharp.CodeFormat/CommentFilter.fs @@ -49,12 +49,12 @@ let rec getSnippets (state:NamedSnippet option) (snippets:NamedSnippet list) match source with | [] -> snippets | (line, tokens)::rest -> - let text = lines.[line].Trim() + let text = lines.[line].Trim(), line match state, text with // We're not inside a snippet and we found a beginning of one | None, String.StartsWithTrim "//" (String.StartsWithTrim "[snippet:" title) -> - let title = title.Substring(0, title.IndexOf(']')) + let title = (fst title).Substring(0, (fst title).IndexOf(']')) getSnippets (Some(title, [])) snippets rest lines // Not inside a snippet and there is a usual line | None, _ -> @@ -92,7 +92,7 @@ let rec shrinkOmittedCode (text:StringBuilder) line content (source:Snippet) = // Take the next line, merge comments and continue looking for end | [], (line, content)::source -> shrinkOmittedCode (text.Append("\n")) line (mergeComments content None []) source - | (String.StartsAndEndsWithTrim ("(*", "*)") "[/omit]", tok)::rest, source + | (String.StartsAndEndsWithTrimS ("(*", "*)") "[/omit]", tok)::rest, source when tok.TokenName = "COMMENT" -> line, rest, source, text | (str, tok)::rest, _ -> @@ -105,13 +105,13 @@ let rec shrinkOmittedCode (text:StringBuilder) line content (source:Snippet) = let rec shrinkLine line (content:SnippetLine) (source:Snippet) = match content with | [] -> [], source - | (String.StartsAndEndsWithTrim ("(*", "*)") (String.StartsAndEndsWithTrim ("[omit:", "]") body), (tok:FSharpTokenInfo))::rest + | (String.StartsAndEndsWithTrimS ("(*", "*)") (String.StartsAndEndsWithTrimS ("[omit:", "]") body), (tok:FSharpTokenInfo))::rest when tok.TokenName = "COMMENT" -> let line, remcontent, source, text = shrinkOmittedCode (StringBuilder()) line rest source let line, source = shrinkLine line remcontent source (body, { tok with TokenName = "OMIT" + (text.ToString()) })::line, source - | (String.StartsWithTrim "//" (String.StartsAndEndsWith ("[fsi:", "]") fsi), (tok:FSharpTokenInfo))::rest -> + | (String.StartsWithTrimS "//" (String.StartsAndEndsWithS ("[fsi:", "]") fsi), (tok:FSharpTokenInfo))::rest -> let line, source = shrinkLine line rest source (fsi, { tok with TokenName = "FSI"})::line, source | (str, tok)::rest -> diff --git a/src/FSharp.Literate/Document.fs b/src/FSharp.Literate/Document.fs index 0441373d4..e7bc9323b 100644 --- a/src/FSharp.Literate/Document.fs +++ b/src/FSharp.Literate/Document.fs @@ -112,4 +112,4 @@ type LiterateDocument(paragraphs, formattedTips, links, source, sourceFile, erro /// Markdown documents. module Matching = let (|LiterateParagraph|_|) = function - | EmbedParagraphs(:? LiterateParagraph as lp) -> Some lp | _ -> None \ No newline at end of file + | EmbedParagraphs(:? LiterateParagraph as lp, _) -> Some lp | _ -> None \ No newline at end of file diff --git a/src/FSharp.Literate/Evaluator.fs b/src/FSharp.Literate/Evaluator.fs index 27a87486f..2241114d5 100644 --- a/src/FSharp.Literate/Evaluator.fs +++ b/src/FSharp.Literate/Evaluator.fs @@ -111,7 +111,7 @@ type FsiEvaluator(?options:string[], ?fsiObj) = /// Registered transformations for pretty printing values /// (the default formats value as a string and emits single CodeBlock) let mutable valueTransformations = - [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "")]) ) ] + [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "", None)]) ) ] /// Register a function that formats (some) values that are produced by the evaluator. /// The specified function should return 'Some' when it knows how to format a value @@ -130,12 +130,12 @@ type FsiEvaluator(?options:string[], ?fsiObj) = match result :?> FsiEvaluationResult, kind with | result, FsiEmbedKind.Output -> let s = defaultArg result.Output "No output has been produced." - [ CodeBlock(s.Trim(), "", "") ] + [ CodeBlock(s.Trim(), "", "", None) ] | { ItValue = Some v }, FsiEmbedKind.ItValue | { Result = Some v }, FsiEmbedKind.Value -> valueTransformations |> Seq.pick (fun f -> lock lockObj (fun () -> f v)) - | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "") ] - | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "") ] + | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "", None) ] + | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "", None) ] /// Evaluates the given text in an fsi session and returns /// an FsiEvaluationResult. diff --git a/src/FSharp.Literate/Formatting.fs b/src/FSharp.Literate/Formatting.fs index 9d02c7c23..93bbad5a3 100644 --- a/src/FSharp.Literate/Formatting.fs +++ b/src/FSharp.Literate/Formatting.fs @@ -27,8 +27,8 @@ module Formatting = /// Try find first-level heading in the paragraph collection let findHeadings paragraphs generateAnchors (outputKind:OutputKind) = paragraphs |> Seq.tryPick (function - | (Heading(1, text)) -> - let doc = MarkdownDocument([Span(text)], dict []) + | Heading(1, text, r) -> + let doc = MarkdownDocument([Span(text, r)], dict []) Some(format doc generateAnchors outputKind) | _ -> None) @@ -37,13 +37,13 @@ module Formatting = let getSourceDocument (doc:LiterateDocument) = match doc.Source with | LiterateSource.Markdown text -> - doc.With(paragraphs = [CodeBlock (text, "", "")]) + doc.With(paragraphs = [CodeBlock (text, "", "", None)]) | LiterateSource.Script snippets -> let paragraphs = [ for Snippet(name, lines) in snippets do if snippets.Length > 1 then - yield Heading(3, [Literal name]) - yield EmbedParagraphs(FormattedCode(lines)) ] + yield Heading(3, [Literal(name, None)], None) + yield EmbedParagraphs(FormattedCode(lines), None) ] doc.With(paragraphs = paragraphs) // -------------------------------------------------------------------------------------- diff --git a/src/FSharp.Literate/Main.fs b/src/FSharp.Literate/Main.fs index bc7b0cef0..37a7314ac 100644 --- a/src/FSharp.Literate/Main.fs +++ b/src/FSharp.Literate/Main.fs @@ -104,7 +104,7 @@ type Literate private () = static member WriteHtml(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) = let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None let doc = Transformations.replaceLiterateParagraphs ctx doc - let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks) + let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks) let sb = new System.Text.StringBuilder() use wr = new StringWriter(sb) Html.formatMarkdown wr ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs @@ -113,7 +113,7 @@ type Literate private () = static member WriteHtml(doc:LiterateDocument, writer:TextWriter, ?prefix, ?lineNumbers, ?generateAnchors) = let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None let doc = Transformations.replaceLiterateParagraphs ctx doc - let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks) + let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks) Html.formatMarkdown writer ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs static member WriteLatex(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) = diff --git a/src/FSharp.Literate/ParseScript.fs b/src/FSharp.Literate/ParseScript.fs index 70edf8308..af7fefc81 100644 --- a/src/FSharp.Literate/ParseScript.fs +++ b/src/FSharp.Literate/ParseScript.fs @@ -35,7 +35,7 @@ module internal CodeBlockUtils = let rec readComments inWhite acc = function | Token(TokenKind.Comment, text, _)::tokens when not inWhite-> readComments false (text::acc) tokens - | Token(TokenKind.Default, String.WhiteSpace _, _)::tokens -> + | Token(TokenKind.Default, String.WhiteSpaceS _, _)::tokens -> readComments true acc tokens | [] -> Some(String.concat "" (List.rev acc)) | _ -> None @@ -54,7 +54,7 @@ module internal CodeBlockUtils = cend match lines with - | (ConcatenatedComments(String.StartsAndEndsWith ("(***", "***)") (ParseCommands cmds)))::lines -> + | (ConcatenatedComments(String.StartsAndEndsWithS ("(***", "***)") (ParseCommandsS cmds)))::lines -> // Ended with a command, yield comment, command & parse the next as a snippet let cend = findCommentEnd comment yield BlockComment (comment.Substring(0, cend)) @@ -68,7 +68,7 @@ module internal CodeBlockUtils = yield BlockComment (comment.Substring(0, cend)) yield! collectSnippet [] lines - | (Line[Token(TokenKind.Comment, String.StartsWith "(**" text, _)])::lines -> + | (Line[Token(TokenKind.Comment, String.StartsWithS "(**" text, _)])::lines -> // Another block of Markdown comment starting... // Yield the previous snippet block and continue parsing more comments let cend = findCommentEnd comment @@ -94,13 +94,13 @@ module internal CodeBlockUtils = BlockSnippet res seq { match lines with - | (ConcatenatedComments(String.StartsAndEndsWith ("(***", "***)") (ParseCommands cmds)))::lines -> + | (ConcatenatedComments(String.StartsAndEndsWithS ("(***", "***)") (ParseCommandsS cmds)))::lines -> // Found a special command, yield snippet, command and parse another snippet if acc <> [] then yield blockSnippet acc yield BlockCommand cmds yield! collectSnippet [] lines - | (Line[Token(TokenKind.Comment, String.StartsWith "(**" text, _)])::lines -> + | (Line[Token(TokenKind.Comment, String.StartsWithS "(**" text, _)])::lines -> // Found a comment - yield snippet & switch to parsing comment state // (Also trim leading spaces to support e.g.: `(** ## Hello **)`) if acc <> [] then yield blockSnippet acc @@ -132,19 +132,19 @@ module internal ParseScript = // Reference to code snippet defined later | BlockCommand(Command "include" ref)::blocks -> - let p = EmbedParagraphs(CodeReference(ref)) + let p = EmbedParagraphs(CodeReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-output" ref)::blocks -> - let p = EmbedParagraphs(OutputReference(ref)) + let p = EmbedParagraphs(OutputReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-it" ref)::blocks -> - let p = EmbedParagraphs(ItValueReference(ref)) + let p = EmbedParagraphs(ItValueReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-value" ref)::blocks -> - let p = EmbedParagraphs(ValueReference(ref)) + let p = EmbedParagraphs(ValueReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "raw" _) ::BlockSnippet(snip):: blocks -> - let p = EmbedParagraphs(RawBlock(snip)) + let p = EmbedParagraphs(RawBlock(snip), None) transformBlocks noEval (p::acc) defs blocks // Parse commands in [foo=bar,zoo], followed by a source code snippet @@ -166,7 +166,7 @@ module internal ParseScript = { Evaluate = not (noEval || cmds.ContainsKey("do-not-eval")) OutputName = outputName Visibility = visibility } - let code = EmbedParagraphs(LiterateCode(snip, opts)) + let code = EmbedParagraphs(LiterateCode(snip, opts), None) transformBlocks noEval (code::acc) defs blocks // Unknown command @@ -178,7 +178,7 @@ module internal ParseScript = transformBlocks noEval acc defs blocks // Ordinary F# code snippet | BlockSnippet(snip)::blocks -> - let p = EmbedParagraphs(FormattedCode(snip)) + let p = EmbedParagraphs(FormattedCode(snip), None) transformBlocks noEval (p::acc) defs blocks // Markdown documentation block | BlockComment(text)::blocks -> diff --git a/src/FSharp.Literate/Transformations.fs b/src/FSharp.Literate/Transformations.fs index 141a42ca0..daba38a90 100644 --- a/src/FSharp.Literate/Transformations.fs +++ b/src/FSharp.Literate/Transformations.fs @@ -21,10 +21,10 @@ module Transformations = /// to colorize. We skip snippets that specify non-fsharp langauge e.g. [lang=csharp]. let rec collectCodeSnippets par = seq { match par with - | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _) + | CodeBlock((String.StartsWithWrappedS ("[", "]") (ParseCommandsS cmds, String.SkipSingleLine code)), language, _, _) when (not (String.IsNullOrWhiteSpace(language)) && language <> "fsharp") || (cmds.ContainsKey("lang") && cmds.["lang"] <> "fsharp") -> () - | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), _, _) - | CodeBlock(Let (dict []) (cmds, code), _, _) -> + | CodeBlock((String.StartsWithWrappedS ("[", "]") (ParseCommandsS cmds, String.SkipSingleLine code)), _, _, _) + | CodeBlock(Let (dict []) (cmds, code), _, _, _) -> let modul = match cmds.TryGetValue("module") with | true, v -> Some v | _ -> None @@ -39,8 +39,8 @@ module Transformations = /// Replace CodeBlock elements with formatted HTML that was processed by the F# snippets tool /// (The dictionary argument is a map from original code snippets to formatted HTML snippets.) let rec replaceCodeSnippets path (codeLookup:IDictionary<_, _>) = function - | CodeBlock ((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _) - | CodeBlock(Let (dict []) (cmds, code), language, _) -> + | CodeBlock ((String.StartsWithWrappedS ("[", "]") (ParseCommandsS cmds, String.SkipSingleLine code)), language, _, r) + | CodeBlock(Let (dict []) (cmds, code), language, _, r) -> if cmds.ContainsKey("hide") then None else let code = if cmds.ContainsKey("file") && cmds.ContainsKey("key") then @@ -56,12 +56,12 @@ module Transformations = else code let lang = match language with - | String.WhiteSpace when cmds.ContainsKey("lang") -> cmds.["lang"] + | String.WhiteSpaceS when cmds.ContainsKey("lang") -> cmds.["lang"] | language -> language if not (String.IsNullOrWhiteSpace(lang)) && lang <> "fsharp" then - Some (EmbedParagraphs(LanguageTaggedCode(lang, code))) + Some (EmbedParagraphs(LanguageTaggedCode(lang, code), r)) else - Some (EmbedParagraphs(FormattedCode(codeLookup.[code]))) + Some (EmbedParagraphs(FormattedCode(codeLookup.[code]), r)) // Recursively process nested paragraphs, other nodes return without change | Matching.ParagraphNested(pn, nested) -> @@ -116,7 +116,7 @@ module Transformations = // Collect IndirectLinks in a span let rec collectSpanReferences span = seq { match span with - | IndirectLink(_, _, key) -> yield key + | IndirectLink(_, _, key, _) -> yield key | Matching.SpanLeaf _ -> () | Matching.SpanNode(_, spans) -> for s in spans do yield! collectSpanReferences s } @@ -137,13 +137,13 @@ module Transformations = let replaceReferences (refIndex:IDictionary) = // Replace IndirectLinks with a nice link given a single span element let rec replaceSpans = function - | IndirectLink(body, original, key) -> - [ yield IndirectLink(body, original, key) + | IndirectLink(body, original, key, r) -> + [ yield IndirectLink(body, original, key, r) match refIndex.TryGetValue(key) with | true, i -> - yield Literal " [" - yield DirectLink([Literal (string i)], ("#rf" + DateTime.Now.ToString("yyMMddhh"), None)) - yield Literal "]" + yield Literal(" [", r) + yield DirectLink([Literal (string i, r)], ("#rf" + DateTime.Now.ToString("yyMMddhh"), None), r) + yield Literal("]", r) | _ -> () ] | Matching.SpanLeaf(sl) -> [Matching.SpanLeaf(sl)] | Matching.SpanNode(nd, spans) -> @@ -179,18 +179,18 @@ module Transformations = if colon > 0 then let auth = title.Substring(0, colon) let name = title.Substring(colon + 1, title.Length - 1 - colon) - yield [Span [ Literal (sprintf "[%d] " i) - DirectLink([Literal (name.Trim())], (link, Some title)) - Literal (" - " + auth)] ] + yield [Span([ Literal (sprintf "[%d] " i, None) + DirectLink([Literal (name.Trim(), None)], (link, Some title), None) + Literal (" - " + auth, None)], None) ] else - yield [Span [ Literal (sprintf "[%d] " i) - DirectLink([Literal title], (link, Some title))]] ] + yield [Span([ Literal (sprintf "[%d] " i, None) + DirectLink([Literal(title, None)], (link, Some title), None)], None)] ] // Return the document together with dictionary for looking up indices let id = DateTime.Now.ToString("yyMMddhh") - [ Paragraph [AnchorLink id]; - Heading(3, [Literal "References"]) - ListBlock(MarkdownListKind.Unordered, refList) ], refLookup + [ Paragraph([AnchorLink(id, None)], None) + Heading(3, [Literal("References", None)], None) + ListBlock(MarkdownListKind.Unordered, refList, None) ], refLookup /// Turn all indirect links into a references /// and add paragraph to the document @@ -270,8 +270,8 @@ module Transformations = | _ -> None match special with | EvalFormat(Some result, _, kind) -> ctx.Evaluator.Value.Format(result, kind) - | EvalFormat(None, ref, _) -> [ CodeBlock("Could not find reference '" + ref + "'", "", "") ] - | other -> [ EmbedParagraphs(other) ] + | EvalFormat(None, ref, _) -> [ CodeBlock("Could not find reference '" + ref + "'", "", "", None) ] + | other -> [ EmbedParagraphs(other, None) ] // Traverse all other structrues recursively | Matching.ParagraphNested(pn, nested) -> @@ -311,7 +311,7 @@ module Transformations = let rec replaceSpecialCodes ctx (formatted:IDictionary<_, _>) = function | Matching.LiterateParagraph(special) -> match special with - | RawBlock lines -> Some (InlineBlock (unparse lines)) + | RawBlock lines -> Some (InlineBlock(unparse lines, None)) | LiterateCode(_, { Visibility = (HiddenCode | NamedCode _) }) -> None | FormattedCode lines | LiterateCode(lines, _) -> Some (formatted.[Choice1Of2 lines]) @@ -352,7 +352,7 @@ module Transformations = | OutputKind.Latex -> sprintf "\\begin{lstlisting}\n%s\n\\end{lstlisting}" code - Some(InlineBlock(inlined)) + Some(InlineBlock(inlined, None)) // Traverse all other structures recursively | Matching.ParagraphNested(pn, nested) -> let nested = List.map (List.choose (replaceSpecialCodes ctx formatted)) nested @@ -379,7 +379,7 @@ module Transformations = | OutputKind.Latex -> CodeFormat.FormatLatex(snippets, ctx.GenerateLineNumbers) let lookup = [ for (key, _), fmtd in Seq.zip replacements formatted.Snippets -> - key, InlineBlock(fmtd.Content) ] |> dict + key, InlineBlock(fmtd.Content, None) ] |> dict // Replace original snippets with formatted HTML/Latex and return document let newParagraphs = List.choose (replaceSpecialCodes ctx lookup) doc.Paragraphs diff --git a/src/FSharp.Markdown/HtmlFormatting.fs b/src/FSharp.Markdown/HtmlFormatting.fs index a256cd19c..02bcb978b 100644 --- a/src/FSharp.Markdown/HtmlFormatting.fs +++ b/src/FSharp.Markdown/HtmlFormatting.fs @@ -67,19 +67,19 @@ let noBreak (ctx:FormattingContext) () = () /// Write MarkdownSpan value to a TextWriter let rec formatSpan (ctx:FormattingContext) = function - | LatexDisplayMath(body) -> + | LatexDisplayMath(body, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ ctx.Writer.Write("\\[" + (htmlEncode body) + "\\]") - | LatexInlineMath(body) -> + | LatexInlineMath(body, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ ctx.Writer.Write("\\(" + (htmlEncode body) + "\\)") - | AnchorLink(id) -> ctx.Writer.Write(" ") - | EmbedSpans(cmd) -> formatSpans ctx (cmd.Render()) - | Literal(str) -> ctx.Writer.Write(str) - | HardLineBreak -> ctx.Writer.Write("
" + ctx.Newline) - | IndirectLink(body, _, LookupKey ctx.Links (link, title)) - | DirectLink(body, (link, title)) -> + | AnchorLink(id, _) -> ctx.Writer.Write(" ") + | EmbedSpans(cmd, _) -> formatSpans ctx (cmd.Render()) + | Literal(str, _) -> ctx.Writer.Write(str) + | HardLineBreak(_) -> ctx.Writer.Write("
" + ctx.Newline) + | IndirectLink(body, _, LookupKey ctx.Links (link, title), _) + | DirectLink(body, (link, title), _) -> ctx.Writer.Write("") - | IndirectLink(body, original, _) -> + | IndirectLink(body, original, _, _) -> ctx.Writer.Write("[") formatSpans ctx body ctx.Writer.Write("]") ctx.Writer.Write(original) - | IndirectImage(body, _, LookupKey ctx.Links (link, title)) - | DirectImage(body, (link, title)) -> + | IndirectImage(body, _, LookupKey ctx.Links (link, title), _) + | DirectImage(body, (link, title), _) -> ctx.Writer.Write("\"") () ctx.Writer.Write("\" />") - | IndirectImage(body, original, _) -> + | IndirectImage(body, original, _, _) -> ctx.Writer.Write("[") ctx.Writer.Write(body) ctx.Writer.Write("]") ctx.Writer.Write(original) - | Strong(body) -> + | Strong(body, _) -> ctx.Writer.Write("") formatSpans ctx body ctx.Writer.Write("") - | InlineCode(body) -> + | InlineCode(body, _) -> ctx.Writer.Write("") ctx.Writer.Write(htmlEncode body) ctx.Writer.Write("") - | Emphasis(body) -> + | Emphasis(body, _) -> ctx.Writer.Write("") formatSpans ctx body ctx.Writer.Write("") @@ -141,10 +141,10 @@ let formatAnchor (ctx:FormattingContext) (spans:MarkdownSpans) = let rec gather (span:MarkdownSpan) : seq = seq { match span with - | Literal str -> yield! extractWords str - | Strong body -> yield! gathers body - | Emphasis body -> yield! gathers body - | DirectLink (body,_) -> yield! gathers body + | Literal(str, _) -> yield! extractWords str + | Strong(body, _) -> yield! gathers body + | Emphasis(body, _) -> yield! gathers body + | DirectLink(body, _, _) -> yield! gathers body | _ -> () } @@ -163,13 +163,13 @@ let withInner ctx f = /// Write a MarkdownParagraph value to a TextWriter let rec formatParagraph (ctx:FormattingContext) paragraph = match paragraph with - | LatexBlock(lines) -> + | LatexBlock(lines, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ let body = String.concat ctx.Newline lines ctx.Writer.Write("

\\[" + (htmlEncode body) + "\\]

") - | EmbedParagraphs(cmd) -> formatParagraphs ctx (cmd.Render()) - | Heading(n, spans) -> + | EmbedParagraphs(cmd, _) -> formatParagraphs ctx (cmd.Render()) + | Heading(n, spans, _) -> ctx.Writer.Write("") if ctx.GenerateHeaderAnchors then let anchorName = formatAnchor ctx spans @@ -179,28 +179,28 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = else formatSpans ctx spans ctx.Writer.Write("") - | Paragraph(spans) -> + | Paragraph(spans, _) -> ctx.ParagraphIndent() ctx.Writer.Write("

") for span in spans do formatSpan ctx span ctx.Writer.Write("

") - | HorizontalRule(_) -> + | HorizontalRule(_, _) -> ctx.Writer.Write("
") - | CodeBlock(code, String.WhiteSpace, _) -> + | CodeBlock(code, String.WhiteSpaceS, _, _) -> if ctx.WrapCodeSnippets then ctx.Writer.Write("
") ctx.Writer.Write("
")
       ctx.Writer.Write(htmlEncode code)
       ctx.Writer.Write("
") if ctx.WrapCodeSnippets then ctx.Writer.Write("
") - | CodeBlock(code, codeLanguage, _) -> + | CodeBlock(code, codeLanguage, _, _) -> if ctx.WrapCodeSnippets then ctx.Writer.Write("
") let langCode = sprintf "language-%s" codeLanguage ctx.Writer.Write(sprintf "
" langCode)
       ctx.Writer.Write(htmlEncode code)
       ctx.Writer.Write("
") if ctx.WrapCodeSnippets then ctx.Writer.Write("
") - | TableBlock(headers, alignments, rows) -> + | TableBlock(headers, alignments, rows, _) -> let aligns = alignments |> List.map (function | AlignLeft -> " align=\"left\"" | AlignRight -> " align=\"right\"" @@ -229,14 +229,14 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write("") ctx.Writer.Write(ctx.Newline) - | ListBlock(kind, items) -> + | ListBlock(kind, items, _) -> let tag = if kind = Ordered then "ol" else "ul" ctx.Writer.Write("<" + tag + ">" + ctx.Newline) for body in items do ctx.Writer.Write("
  • ") match body with // Simple Paragraph - | [ Paragraph [MarkdownSpan.Literal s] ] when not (s.Contains(ctx.Newline)) -> + | [ Paragraph([MarkdownSpan.Literal(s, _)], _) ] when not (s.Contains(ctx.Newline)) -> ctx.Writer.Write s | _ -> let inner = @@ -249,15 +249,15 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(wrappedInner) ctx.Writer.Write("
    • " + ctx.Newline) ctx.Writer.Write("") - | QuotedBlock(body) -> + | QuotedBlock(body, _) -> ctx.ParagraphIndent() ctx.Writer.Write("
    " + ctx.Newline) formatParagraphs { ctx with ParagraphIndent = fun () -> ctx.ParagraphIndent() (*; ctx.Writer.Write(" ")*) } body ctx.ParagraphIndent() ctx.Writer.Write("
    ") - | Span spans -> + | Span(spans, _) -> formatSpans ctx spans - | InlineBlock(code) -> + | InlineBlock(code, _) -> ctx.Writer.Write(code) ctx.LineBreak() diff --git a/src/FSharp.Markdown/LatexFormatting.fs b/src/FSharp.Markdown/LatexFormatting.fs index cb3bb8bc0..6494090b8 100644 --- a/src/FSharp.Markdown/LatexFormatting.fs +++ b/src/FSharp.Markdown/LatexFormatting.fs @@ -55,25 +55,25 @@ let noBreak (ctx:FormattingContext) () = () /// Write MarkdownSpan value to a TextWriter let rec formatSpan (ctx:FormattingContext) = function - | LatexInlineMath(body) -> ctx.Writer.Write(sprintf "$%s$" body) - | LatexDisplayMath(body) -> ctx.Writer.Write(sprintf "$$%s$$" body) - | EmbedSpans(cmd) -> formatSpans ctx (cmd.Render()) - | Literal(str) -> ctx.Writer.Write(latexEncode str) - | HardLineBreak -> ctx.LineBreak(); ctx.LineBreak() + | LatexInlineMath(body, _) -> ctx.Writer.Write(sprintf "$%s$" body) + | LatexDisplayMath(body, _) -> ctx.Writer.Write(sprintf "$$%s$$" body) + | EmbedSpans(cmd, _) -> formatSpans ctx (cmd.Render()) + | Literal(str, _) -> ctx.Writer.Write(latexEncode str) + | HardLineBreak(_) -> ctx.LineBreak(); ctx.LineBreak() | AnchorLink _ -> () - | IndirectLink(body, _, LookupKey ctx.Links (link, _)) - | DirectLink(body, (link, _)) - | IndirectLink(body, link, _) -> + | IndirectLink(body, _, LookupKey ctx.Links (link, _), _) + | DirectLink(body, (link, _), _) + | IndirectLink(body, link, _, _) -> ctx.Writer.Write(@"\href{") ctx.Writer.Write(latexEncode link) ctx.Writer.Write("}{") formatSpans ctx body ctx.Writer.Write("}") - | IndirectImage(body, _, LookupKey ctx.Links (link, _)) - | DirectImage(body, (link, _)) - | IndirectImage(body, link, _) -> + | IndirectImage(body, _, LookupKey ctx.Links (link, _), _) + | DirectImage(body, (link, _), _) + | IndirectImage(body, link, _, _) -> // Use the technique introduced at // http://stackoverflow.com/q/14014827 if not (System.String.IsNullOrWhiteSpace(body)) then @@ -91,15 +91,15 @@ let rec formatSpan (ctx:FormattingContext) = function ctx.Writer.Write(@"\end{figure}") ctx.LineBreak() - | Strong(body) -> + | Strong(body, _) -> ctx.Writer.Write(@"\textbf{") formatSpans ctx body ctx.Writer.Write("}") - | InlineCode(body) -> + | InlineCode(body, _) -> ctx.Writer.Write(@"\texttt{") ctx.Writer.Write(latexEncode body) ctx.Writer.Write("}") - | Emphasis(body) -> + | Emphasis(body, _) -> ctx.Writer.Write(@"\emph{") formatSpans ctx body ctx.Writer.Write("}") @@ -110,7 +110,7 @@ and formatSpans ctx = List.iter (formatSpan ctx) /// Write a MarkdownParagraph value to a TextWriter let rec formatParagraph (ctx:FormattingContext) paragraph = match paragraph with - | LatexBlock(lines) -> + | LatexBlock(lines, _) -> ctx.LineBreak(); ctx.LineBreak() ctx.Writer.Write("\["); ctx.LineBreak() for line in lines do @@ -119,8 +119,8 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write("\]") ctx.LineBreak(); ctx.LineBreak() - | EmbedParagraphs(cmd) -> formatParagraphs ctx (cmd.Render()) - | Heading(n, spans) -> + | EmbedParagraphs(cmd, _) -> formatParagraphs ctx (cmd.Render()) + | Heading(n, spans, _) -> let level = match n with | 1 -> @"\section*" @@ -133,7 +133,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = formatSpans ctx spans ctx.Writer.Write("}") ctx.LineBreak() - | Paragraph(spans) -> + | Paragraph(spans, _) -> ctx.LineBreak(); ctx.LineBreak() for span in spans do formatSpan ctx span @@ -143,7 +143,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\noindent\makebox[\linewidth]{\rule{\linewidth}{0.4pt}}\medskip") ctx.LineBreak() - | CodeBlock(code, _, _) -> + | CodeBlock(code, _, _, _) -> ctx.Writer.Write(@"\begin{lstlisting}") ctx.LineBreak() ctx.Writer.Write(code) @@ -151,7 +151,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{lstlisting}") ctx.LineBreak() - | TableBlock(headers, alignments, rows) -> + | TableBlock(headers, alignments, rows, _) -> let aligns = alignments |> List.map (function | AlignRight -> "|r" | AlignCenter -> "|c" @@ -178,7 +178,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{tabular}") ctx.LineBreak() - | ListBlock(kind, items) -> + | ListBlock(kind, items, _) -> let tag = if kind = Ordered then "enumerate" else "itemize" ctx.Writer.Write(@"\begin{" + tag + "}") ctx.LineBreak() @@ -189,16 +189,16 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{" + tag + "}") ctx.LineBreak() - | QuotedBlock(body) -> + | QuotedBlock(body, _) -> ctx.Writer.Write(@"\begin{quote}") ctx.LineBreak() formatParagraphs ctx body ctx.Writer.Write(@"\end{quote}") ctx.LineBreak() - | Span spans -> + | Span(spans, _) -> formatSpans ctx spans - | InlineBlock(code) -> + | InlineBlock(code, _) -> ctx.Writer.Write(code) ctx.LineBreak() diff --git a/src/FSharp.Markdown/Main.fs b/src/FSharp.Markdown/Main.fs index 2e174bd7a..21fc0bc8d 100644 --- a/src/FSharp.Markdown/Main.fs +++ b/src/FSharp.Markdown/Main.fs @@ -48,17 +48,19 @@ type Markdown = use reader = new StringReader(text) let lines = [ let line = ref "" + let mutable lineNo = 1 while (line := reader.ReadLine(); line.Value <> null) do - yield line.Value + yield (line.Value, lineNo) + lineNo <- lineNo + 1 if text.EndsWith(newline) then - yield "" ] + yield ("", lineNo) ] //|> Utils.replaceTabs 4 let links = Dictionary<_, _>() //let (Lines.TrimBlank lines) = lines - let ctx : ParsingContext = { Newline = newline; Links = links } + let ctx : ParsingContext = { Newline = newline; Links = links; CurrentRange = Some({ Line = 1 }) } let paragraphs = lines - |> FSharp.Collections.List.skipWhile String.IsNullOrWhiteSpace + |> FSharp.Collections.List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> parseParagraphs ctx |> List.ofSeq MarkdownDocument(paragraphs, links) diff --git a/src/FSharp.Markdown/Markdown.fs b/src/FSharp.Markdown/Markdown.fs index a70fc0e00..afd942d4d 100644 --- a/src/FSharp.Markdown/Markdown.fs +++ b/src/FSharp.Markdown/Markdown.fs @@ -13,6 +13,8 @@ open System.Collections.Generic // Definition of the Markdown format // -------------------------------------------------------------------------------------- +type MarkdownRange = { Line : int } + /// A list kind can be `Ordered` or `Unordered` corresponding to `
      ` and `