add wiki link formatter and translating functionality

vsch · Apr 10, 2020 · 4c2befc · 4c2befc
1 parent 9c77bd7
commit 4c2befc
Show file tree

Hide file tree

Showing 26 changed files with 2,692 additions and 203 deletions.
diff --git a/.idea/dictionaries/vlad.xml b/.idea/dictionaries/vlad.xml
diff --git a/VERSION-TODO.md b/VERSION-TODO.md
@@ -218,9 +218,19 @@ Please give feedback on the upcoming changes if you have concerns about breaking
 
 ## 0.61.4
 
-* [ ] Fix: wiki links should not be wrapped during formatting.
 * Fix: merge [#397, PR: Add base64 image support with docx rendering] thanks to [@Xaelis]
-* Add: test for base64 encoded docx conversion
+  * Add: test for base64 encoded docx conversion
+* Break: `WikiLinkLinkResolver` to take `WikiLinkExtension.ALLOW_ANCHORS` and
+  `WikiLinkExtension.ALLOW_ANCHOR_ESCAPE` into account when extracting page ref from link.
+  * Fix: `#` or `\` included in the URL of the resolved link are now URL encoded.
+  * Fix: default `WikiLinkLinkResolver` handles its own text unescaping. When resolving wiki  
+    link with default resolver do not unescape. If replacing default resolver ensure you
+    unescape the text. **This is needed to properly handle anchor escaping.**
+  * Fix: `PegdownOptionsAdapter` to set `WikiLinkExtension.ALLOW_ANCHORS` to `true` for pegdown
+    compatibility
+* Fix: wiki links should not be wrapped during formatting.
+  * Add: WikiLink formatter extension and tests
+  * Add: WikiLink translating formatter functionality and tests
 
 ## 0.61.2
 

diff --git a/...s/src/main/java/com/vladsch/flexmark/ext/attributes/internal/AttributesNodeFormatter.java b/...s/src/main/java/com/vladsch/flexmark/ext/attributes/internal/AttributesNodeFormatter.java
@@ -5,7 +5,7 @@
 import com.vladsch.flexmark.ast.util.AnchorRefTargetBlockVisitor;
 import com.vladsch.flexmark.ext.attributes.*;
 import com.vladsch.flexmark.formatter.*;
-import com.vladsch.flexmark.formatter.internal.CoreNodeFormatter;
+import com.vladsch.flexmark.formatter.Formatter;
 import com.vladsch.flexmark.html.renderer.HtmlIdGenerator;
 import com.vladsch.flexmark.util.ast.Document;
 import com.vladsch.flexmark.util.ast.Node;
@@ -30,7 +30,7 @@ public class AttributesNodeFormatter implements PhasedNodeFormatter, ExplicitAtt
     final public static DataKey<Set<Node>> PROCESSED_ATTRIBUTES = new DataKey<>("PROCESSED_ATTRIBUTES", HashSet::new);
 
     // need to have this one available in core formatter
-    final public static DataKey<Map<String, String>> ATTRIBUTE_UNIQUIFICATION_ID_MAP = CoreNodeFormatter.ATTRIBUTE_UNIQUIFICATION_ID_MAP;
+    final public static DataKey<Map<String, String>> ATTRIBUTE_UNIQUIFICATION_ID_MAP = Formatter.ATTRIBUTE_UNIQUIFICATION_ID_MAP;
 
     final public static DataKey<Map<String, String>> ATTRIBUTE_UNIQUIFICATION_CATEGORY_MAP = new DataKey<>("ATTRIBUTE_UNIQUIFICATION_CATEGORY_MAP", HashMap::new);
     final public static DataKey<Integer> ATTRIBUTE_TRANSLATION_ID = new DataKey<>("ATTRIBUTE_TRANSLATION_ID", 0); // next attribute index

diff --git a/flexmark-ext-toc/src/test/java/com/vladsch/flexmark/ext/toc/ExtTocFormatterTestSuite.java b/flexmark-ext-toc/src/test/java/com/vladsch/flexmark/ext/toc/ExtTocFormatterTestSuite.java
diff --git a/flexmark-ext-toc/src/test/java/com/vladsch/flexmark/ext/toc/ExtTocTestSuite.java b/flexmark-ext-toc/src/test/java/com/vladsch/flexmark/ext/toc/ExtTocTestSuite.java
@@ -8,6 +8,8 @@
         TocOptionsParserTest.class,
         ComboTocSpecTest.class,
         ComboSimTocSpecTest.class,
+        ComboSimTocMdFormatterSpecTest.class,
+        ComboSimTocHtmlFormatterSpecTest.class,
 })
 public class ExtTocTestSuite {
 }
diff --git a/flexmark-ext-wikilink/flexmark-ext-wikilink.iml b/flexmark-ext-wikilink/flexmark-ext-wikilink.iml
@@ -26,5 +26,6 @@
     <orderEntry type="module" module-name="flexmark-core-test" scope="TEST" />
     <orderEntry type="module" module-name="flexmark-ext-typographic" scope="TEST" />
     <orderEntry type="library" name="org.jetbrains:annotations" level="project" />
+    <orderEntry type="module" module-name="flexmark-util-format" />
   </component>
-</module>
+</module>
diff --git a/flexmark-ext-wikilink/src/main/java/com/vladsch/flexmark/ext/wikilink/WikiLinkExtension.java b/flexmark-ext-wikilink/src/main/java/com/vladsch/flexmark/ext/wikilink/WikiLinkExtension.java
@@ -1,9 +1,7 @@
 package com.vladsch.flexmark.ext.wikilink;
 
-import com.vladsch.flexmark.ext.wikilink.internal.WikiLinkJiraRenderer;
-import com.vladsch.flexmark.ext.wikilink.internal.WikiLinkLinkRefProcessor;
-import com.vladsch.flexmark.ext.wikilink.internal.WikiLinkLinkResolver;
-import com.vladsch.flexmark.ext.wikilink.internal.WikiLinkNodeRenderer;
+import com.vladsch.flexmark.ext.wikilink.internal.*;
+import com.vladsch.flexmark.formatter.Formatter;
 import com.vladsch.flexmark.html.HtmlRenderer;
 import com.vladsch.flexmark.html.renderer.LinkType;
 import com.vladsch.flexmark.parser.Parser;
@@ -18,7 +16,7 @@
  * <p>
  * The parsed emoji shortcuts text regions are turned into {@link WikiLink} nodes.
  */
-public class WikiLinkExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension {
+public class WikiLinkExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension, Formatter.FormatterExtension {
     final public static DataKey<Boolean> ALLOW_INLINES = new DataKey<>("ALLOW_INLINES", false);
     final public static DataKey<Boolean> ALLOW_ANCHORS = new DataKey<>("ALLOW_ANCHORS", false);
     final public static DataKey<Boolean> ALLOW_ANCHOR_ESCAPE = new DataKey<>("ALLOW_ANCHOR_ESCAPE", false);
@@ -88,6 +86,12 @@ public void parserOptions(MutableDataHolder options) {
 
     }
 
+    @Override
+    public void extend(Formatter.Builder formatterBuilder) {
+        formatterBuilder.nodeFormatterFactory(new WikiLinkNodeFormatter.Factory());
+//        formatterBuilder.linkResolverFactory(new WikiLinkLinkResolver.Factory());
+    }
+
     @Override
     public void extend(Parser.Builder parserBuilder) {
         parserBuilder.linkRefProcessorFactory(new WikiLinkLinkRefProcessor.Factory());

diff --git a/flexmark-ext-wikilink/src/main/java/com/vladsch/flexmark/ext/wikilink/WikiNode.java b/flexmark-ext-wikilink/src/main/java/com/vladsch/flexmark/ext/wikilink/WikiNode.java
@@ -11,6 +11,8 @@
 import org.jetbrains.annotations.NotNull;
 
 public class WikiNode extends Node implements DoNotDecorate, TextContainer {
+    final public static char SEPARATOR_CHAR = '|';
+
     protected BasedSequence openingMarker = BasedSequence.NULL;
     protected BasedSequence link = BasedSequence.NULL;
     protected BasedSequence pageRef = BasedSequence.NULL;
@@ -178,12 +180,12 @@ public void setLinkChars(BasedSequence linkChars, boolean allowAnchors, boolean
         if (linkIsFirst) {
             pos = linkChars.length() - 2;
             do {
-                pos = linkChars.lastIndexOf('|', pos - 1);
+                pos = linkChars.lastIndexOf(SEPARATOR_CHAR, pos - 1);
             } while (pos != -1 && canEscapePipe && (pos > 0 && linkChars.charAt(pos - 1) == '\\' && (linkChars.subSequence(0, pos).countTrailing(CharPredicate.BACKSLASH) & 1) == 1));
         } else {
             pos = -1;
             do {
-                pos = linkChars.indexOf('|', pos + 1);
+                pos = linkChars.indexOf(SEPARATOR_CHAR, pos + 1);
             } while (pos != -1 && canEscapePipe && (pos > 0 && linkChars.charAt(pos - 1) == '\\' && (linkChars.subSequence(0, pos).countTrailing(CharPredicate.BACKSLASH) & 1) == 1));
         }
 

diff --git a/...kilink/src/main/java/com/vladsch/flexmark/ext/wikilink/internal/WikiLinkLinkResolver.java b/...kilink/src/main/java/com/vladsch/flexmark/ext/wikilink/internal/WikiLinkLinkResolver.java
@@ -34,27 +34,49 @@ public ResolvedLink resolveLink(@NotNull Node node, @NotNull LinkResolverBasicCo
             sb.append(isWikiImage ? options.getImagePrefix(absolute) : options.getLinkPrefix(absolute));
 
             boolean hadAnchorRef = false;
+            boolean isEscaped = false;
 
             String linkEscapeChars = options.linkEscapeChars;
             String linkReplaceChars = options.linkReplaceChars;
             for (int i = absolute ? 1 : 0; i < iMax; i++) {
                 char c = wikiLink.charAt(i);
 
-                if (c == '#') {
-                    if (hadAnchorRef) continue;
+                if (c == '#' && !hadAnchorRef && options.allowAnchors && !(isEscaped && options.allowAnchorEscape)) {
                     sb.append(isWikiImage ? options.imageFileExtension : options.linkFileExtension);
+                    sb.append(c);
                     hadAnchorRef = true;
+                    isEscaped = false;
+                    continue;
                 }
 
-                int pos = linkEscapeChars.indexOf(c);
-
-                if (pos < 0) {
-                    sb.append(c);
+                if (c == '\\') {
+                    if (isEscaped) {
+                        // need to URL encode \
+                        sb.append("%5C");
+                    }
+                    isEscaped = !isEscaped;
                 } else {
-                    sb.append(linkReplaceChars.charAt(pos));
+                    isEscaped = false;
+                    if (c == '#' && !hadAnchorRef) {
+                        // need to URL encode the #
+                        sb.append("%23");
+                    } else {
+                        int pos = linkEscapeChars.indexOf(c);
+
+                        if (pos < 0) {
+                            sb.append(c);
+                        } else {
+                            sb.append(linkReplaceChars.charAt(pos));
+                        }
+                    }
                 }
             }
 
+            if (isEscaped) {
+                // need to add dangling URL encoded \
+                sb.append("%5C");
+            }
+
             if (!hadAnchorRef) {
                 sb.append(isWikiImage ? options.imageFileExtension : options.linkFileExtension);
             }