diff --git a/compiler/commands.nim b/compiler/commands.nim index 994b224f1b61a..8f8df71a781e5 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -443,7 +443,7 @@ proc parseCommand*(command: string): Command = of "check": cmdCheck of "e": cmdNimscript of "doc0": cmdDoc0 - of "doc2", "doc": cmdDoc2 + of "doc2", "doc": cmdDoc of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html of "rst2tex": cmdRst2tex diff --git a/compiler/docgen.nim b/compiler/docgen.nim index dc2a63f48bc5c..2c99adba30a54 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -28,6 +28,24 @@ const DocColOffset = "## ".len # assuming that a space was added after ## type + ItemFragment = object ## A fragment from each item will be eventually + ## constructed by converting `rst` fields to strings. + case isRst: bool + of true: + rst: PRstNode + of false: ## contains ready markup e.g. from runnableExamples + str: string + ItemPre = seq[ItemFragment] ## A pre-processed item. + Item = object ## Any item in documentation, e.g. symbol + ## entry. Configuration variable ``doc.item`` + ## is used for its HTML rendering. + descRst: ItemPre ## Description of the item (may contain + ## runnableExamples). + substitutions: seq[string] ## Variable names in `doc.item`... + ModSection = object ## Section like Procs, Types, etc. + secItems: seq[Item] ## Pre-processed items. + finalMarkup: string ## The items, after RST pass 2 and rendering. + ModSections = array[TSymKind, ModSection] TSections = array[TSymKind, string] ExampleGroup = ref object ## a group of runnableExamples with same rdoccmd @@ -35,17 +53,25 @@ type docCmd: string ## from user config, e.g. --doccmd:-d:foo code: string ## contains imports; each import contains `body` index: int ## group index + JsonItem = object # pre-processed item: `rst` should be finalized + json: JsonNode + rst: PRstNode + rstField: string TDocumentor = object of rstgen.RstGenerator - modDesc: string # module description + modDescPre: ItemPre # module description, not finalized + modDescFinal: string # module description, after RST pass 2 and rendering module: PSym modDeprecationMsg: string - toc, toc2, section: TSections + section: ModSections # entries of ``.nim`` file (for `proc`s, etc) + toc, toc2: TSections # toc2 - grouped TOC tocTable: array[TSymKind, Table[string, string]] indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. - jArray: JsonNode + jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content + jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable + sharedState: PRstSharedState isPureRst: bool conf*: ConfigRef cache*: IdentCache @@ -58,6 +84,9 @@ type PDoc* = ref TDocumentor ## Alias to type less. +proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) +proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) + proc prettyString(a: object): string = # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906 for k, v in fieldPairs(a): @@ -151,12 +180,12 @@ template declareClosures = if not fileExists(result): result = "" proc parseRst(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, rstOptions: RstParseOptions; - conf: ConfigRef): PRstNode = + conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() - result = rstParse(text, filename, line, column, hasToc, rstOptions, - docgenFindFile, compilerMsgHandler) + result = rstParsePass1(text, filename, line, column, rstOptions, + sharedState) proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, guessTarget: bool): AbsoluteFile = @@ -179,10 +208,13 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.conf = conf result.cache = cache result.outDir = conf.outDir.string + const options = {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown, roNimFile} + result.sharedState = newRstSharedState( + options, filename.string, + docgenFindFile, compilerMsgHandler) initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), - conf.configVars, filename.string, - {roSupportRawDirective, roSupportMarkdown, - roPreferMarkdown, roNimFile}, + conf.configVars, filename.string, options, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -203,7 +235,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 - result.jArray = newJArray() + result.jEntriesFinal = newJArray() initStrTable result.types result.onTestSnippet = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = @@ -261,37 +293,31 @@ proc getVarIdx(varnames: openArray[string], id: string): int = return i result = -1 -proc genComment(d: PDoc, n: PNode): string = - result = "" +proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - let comment = n.comment - when false: - # RFC: to preseve newlines in comments, this would work: - comment = comment.replace("\n", "\n\n") - renderRstToOut(d[], - parseRst(comment, toFullPath(d.conf, n.info), - toLinenumber(n.info), - toColumn(n.info) + DocColOffset, - (var dummy: bool; dummy), d.options, d.conf), - result) - -proc genRecCommentAux(d: PDoc, n: PNode): string = - if n == nil: return "" + result = parseRst(n.comment, toFullPath(d.conf, n.info), + toLinenumber(n.info), + toColumn(n.info) + DocColOffset, + d.options, d.conf, + d.sharedState) + +proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = + if n == nil: return nil result = genComment(d, n) - if result == "": + if result == nil: if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}: # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in 0.. 0: msg.add " cmd: " & rdoccmd - dispA(d.conf, dest, "\n

$1

\n", + var s: string + dispA(d.conf, s, "\n

$1

\n", "\n\n\\textbf{$1}\n", [msg]) + dest.add s inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) @@ -597,7 +625,7 @@ proc getRoutineBody(n: PNode): PNode = doAssert result.len == 2 result = result[1] -proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var string) = +proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) = var n = n var state = rsStart template fn(n2, topLevel) = @@ -706,7 +734,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string = ## types will be added with a preceding dash. Return types won't be added. ## ## If you modify the output of this proc, please update the anchor generation - ## section of ``doc/docgen.txt``. + ## section of ``doc/docgen.rst``. result = baseName case k of skProc, skFunc: discard @@ -804,7 +832,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = var result = "" var literal, plainName = "" var kind = tkEof - var comm = "" + var comm: ItemPre if n.kind in routineDefs: getAllRunnableExamples(d, n, comm) else: @@ -840,9 +868,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].add(getConfigVar(d.conf, "doc.item") % - ["name", name, "uniqueName", uniqueName, - "header", result, "desc", comm, "itemID", $d.id, + d.section[k].secItems.add Item(descRst: comm, substitutions: @[ + "name", name, "uniqueName", uniqueName, + "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, @@ -886,25 +914,26 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if k == skType and nameNode.kind == nkSym: d.types.strTableAdd nameNode.sym -proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = +proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem = if not isVisible(d, nameNode): return var name = getName(d, nameNode) comm = genRecComment(d, n) r: TSrcGen initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - result = %{ "name": %name, "type": %($k), "line": %n.info.line.int, - "col": %n.info.col} - if comm.len > 0: - result["description"] = %comm + result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int, + "col": %n.info.col} + if comm != nil: + result.rst = comm + result.rstField = "description" if r.buf.len > 0: - result["code"] = %r.buf + result.json["code"] = %r.buf if k in routineKinds: - result["signature"] = newJObject() + result.json["signature"] = newJObject() if n[paramsPos][0].kind != nkEmpty: - result["signature"]["return"] = %($n[paramsPos][0]) + result.json["signature"]["return"] = %($n[paramsPos][0]) if n[paramsPos].len > 1: - result["signature"]["arguments"] = newJArray() + result.json["signature"]["arguments"] = newJArray() for paramIdx in 1 ..< n[paramsPos].len: for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2: let @@ -912,22 +941,22 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = paramType = $n[paramsPos][paramIdx][^2] if n[paramsPos][paramIdx][^1].kind != nkEmpty: let paramDefault = $n[paramsPos][paramIdx][^1] - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} else: - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} if n[pragmasPos].kind != nkEmpty: - result["signature"]["pragmas"] = newJArray() + result.json["signature"]["pragmas"] = newJArray() for pragma in n[pragmasPos]: - result["signature"]["pragmas"].add %($pragma) + result.json["signature"]["pragmas"].add %($pragma) if n[genericParamsPos].kind != nkEmpty: - result["signature"]["genericParams"] = newJArray() + result.json["signature"]["genericParams"] = newJArray() for genericParam in n[genericParamsPos]: var param = %{"name": %($genericParam)} if genericParam.sym.typ.sons.len > 0: param["types"] = newJArray() for kind in genericParam.sym.typ.sons: param["types"].add %($kind) - result["signature"]["genericParams"].add param + result.json["signature"]["genericParams"].add param proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 @@ -946,8 +975,8 @@ proc traceDeps(d: PDoc, it: PNode) = traceDeps(d, a) elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): let external = externalDep(d, it.sym) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -956,8 +985,8 @@ proc exportSym(d: PDoc; s: PSym) = const k = exportSection if s.kind == skModule and belongsToPackage(d.conf, s): let external = externalDep(d, s) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -968,9 +997,9 @@ proc exportSym(d: PDoc; s: PSym) = complexSymbol = complexName(s.kind, s.ast, s.name.s) symbolOrId = d.newUniquePlainSymbol(complexSymbol) external = externalDep(d, module) - if d.section[k] != "": d.section[k].add(", ") + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") # XXX proper anchor generation here - dispA(d.conf, d.section[k], + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, s.name.s), changeFileExt(external, "html"), @@ -1034,13 +1063,16 @@ proc documentRaises*(cache: IdentCache; n: PNode) = if p5 != nil: n[pragmasPos].add p5 proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = + ## Goes through nim nodes recursively and collects doc comments. + ## Main function for `doc`:option: command, + ## which is implemented in ``docgen2.nim``. template genItemAux(skind) = genItem(d, n, n[namePos], skind, docFlags) case n.kind of nkPragma: let pragmaNode = findPragma(n, wDeprecated) d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode)) - of nkCommentStmt: d.modDesc.add(genComment(d, n)) + of nkCommentStmt: d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) genItemAux(skProc) @@ -1079,21 +1111,63 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept" of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0]) of nkCallKinds: - var comm = "" + var comm: ItemPre getAllRunnableExamples(d, n, comm) - if comm != "": d.modDesc.add(comm) + if comm.len != 0: d.modDescPre.add(comm) else: discard -proc add(d: PDoc; j: JsonNode) = - if j != nil: d.jArray.add j +proc finishGenerateDoc*(d: var PDoc) = + ## Perform 2nd RST pass for resolution of links/footnotes/headings... + # Main title/subtitle are allowed only in the first RST fragment of document + var firstRst = PRstNode(nil) + for fragment in d.modDescPre: + if fragment.isRst: + firstRst = fragment.rst + break + preparePass2(d.sharedState, firstRst) + + # Finalize fragments of ``.nim`` or ``.rst`` file + proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = + for f in fragments: + case f.isRst: + of true: + var resolved = resolveSubs(d.sharedState, f.rst) + renderRstToOut(d[], resolved, result) + of false: result &= f.str + for k in TSymKind: + for item in d.section[k].secItems: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @["desc", itemDesc])) + itemDesc = "" + d.section[k].secItems.setLen 0 + renderItemPre(d, d.modDescPre, d.modDescFinal) + d.modDescPre.setLen 0 + d.hasToc = d.hasToc or d.sharedState.hasToc + + # Finalize fragments of ``.json`` file + for i, entry in d.jEntriesPre: + if entry.rst != nil: + let resolved = resolveSubs(d.sharedState, entry.rst) + var str: string + renderRstToOut(d[], resolved, str) + entry.json[entry.rstField] = %str + d.jEntriesFinal.add entry.json + d.jEntriesPre[i].rst = nil + +proc add(d: PDoc; j: JsonItem) = + if j.json != nil or j.rst != nil: d.jEntriesPre.add j proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = case n.kind of nkCommentStmt: if includeComments: - d.add %*{"comment": genComment(d, n)} + d.add JsonItem(rst: genComment(d, n), rstField: "comment", + json: %Table[string, string]()) else: - d.modDesc.add(genComment(d, n)) + d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) d.add genJsonItem(d, n, n[namePos], skProc) @@ -1173,11 +1247,11 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) = "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs", "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports" ] - if d.section[kind] == "": return + if d.section[kind].finalMarkup == "": return var title = sectionNames[kind] - d.section[kind] = getConfigVar(d.conf, "doc.section") % [ + d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [ "sectionid", $ord(kind), "sectionTitle", title, - "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind]] + "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup] var tocSource = d.toc if groupedToc: @@ -1209,7 +1283,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = if toc != "" or d.target == outLatex: # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set toc = getConfigVar(d.conf, "doc.toc") % ["content", toc] - for i in TSymKind: code.add(d.section[i]) + for i in TSymKind: code.add(d.section[i].finalMarkup) # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: @@ -1234,7 +1308,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = let seeSrc = genSeeSrc(d, d.filename, 1) content = getConfigVar(d.conf, bodyname) % [ "title", title, "subtitle", subtitle, - "tableofcontents", toc, "moduledesc", d.modDesc, "date", getDateStr(), + "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", code, "deprecationMsg", d.modDeprecationMsg, "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile, @@ -1248,7 +1322,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile), "title", title, "subtitle", subtitle, "tableofcontents", toc, - "moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(), + "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", content, "author", d.meta[metaAuthor], "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics, "deprecationMsg", d.modDeprecationMsg] @@ -1297,12 +1371,12 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = proc writeOutputJson*(d: PDoc, useWarning = false) = runAllExamples(d) var modDesc: string - for desc in d.modDesc: + for desc in d.modDescFinal: modDesc &= desc let content = %*{"orig": d.filename, "nimble": getPackageName(d.conf, d.filename), "moduleDescription": modDesc, - "entries": d.jArray} + "entries": d.jEntriesFinal} if optStdout in d.conf.globalOptions: write(stdout, $content) else: @@ -1324,12 +1398,14 @@ proc handleDocOutputOptions*(conf: ConfigRef) = conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + ## implementation of deprecated ``doc0`` command (without semantic checking) handleDocOutputOptions conf var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) d.hasToc = true generateDoc(d, ast, ast) + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1339,14 +1415,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; var d = newDocumentor(filen, cache, conf, outExt) d.isPureRst = true - var rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, - d.hasToc, - {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}, - conf) - var modDesc = newStringOfCap(30_000) - renderRstToOut(d[], rst, modDesc) - d.modDesc = modDesc + {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown}, conf, + d.sharedState) + d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1357,6 +1432,7 @@ proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) = commandRstAux(cache, conf, conf.projectFull, TexExt) proc commandJson*(cache: IdentCache, conf: ConfigRef) = + ## implementation of a deprecated jsondoc0 command var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) @@ -1366,7 +1442,8 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) = warnUser, "the ':test:' attribute is not supported by this backend") d.hasToc = true generateJson(d, ast) - let json = d.jArray + finishGenerateDoc(d) + let json = d.jEntriesFinal let content = pretty(json) if optStdout in d.conf.globalOptions: diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index d077ddc67200b..bfdb4568ca2ae 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -31,6 +31,7 @@ template closeImpl(body: untyped) {.dirty.} = let useWarning = sfMainModule notin g.module.flags let groupedToc = true if shouldProcess(g): + finishGenerateDoc(g.doc) body try: generateIndex(g.doc) diff --git a/compiler/main.nim b/compiler/main.nim index 7824e8cb2c447..1c8034c6cd7b7 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -282,7 +282,7 @@ proc mainCommand*(graph: ModuleGraph) = else: rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc") of cmdDoc0: docLikeCmd commandDoc(cache, conf) - of cmdDoc2: + of cmdDoc: docLikeCmd(): conf.setNoteDefaults(warnLockLevel, false) # issue #13218 conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 diff --git a/compiler/options.nim b/compiler/options.nim index 05d8ee5b7504d..ffb0f450136c4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -143,8 +143,8 @@ type cmdIdeTools # ide tools (e.g. nimsuggest) cmdNimscript # evaluate nimscript cmdDoc0 - cmdDoc2 - cmdDoc2tex + cmdDoc # convert .nim doc comments to HTML + cmdDoc2tex # convert .nim doc comments to LaTeX cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX cmdJsondoc0 @@ -161,7 +161,7 @@ type const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun} - cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, + cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex} type diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 336caa2dfa61c..031a86d98c997 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -441,7 +441,7 @@ proc rawGetTok(L: var Lexer, tok: var Token) = inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq) = var L: Lexer var length = tokens.len L.buf = cstring(buffer) @@ -489,7 +489,7 @@ type autoSymIdx: int # order of occurence: fnAutoSymbol label: string # valid for fnAutoNumberLabel - SharedState = object + RstSharedState = object options: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles hTitleCnt: int # =0 if no title, =1 if only main title, @@ -498,8 +498,8 @@ type currRole: string # current interpreted text role currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions - refs: seq[Substitution] # references - anchors: seq[AnchorSubst] # internal target substitutions + refs*: seq[Substitution] # references + anchors*: seq[AnchorSubst] # internal target substitutions lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] @@ -508,19 +508,19 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. + filename: string + hasToc*: bool - PSharedState = ref SharedState + PRstSharedState* = ref RstSharedState RstParser = object of RootObj idx*: int tok*: TokenSeq - s*: PSharedState + s*: PRstSharedState indentStack*: seq[int] - filename*: string line*, col*: int ## initial line/column of whole text or ## documenation fragment that will be added ## in case of error/warning reporting to ## (relative) line/column of the token. - hasToc*: bool curAnchor*: string # variable to track latest anchor in s.anchors EParseError* = object of ValueError @@ -578,35 +578,42 @@ proc whichRoleAux(sym: string): RstNodeKind = else: # unknown role result = rnUnknownRole -proc newSharedState(options: RstParseOptions, - findFile: FindFileHandler, - msgHandler: MsgHandler): PSharedState = - new(result) - result.currRole = defaultRole(options) - result.currRoleKind = whichRoleAux(result.currRole) - result.subs = @[] - result.refs = @[] - result.options = options - result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler - result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc newRstSharedState*(options: RstParseOptions, + filename: string, + findFile: FindFileHandler, + msgHandler: MsgHandler): PRstSharedState = + let r = defaultRole(options) + result = PRstSharedState( + currRole: r, + currRoleKind: whichRoleAux(r), + options: options, + msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, + filename: filename, + findFile: if not isNil(findFile): findFile else: defaultFindFile + ) proc curLine(p: RstParser): int = p.line + currentTok(p).line proc findRelativeFile(p: RstParser; filename: string): string = - result = p.filename.splitFile.dir / filename + result = p.s.filename.splitFile.dir / filename if not fileExists(result): result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = + ## Print warnings for footnotes/substitutions. + ## TODO: their line/column info is not known, to fix it. + s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.filename, p.line + line, + p.s.msgHandler(p.s.filename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, currentTok(p).symbol) @@ -680,12 +687,10 @@ proc popInd(p: var RstParser) = # whether it should continue its processing or not, and decided not to, # then this B.E. handler should step back (e.g. do `dec p.idx`). -proc initParser(p: var RstParser, sharedState: PSharedState) = +proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.indentStack = @[0] p.tok = @[] p.idx = 0 - p.filename = "" - p.hasToc = false p.col = ColRstInit p.line = LineRstInit p.s = sharedState @@ -754,14 +759,14 @@ proc rstnodeToRefname(n: PRstNode): string = var b = false rstnodeToRefnameAux(n, result, b) -proc findSub(p: var RstParser, n: PRstNode): int = +proc findSub(s: PRstSharedState, n: PRstNode): int = var key = addNodes(n) # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: + for i in countup(0, high(s.subs)): + if key == s.subs[i].key: return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: + for i in countup(0, high(s.subs)): + if cmpIgnoreStyle(key, s.subs[i].key) == 0: return i result = -1 @@ -783,10 +788,10 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = return p.s.refs.add(Substitution(key: key, value: value)) -proc findRef(p: var RstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value +proc findRef(s: PRstSharedState, key: string): PRstNode = + for i in countup(0, high(s.refs)): + if key == s.refs[i].key: + return s.refs[i].value proc addAnchor(p: var RstParser, refn: string, reset: bool) = ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` @@ -800,8 +805,8 @@ proc addAnchor(p: var RstParser, refn: string, reset: bool) = else: p.curAnchor = refn -proc findMainAnchor(p: RstParser, refn: string): string = - for subst in p.s.anchors: +proc findMainAnchor(s: PRstSharedState, refn: string): string = + for subst in s.anchors: if subst.mainAnchor == refn: # no need to rename result = subst.mainAnchor break @@ -838,28 +843,28 @@ proc addFootnoteSymAuto(p: var RstParser) = p.s.lineFootnoteSym.add curLine(p) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) -proc orderFootnotes(p: var RstParser) = +proc orderFootnotes(s: PRstSharedState) = ## numerate auto-numbered footnotes taking into account that all ## manually numbered ones always have preference. - ## Save the result back to p.s.footnotes. + ## Save the result back to `s.footnotes`. # Report an error if found any mismatch in number of automatic footnotes proc listFootnotes(lines: seq[int]): string = result.add $lines.len & " (lines " & join(lines, ", ") & ")" - if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteNum), - listFootnotes(p.s.lineFootnoteNumRef)] & + if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteNum), + listFootnotes(s.lineFootnoteNumRef)] & " for auto-numbered footnotes") - if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteSym), - listFootnotes(p.s.lineFootnoteSymRef)] & + if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteSym), + listFootnotes(s.lineFootnoteSymRef)] & " for auto-symbol footnotes") var result: seq[FootnoteSubst] var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] - for fs in p.s.footnotes: + for fs in s.footnotes: if fs.kind == fnManualNumber: manuallyN.add fs elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs else: autoSymbol.add fs @@ -906,26 +911,26 @@ proc orderFootnotes(p: var RstParser) = let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) - p.s.footnotes = result + s.footnotes = result -proc getFootnoteNum(p: var RstParser, label: string): int = +proc getFootnoteNum(s: PRstSharedState, label: string): int = ## get number from label. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.label == label: return fnote.number -proc getFootnoteNum(p: var RstParser, order: int): int = +proc getFootnoteNum(s: PRstSharedState, order: int): int = ## get number from occurrence. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoNumIdx == order: return fnote.number -proc getAutoSymbol(p: var RstParser, order: int): string = +proc getAutoSymbol(s: PRstSharedState, order: int): string = ## get symbol from occurrence of auto-symbol footnote. result = "???" - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoSymIdx == order: return fnote.label @@ -1765,17 +1770,18 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = line: curLine(p), hasPeers: false) result = p.s.hLevels.len - 1 -proc countTitles(p: var RstParser, n: PRstNode) = - ## Fill `p.s.hTitleCnt` +proc countTitles(s: PRstSharedState, n: PRstNode) = + ## Fill `s.hTitleCnt` + if n == nil: return for node in n.sons: if node != nil: if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: break if node.kind == rnOverline: - if p.s.hLevels[p.s.hTitleCnt].hasPeers: + if s.hLevels[s.hTitleCnt].hasPeers: break - inc p.s.hTitleCnt - if p.s.hTitleCnt >= 2: + inc s.hTitleCnt + if s.hTitleCnt >= 2: break proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = @@ -2103,8 +2109,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = cols[j] q.line = line - 1 - q.filename = p.filename - q.col += getTokens(row[j], q.tok) + getTokens(row[j], q.tok) b = newRstNode(rnTableDataCell) b.add(parseDoc(q)) a.add(b) @@ -2156,8 +2161,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = p.col q.line = currentTok(p).line - 1 - q.filename = p.filename - q.col += getTokens(getColContents(p, row[j]), q.tok) + getTokens(getColContents(p, row[j]), q.tok) b.add(parseDoc(q)) a.add(b) result.add(a) @@ -2562,8 +2566,8 @@ proc dirInclude(p: var RstParser): PRstNode = var q: RstParser initParser(q, p.s) - q.filename = path - q.col += getTokens( + q.s.filename = path + getTokens( inputString[startPosition..endPosition].strip(), q.tok) # workaround a GCC bug; more like the interior pointer bug? @@ -2806,7 +2810,27 @@ proc parseDotDot(p: var RstParser): PRstNode = else: result = parseComment(p) -proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = +proc rstParsePass1*(fragment, filename: string, + line, column: int, + options: RstParseOptions, + sharedState: PRstSharedState): PRstNode = + ## Parses an RST `fragment`. + ## The result should be further processed by + ## `preparePass2` and `resolveSubs` (which is pass 2). + var p: RstParser + initParser(p, sharedState) + p.line = line + p.col = column + getTokens(fragment, p.tok) + result = parseDoc(p) + +proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = + ## Records titles in node `mainNode` and orders footnotes. + countTitles(s, mainNode) + orderFootnotes(s) + +proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = + ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. ## Takes input node `n` and returns the same node with recursive ## substitutions in `n.sons` to `result`. @@ -2814,32 +2838,32 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = if n == nil: return case n.kind of rnSubstitutionReferences: - var x = findSub(p, n) + var x = findSub(s, n) if x >= 0: - result = p.s.subs[x].value + result = s.subs[x].value else: var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(p, mwUnknownSubstitution, key) + else: rstMessage(s, mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle - if p.s.hTitleCnt == 2: + if s.hTitleCnt == 2: if n.level == 1: # it's the subtitle n.level = 0 elif n.level >= 2: # normal sections n.level -= 1 - elif p.s.hTitleCnt == 0: + elif s.hTitleCnt == 0: n.level += 1 of rnRef: let refn = rstnodeToRefname(n) - var y = findRef(p, refn) + var y = findRef(s, refn) if y != nil: result = newRstNode(rnHyperlink) let text = newRstNode(rnInner, n.sons) result.sons = @[text, y] else: - let s = findMainAnchor(p, refn) + let s = findMainAnchor(s, refn) if s != "": result = newRstNode(rnInternalRef) let text = newRstNode(rnInner, n.sons) @@ -2853,16 +2877,16 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = of fnAutoNumberLabel, fnAutoNumber: if fnType == fnAutoNumberLabel: let labelR = rstnodeToRefname(n.sons[0]) - num = getFootnoteNum(p, labelR) + num = getFootnoteNum(s, labelR) else: - num = getFootnoteNum(p, n.order) + num = getFootnoteNum(s, n.order) var nn = newRstNode(rnInner) nn.add newLeaf($num) result.sons[0] = nn of fnAutoSymbol: - let sym = getAutoSymbol(p, n.order) + let sym = getAutoSymbol(s, n.order) n.sons[0].sons[0].text = sym - n.sons[1] = resolveSubs(p, n.sons[1]) + n.sons[1] = resolveSubs(s, n.sons[1]) of rnFootnoteRef: var (fnType, num) = getFootnoteType(n.sons[0]) template addLabel(number: int | string) = @@ -2877,31 +2901,31 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = addLabel num refn.add $num of fnAutoNumber: - addLabel getFootnoteNum(p, n.order) + addLabel getFootnoteNum(s, n.order) refn.add $n.order of fnAutoNumberLabel: - addLabel getFootnoteNum(p, rstnodeToRefname(n)) + addLabel getFootnoteNum(s, rstnodeToRefname(n)) refn.add rstnodeToRefname(n) of fnAutoSymbol: - addLabel getAutoSymbol(p, n.order) + addLabel getAutoSymbol(s, n.order) refn.add $n.order of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let s = findMainAnchor(p, refn) - if s != "": - result.add newLeaf(s) # add link + let anch = findMainAnchor(s, refn) + if anch != "": + result.add newLeaf(anch) # add link else: - rstMessage(p, mwUnknownSubstitution, refn) + rstMessage(s, mwUnknownSubstitution, refn) result.add newLeaf(refn) # add link of rnLeaf: discard of rnContents: - p.hasToc = true + s.hasToc = true else: var regroup = false for i in 0 ..< n.len: - n.sons[i] = resolveSubs(p, n.sons[i]) + n.sons[i] = resolveSubs(s, n.sons[i]) if n.sons[i] != nil and n.sons[i].kind == rnFootnote: regroup = true if regroup: # group footnotes together into rnFootnoteGroup @@ -2924,13 +2948,10 @@ proc rstParse*(text, filename: string, options: RstParseOptions, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil): PRstNode = - var p: RstParser - initParser(p, newSharedState(options, findFile, msgHandler)) - p.filename = filename - p.line = line - p.col = column + getTokens(text, p.tok) - let unresolved = parseDoc(p) - countTitles(p, unresolved) - orderFootnotes(p) - result = resolveSubs(p, unresolved) - hasToc = p.hasToc + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`. + var sharedState = newRstSharedState(options, filename, findFile, msgHandler) + let unresolved = rstParsePass1(text, filename, line, column, + options, sharedState) + preparePass2(sharedState, unresolved) + result = resolveSubs(sharedState, unresolved) + hasToc = sharedState.hasToc diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index b49a777c8e2d7..6dc3953af0ded 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,8 +1,3 @@ -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC @@ -11,3 +6,8 @@ someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header +Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header +And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on +More headers subdir/subdir_b/utils.html#more-headers More headers +Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 34b172935ec26..300229648a5f1 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -997,7 +997,7 @@ Test1 """ var error5 = new string let output5 = input5.toHtml(error=error5) - check(error5[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " & "footnotes") @@ -1011,7 +1011,7 @@ Test1 """ var error6 = new string let output6 = input6.toHtml(error=error6) - check(error6[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " & "footnotes") @@ -1034,7 +1034,8 @@ Test1 """ var warnings8 = new seq[string] let output8 = input8.toHtml(warnings=warnings8) - check(warnings8[] == @["input(4, 1) Warning: unknown substitution " & + # TODO: the line 1 is arbitrary because reference lines are not preserved + check(warnings8[] == @["input(1, 1) Warning: unknown substitution " & "\'citation-som\'"]) # check that footnote group does not break parsing of other directives: