From cb39fc51cf33e9bc1cc6d3e957fac94b822e05e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 17 Jan 2020 12:19:14 -0800 Subject: [PATCH 01/12] fix #13150 `nim doc --project` works with duplicate names and with imports below main project file --- compiler/ast.nim | 2 ++ compiler/commands.nim | 3 +++ compiler/docgen.nim | 35 +++++++++++++++++++++++++++-------- compiler/docgen2.nim | 4 ++-- compiler/main.nim | 3 +++ compiler/modules.nim | 2 ++ compiler/options.nim | 32 ++++++++++++++++++++++++++++++++ compiler/packagehandling.nim | 13 ++++++++----- 8 files changed, 79 insertions(+), 15 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 3d05fd2ca780..5a31a2f283fd 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -823,6 +823,7 @@ type # check for the owner when touching 'usedGenerics'. usedGenerics*: seq[PInstantiation] tab*: TStrTable # interface table for modules + nimblePkg*: PSym of skLet, skVar, skField, skForVar: guard*: PSym bitsize*: int @@ -1391,6 +1392,7 @@ proc createModuleAlias*(s: PSym, newIdent: PIdent, info: TLineInfo; result.ast = s.ast result.id = s.id result.flags = s.flags + result.nimblePkg = s.nimblePkg system.shallowCopy(result.tab, s.tab) result.options = s.options result.position = s.position diff --git a/compiler/commands.nim b/compiler/commands.nim index 6633c1aa2702..ffb7ff7a0804 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -413,6 +413,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "docseesrcurl": expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg + of "docroot": + expectArg(conf, switch, arg, pass, info) + conf.docRoot = processPath(conf, arg, info, notRelativeToProj=true) of "mainmodule", "m": discard "allow for backwards compatibility, but don't do anything" of "define", "d": diff --git a/compiler/docgen.nim b/compiler/docgen.nim index a20d0ea04edd..d95ebd130c4b 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -48,6 +48,20 @@ type PDoc* = ref TDocumentor ## Alias to type less. +proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = + let file2 = $file + let docRoot = conf.getDocRoot() + case $docRoot: + # of "$mixed": relativeToPkg(conf, file) + of "$pkg": # TODO: check this works; right now failing at command line step + let dir = getNimbleFile(conf, file2).parentDir.AbsoluteDir + result = relativeTo(file, dir) + of "$path": + result = getRelativePathFromConfigPath(conf, file) + else: + result = relativeTo(file, docRoot) + result = result.string.replace("..", "@@").RelativeFile + proc whichType(d: PDoc; n: PNode): PSym = if n.kind == nkSym: if d.types.strTableContains(n.sym): @@ -175,7 +189,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, if execShellCmd(c2) != status: rawMessage(conf, errGenerated, "executing of external program failed: " & c2) result.emitted = initIntSet() - result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath), + result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, htmldocsDir, false) result.thisDir = result.destFile.splitFile.dir @@ -295,14 +309,14 @@ proc getPlainDocstring(n: PNode): string = if result.len > 0: return proc belongsToPackage(conf: ConfigRef; module: PSym): bool = - result = module.kind == skModule and module.owner != nil and - module.owner.id == conf.mainPackageId + result = module.kind == skModule and module.nimblePkg != nil and + module.nimblePkg.id == conf.mainPackageId proc externalDep(d: PDoc; module: PSym): string = if optWholeProject in d.conf.globalOptions: let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position) - let tmp = getOutFile2(d.conf, full.relativeTo(d.conf.projectPath), HtmlExt, - htmldocsDir, sfMainModule notin module.flags) + let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt, + htmldocsDir, sfMainModule notin module.flags) result = relativeTo(tmp, d.thisDir, '/').string else: result = extractFilename toFullPath(d.conf, FileIndex module.position) @@ -1013,6 +1027,12 @@ proc genSection(d: PDoc, kind: TSymKind) = proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope = rope($relativeTo(outDir / RelativeFile"nimdoc.out.css", destFile.splitFile().dir, '/')) +proc nativeToUnix(path: string): string = + doAssert not path.isAbsolute # absolute files need more care for the drive + when DirSep == '\\': + result = replace(path, '\\', '/') + else: result = path + proc genOutFile(d: PDoc): Rope = var code, content: Rope @@ -1031,7 +1051,7 @@ proc genOutFile(d: PDoc): Rope = # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: title = d.meta[metaTitle] - let external = AbsoluteFile(d.filename).relativeTo(d.conf.projectPath, '/').changeFileExt(HtmlExt).string + let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnix setIndexTerm(d[], external, "", title) else: # Modules get an automatic title for the HTML, but no entry in the index. @@ -1060,8 +1080,7 @@ proc generateIndex*(d: PDoc) = let dir = if not d.conf.outDir.isEmpty: d.conf.outDir else: d.conf.projectPath / htmldocsDir createDir(dir) - let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename, - d.conf.projectPath), IndexExt) + let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt) writeIndexFile(d[], dest.string) proc writeOutput*(d: PDoc, useWarning = false) = diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index aa4d9cbef497..4aeaaf35053d 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -22,8 +22,8 @@ type config: ConfigRef PGen = ref TGen -template shouldProcess(g): bool = - (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or +proc shouldProcess(g: PGen): bool = + (g.module.nimblePkg.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex template closeImpl(body: untyped) {.dirty.} = diff --git a/compiler/main.nim b/compiler/main.nim index 5efdd87af91e..c8bdc9447566 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -293,7 +293,9 @@ proc mainCommand*(graph: ModuleGraph) = for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s) var libpaths = newJArray() + var lazyPaths = newJArray() for dir in conf.searchPaths: libpaths.elems.add(%dir.string) + for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string) var hints = newJObject() # consider factoring with `listHints` for a in hintMin..hintMax: @@ -311,6 +313,7 @@ proc mainCommand*(graph: ModuleGraph) = (key: "project_path", val: %conf.projectFull.string), (key: "defined_symbols", val: definedSymbols), (key: "lib_paths", val: %libpaths), + (key: "lazyPaths", val: %lazyPaths), (key: "outdir", val: %conf.outDir.string), (key: "out", val: %conf.outFile.string), (key: "nimcache", val: %getNimcacheDir(conf).string), diff --git a/compiler/modules.nim b/compiler/modules.nim index 9ef9dfead47f..4525b8c5c27d 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -27,7 +27,9 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info) initStrTable(packSym.tab) graph.packageSyms.strTableAdd(packSym) + result.nimblePkg = packSym else: + result.nimblePkg = packSym let existing = strTableGet(packSym.tab, result.name) if existing != nil and existing.info.fileIndex != result.info.fileIndex: when false: diff --git a/compiler/options.nim b/compiler/options.nim index d8f6fd0b9d0f..35e87ceb40df 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -263,6 +263,9 @@ type implicitIncludes*: seq[string] # modules that are to be implicitly included docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \ # The string uses the formatting variables `path` and `line`. + docRoot*: AbsoluteDir + # `nim doc --docRoot:foo --project --outdir:docs foo/sub/main.nim` + # genrates: docs/sub/main.html # the used compiler cIncludes*: seq[AbsoluteDir] # directories to search for included files @@ -500,6 +503,9 @@ proc absOutFile*(conf: ConfigRef): AbsoluteFile = if dirExists(result.string): result.string.add ".out" +proc getDocRoot*(conf: ConfigRef): AbsoluteDir = + if conf.docRoot.isEmpty: conf.projectPath else: conf.docRoot + proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile = ## Create the output directory and returns a full path to the output file createDir conf.outDir @@ -656,6 +662,32 @@ template patchModule(conf: ConfigRef) {.dirty.} = let ov = conf.moduleOverrides[key] if ov.len > 0: result = AbsoluteFile(ov) +proc isRelativeTo(path: string, base: string): bool= + # PENDING #13212 + let path = path.normalizedPath + let base = base.normalizedPath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." + +proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeFile = + let f = $f + template search(paths) = + for it in paths: + let it = $it + if f.isRelativeTo(it): + return relativePath(f, it).RelativeFile + search(conf.searchPaths) + search(conf.lazyPaths) + +proc relativeToPkg*(conf: ConfigRef, file: AbsoluteFile): string = + let file2 = $file + let dir = getNimbleFile(conf, file2).parentDir + result = relativePath(file2, dir) + # take care of things like in stdlib with multiple `--path:lib/pure` etc + let path2 = getRelativePathFromConfigPath(conf, file).string + if path2.len > result.len: result = path2 + if result.len == 0: result = file2 + proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile = if f.isAbsolute: result = if f.existsFile: AbsoluteFile(f) else: AbsoluteFile"" diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim index e807ae3774b1..2243e7063479 100644 --- a/compiler/packagehandling.nim +++ b/compiler/packagehandling.nim @@ -15,10 +15,8 @@ iterator myParentDirs(p: string): string = if current.len == 0: break yield current -proc resetPackageCache*(conf: ConfigRef) = - conf.packageCache = newPackageCache() - -proc getPackageName*(conf: ConfigRef; path: string): string = +proc getNimbleFile*(conf: ConfigRef; path: string): string = + ## returns absolute path to nimble file, eg: /pathto/cligen.nimble var parents = 0 block packageSearch: for d in myParentDirs(path): @@ -27,7 +25,7 @@ proc getPackageName*(conf: ConfigRef; path: string): string = return conf.packageCache[d] inc parents for file in walkFiles(d / "*.nimble"): - result = file.splitFile.name + result = file break packageSearch # we also store if we didn't find anything: when not defined(nimNoNilSeqs): @@ -38,6 +36,11 @@ proc getPackageName*(conf: ConfigRef; path: string): string = dec parents if parents <= 0: break +proc getPackageName*(conf: ConfigRef; path: string): string = + ## returns nimble package name, eg: `cligen` + let path = getNimbleFile(conf, path) + result = path.splitFile.name + proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string = # Convert `path` so that 2 modules with same name # in different directory get different name and they can be From 2ef2c67be12c2a41b4eecf4c4f38c6e01c2fee7e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 22 Jan 2020 15:57:26 -0800 Subject: [PATCH 02/12] add to help; fixup after #13212 isRelativeTo got merged --- compiler/options.nim | 17 ++++++++--------- doc/advopt.txt | 2 ++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 35e87ceb40df..288764e45a3f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -263,9 +263,7 @@ type implicitIncludes*: seq[string] # modules that are to be implicitly included docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \ # The string uses the formatting variables `path` and `line`. - docRoot*: AbsoluteDir - # `nim doc --docRoot:foo --project --outdir:docs foo/sub/main.nim` - # genrates: docs/sub/main.html + docRoot*: AbsoluteDir ## see nim --fullhelp for --docRoot # the used compiler cIncludes*: seq[AbsoluteDir] # directories to search for included files @@ -662,12 +660,13 @@ template patchModule(conf: ConfigRef) {.dirty.} = let ov = conf.moduleOverrides[key] if ov.len > 0: result = AbsoluteFile(ov) -proc isRelativeTo(path: string, base: string): bool= - # PENDING #13212 - let path = path.normalizedPath - let base = base.normalizedPath - let ret = relativePath(path, base) - result = path.len > 0 and not ret.startsWith ".." +when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): + proc isRelativeTo(path, base: string): bool = + # pending #13212 use os.isRelativeTo + let path = path.normalizedPath + let base = base.normalizedPath + let ret = relativePath(path, base) + result = path.len > 0 and not ret.startsWith ".." proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeFile = let f = $f diff --git a/doc/advopt.txt b/doc/advopt.txt index 9dd59333cd2c..87de986fd9c7 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -67,6 +67,8 @@ Advanced options: --clib:LIBNAME link an additional C library (you should omit platform-specific extensions) --project document the whole project (doc2) + --docRoot:path nim doc --docRoot:foo --project --outdir:docs foo/sub/main.nim + generates: docs/sub/main.html --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) --docInternal also generate documentation for non-exported symbols From f6cb8f86003a602a5fc6dca08bd601641ff99456 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 22 Jan 2020 17:14:07 -0800 Subject: [PATCH 03/12] make --- compiler/commands.nim | 2 +- compiler/docgen.nim | 25 +++++++++++++++++-------- compiler/options.nim | 5 +---- compiler/pathutils.nim | 3 ++- doc/advopt.txt | 5 ++++- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index ffb7ff7a0804..dc54155f01c6 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -415,7 +415,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.docSeeSrcUrl = arg of "docroot": expectArg(conf, switch, arg, pass, info) - conf.docRoot = processPath(conf, arg, info, notRelativeToProj=true) + conf.docRoot = arg of "mainmodule", "m": discard "allow for backwards compatibility, but don't do anything" of "define", "d": diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d95ebd130c4b..06feeb76cf60 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -49,18 +49,27 @@ type PDoc* = ref TDocumentor ## Alias to type less. proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = + ## returns a relative file that will be appended to outDir let file2 = $file - let docRoot = conf.getDocRoot() - case $docRoot: - # of "$mixed": relativeToPkg(conf, file) - of "$pkg": # TODO: check this works; right now failing at command line step + template bail() = + result = relativeTo(file, conf.projectPath) + case conf.docRoot: + of "@pkg": let dir = getNimbleFile(conf, file2).parentDir.AbsoluteDir - result = relativeTo(file, dir) - of "$path": + if dir.isEmpty: bail() + else: result = relativeTo(file, dir) + of "@path": result = getRelativePathFromConfigPath(conf, file) + if result.isEmpty: bail() + # of "@mixed": result = relativeToPkg(conf, file) # consider enabling this + elif conf.docRoot.len > 0: + doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError + doAssert conf.docRoot.existsDir, conf.docRoot + result = relativeTo(file, conf.docRoot.AbsoluteDir) else: - result = relativeTo(file, docRoot) - result = result.string.replace("..", "@@").RelativeFile + bail() + result = result.string.replace("..", "@@").RelativeFile ## refs #13223 + doAssert not result.isEmpty proc whichType(d: PDoc; n: PNode): PSym = if n.kind == nkSym: diff --git a/compiler/options.nim b/compiler/options.nim index 288764e45a3f..416154f4883d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -263,7 +263,7 @@ type implicitIncludes*: seq[string] # modules that are to be implicitly included docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \ # The string uses the formatting variables `path` and `line`. - docRoot*: AbsoluteDir ## see nim --fullhelp for --docRoot + docRoot*: string ## see nim --fullhelp for --docRoot # the used compiler cIncludes*: seq[AbsoluteDir] # directories to search for included files @@ -501,9 +501,6 @@ proc absOutFile*(conf: ConfigRef): AbsoluteFile = if dirExists(result.string): result.string.add ".out" -proc getDocRoot*(conf: ConfigRef): AbsoluteDir = - if conf.docRoot.isEmpty: conf.projectPath else: conf.docRoot - proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile = ## Create the output directory and returns a full path to the output file createDir conf.outDir diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index e3dd69628856..488d02843fea 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -67,7 +67,7 @@ when true: proc `/`*(base: AbsoluteDir; f: RelativeFile): AbsoluteFile = let base = postProcessBase(base) - assert(not isAbsolute(f.string)) + assert(not isAbsolute(f.string), f.string) result = AbsoluteFile newStringOfCap(base.string.len + f.string.len) var state = 0 addNormalizePath(base.string, result.string, state) @@ -83,6 +83,7 @@ when true: proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir; sep = DirSep): RelativeFile = + assert not baseFilename.isEmpty, $fullPath # else, would return an absolute file RelativeFile(relativePath(fullPath.string, baseFilename.string, sep)) proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile = diff --git a/doc/advopt.txt b/doc/advopt.txt index 87de986fd9c7..a38c41eda1af 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -67,8 +67,11 @@ Advanced options: --clib:LIBNAME link an additional C library (you should omit platform-specific extensions) --project document the whole project (doc2) - --docRoot:path nim doc --docRoot:foo --project --outdir:docs foo/sub/main.nim + --docRoot:path nim doc --docRoot:/foo --project --outdir:docs /foo/sub/main.nim generates: docs/sub/main.html + if path == @pkg, will use nimble file enclosing dir + if path == @path, will use first matching dir in --path + if these are nonexistant, will use project path --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) --docInternal also generate documentation for non-exported symbols From 0634607a7f5a8e148924b69e9adc255de395d7fa Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 22 Jan 2020 17:20:35 -0800 Subject: [PATCH 04/12] remove relativeToPkg --- compiler/docgen.nim | 2 +- compiler/options.nim | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 06feeb76cf60..ec15c8c694f3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -61,7 +61,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = of "@path": result = getRelativePathFromConfigPath(conf, file) if result.isEmpty: bail() - # of "@mixed": result = relativeToPkg(conf, file) # consider enabling this + # we could consider a @besteffort mode that would returned the "best" match among @pkg and @path elif conf.docRoot.len > 0: doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError doAssert conf.docRoot.existsDir, conf.docRoot diff --git a/compiler/options.nim b/compiler/options.nim index 416154f4883d..c39ffa792b0f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -675,15 +675,6 @@ proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeF search(conf.searchPaths) search(conf.lazyPaths) -proc relativeToPkg*(conf: ConfigRef, file: AbsoluteFile): string = - let file2 = $file - let dir = getNimbleFile(conf, file2).parentDir - result = relativePath(file2, dir) - # take care of things like in stdlib with multiple `--path:lib/pure` etc - let path2 = getRelativePathFromConfigPath(conf, file).string - if path2.len > result.len: result = path2 - if result.len == 0: result = file2 - proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile = if f.isAbsolute: result = if f.existsFile: AbsoluteFile(f) else: AbsoluteFile"" From 59b5f9b1103b04606fe5e0ec1cc5274be25d1054 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 22 Jan 2020 19:40:53 -0800 Subject: [PATCH 05/12] fix test tests/compilerapi/tcompilerapi.nim --- compiler/pathutils.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 488d02843fea..1c35eb0b23b5 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -83,8 +83,10 @@ when true: proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir; sep = DirSep): RelativeFile = - assert not baseFilename.isEmpty, $fullPath # else, would return an absolute file - RelativeFile(relativePath(fullPath.string, baseFilename.string, sep)) + # this currently fails for `tests/compilerapi/tcompilerapi.nim` + # it's needed otherwise would returns an absolute path + # assert not baseFilename.isEmpty, $fullPath + result = RelativeFile(relativePath(fullPath.string, baseFilename.string, sep)) proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile = if isAbsolute(file): result = AbsoluteFile(file) From 7517a49c8504ed59b8cfeae97e074a586e5ba2d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 24 Jan 2020 08:35:47 -0800 Subject: [PATCH 06/12] remove nimblePkg field; compute on the fly instead --- compiler/ast.nim | 22 ++++++++++++++++++++-- compiler/docgen.nim | 3 +-- compiler/docgen2.nim | 2 +- compiler/modules.nim | 5 ++--- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 5a31a2f283fd..6ac4a7175b40 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -823,7 +823,6 @@ type # check for the owner when touching 'usedGenerics'. usedGenerics*: seq[PInstantiation] tab*: TStrTable # interface table for modules - nimblePkg*: PSym of skLet, skVar, skField, skForVar: guard*: PSym bitsize*: int @@ -1027,6 +1026,26 @@ const defaultAlignment = -1 defaultOffset = -1 + +proc getnimblePkg*(a: PSym): PSym = + result = a + while result != nil: + case result.kind + of skModule: + result = result.owner + assert result.kind == skPackage + of skPackage: + if result.owner == nil: + break + else: + result = result.owner + else: + assert false, $result.kind + +proc getnimblePkgId*(a: PSym): int = + let b = a.getnimblePkg + result = if b == nil: -1 else: b.id + var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things #var # gMainPackageId*: int @@ -1392,7 +1411,6 @@ proc createModuleAlias*(s: PSym, newIdent: PIdent, info: TLineInfo; result.ast = s.ast result.id = s.id result.flags = s.flags - result.nimblePkg = s.nimblePkg system.shallowCopy(result.tab, s.tab) result.options = s.options result.position = s.position diff --git a/compiler/docgen.nim b/compiler/docgen.nim index ec15c8c694f3..8411bcbf8cf5 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -318,8 +318,7 @@ proc getPlainDocstring(n: PNode): string = if result.len > 0: return proc belongsToPackage(conf: ConfigRef; module: PSym): bool = - result = module.kind == skModule and module.nimblePkg != nil and - module.nimblePkg.id == conf.mainPackageId + result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId proc externalDep(d: PDoc; module: PSym): string = if optWholeProject in d.conf.globalOptions: diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index 4aeaaf35053d..b5ef7e32d955 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -23,7 +23,7 @@ type PGen = ref TGen proc shouldProcess(g: PGen): bool = - (g.module.nimblePkg.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or + (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex template closeImpl(body: untyped) {.dirty.} = diff --git a/compiler/modules.nim b/compiler/modules.nim index 4525b8c5c27d..47d946a01bba 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -27,9 +27,7 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info) initStrTable(packSym.tab) graph.packageSyms.strTableAdd(packSym) - result.nimblePkg = packSym else: - result.nimblePkg = packSym let existing = strTableGet(packSym.tab, result.name) if existing != nil and existing.info.fileIndex != result.info.fileIndex: when false: @@ -41,7 +39,8 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil # but starting with version 0.20 we now produce a fake Nimble package instead # to resolve the conflicts: let pck3 = fakePackageName(graph.config, filename) - packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info) + # this makes the new `packSym`'s owner be the original `packSym` + packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, result.info) initStrTable(packSym.tab) graph.packageSyms.strTableAdd(packSym) From cb9eec6d81487d2b6247771eba4354cd0c3d7267 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 24 Jan 2020 08:45:21 -0800 Subject: [PATCH 07/12] improve title --- compiler/docgen.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8411bcbf8cf5..a1fa08db185d 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -48,7 +48,7 @@ type PDoc* = ref TDocumentor ## Alias to type less. -proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = +proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, mangleDotDot = true): RelativeFile = ## returns a relative file that will be appended to outDir let file2 = $file template bail() = @@ -68,7 +68,8 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = result = relativeTo(file, conf.docRoot.AbsoluteDir) else: bail() - result = result.string.replace("..", "@@").RelativeFile ## refs #13223 + if mangleDotDot: + result = result.string.replace("..", "@@").RelativeFile ## refs #13223 doAssert not result.isEmpty proc whichType(d: PDoc; n: PNode): PSym = @@ -1063,7 +1064,8 @@ proc genOutFile(d: PDoc): Rope = setIndexTerm(d[], external, "", title) else: # Modules get an automatic title for the HTML, but no entry in the index. - title = extractFilename(changeFileExt(d.filename, "")) + # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups + title = $presentationPath(d.conf, AbsoluteFile d.filename, mangleDotDot = false) let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group" elif d.hasToc: "doc.body_toc" From f3ac2ffacb717dbf200539aa609e77f248990a25 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 24 Jan 2020 11:35:15 -0700 Subject: [PATCH 08/12] fix outFile for SuccessX --- compiler/docgen.nim | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index a1fa08db185d..6004d8252f0a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1065,7 +1065,7 @@ proc genOutFile(d: PDoc): Rope = else: # Modules get an automatic title for the HTML, but no entry in the index. # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups - title = $presentationPath(d.conf, AbsoluteFile d.filename, mangleDotDot = false) + title = $presentationPath(d.conf, AbsoluteFile d.filename, mangleDotDot = false).changeFileExt("") let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group" elif d.hasToc: "doc.body_toc" @@ -1093,6 +1093,11 @@ proc generateIndex*(d: PDoc) = let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt) writeIndexFile(d[], dest.string) +proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = + if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html + if d.conf.outFile.isEmpty and not d.conf.outDir.isEmpty: + d.conf.outFile = outfile.relativeTo(d.conf.outDir) + proc writeOutput*(d: PDoc, useWarning = false) = runAllExamples(d) var content = genOutFile(d) @@ -1102,7 +1107,7 @@ proc writeOutput*(d: PDoc, useWarning = false) = template outfile: untyped = d.destFile #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt, htmldocsDir) createDir(outfile.splitFile.dir) - d.conf.outFile = outfile.extractFilename.RelativeFile + updateOutfile(d, outfile) if not writeRope(content, outfile): rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile, outfile.string) @@ -1129,7 +1134,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) = if open(f, d.destFile.string, fmWrite): write(f, $content) close(f) - d.conf.outFile = d.destFile.extractFilename.RelativeFile + updateOutfile(d, d.destFile) else: localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1), warnUser, "unable to open file \"" & d.destFile.string & From 140cfb61379fe7af3a427633d32189fa1d1ef657 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 24 Jan 2020 15:17:05 -0700 Subject: [PATCH 09/12] fix tests --- nimdoc/testproject/expected/subdir/subdir_b/utils.html | 4 ++-- tools/kochdocs.nim | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index dc9143390365..75cb2508f786 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -17,7 +17,7 @@ -utils +subdir/subdir_b/utils @@ -71,7 +71,7 @@
-

utils

+

subdir/subdir_b/utils

diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 073dff919048..e22303d6f762 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -319,9 +319,12 @@ proc buildDoc(nimArgs, destPath: string) = destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc for d in items(doc): - commands[i] = nim & " doc $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): + when false: # enable this when bugs fixed + nimArgs2.add " --docroot:@pkg" + commands[i] = nim & " doc $# --git.url:$# --outdir:$# --index:on $#" % + [nimArgs2, gitUrl, destPath, d] i.inc for d in items(withoutIndex): commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" % From 5a4c363b33da057d502dfe273a7de23eb2b49d07 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 25 Jan 2020 11:30:48 -0700 Subject: [PATCH 10/12] fix for windows --- compiler/docgen.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 6004d8252f0a..0085569a429e 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -48,7 +48,13 @@ type PDoc* = ref TDocumentor ## Alias to type less. -proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, mangleDotDot = true): RelativeFile = +proc nativeToUnix(path: string): string = + doAssert not path.isAbsolute # absolute files need more care for the drive + when DirSep == '\\': + result = replace(path, '\\', '/') + else: result = path + +proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile = ## returns a relative file that will be appended to outDir let file2 = $file template bail() = @@ -68,7 +74,9 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, mangleDotDot = true) result = relativeTo(file, conf.docRoot.AbsoluteDir) else: bail() - if mangleDotDot: + if isTitle: + result = result.string.nativeToUnix.RelativeFile + else: result = result.string.replace("..", "@@").RelativeFile ## refs #13223 doAssert not result.isEmpty @@ -1036,12 +1044,6 @@ proc genSection(d: PDoc, kind: TSymKind) = proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope = rope($relativeTo(outDir / RelativeFile"nimdoc.out.css", destFile.splitFile().dir, '/')) -proc nativeToUnix(path: string): string = - doAssert not path.isAbsolute # absolute files need more care for the drive - when DirSep == '\\': - result = replace(path, '\\', '/') - else: result = path - proc genOutFile(d: PDoc): Rope = var code, content: Rope @@ -1065,7 +1067,7 @@ proc genOutFile(d: PDoc): Rope = else: # Modules get an automatic title for the HTML, but no entry in the index. # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups - title = $presentationPath(d.conf, AbsoluteFile d.filename, mangleDotDot = false).changeFileExt("") + title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("") let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group" elif d.hasToc: "doc.body_toc" From 9ee60cfc335f82c66802d66d5517ee0a424bc932 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 27 Jan 2020 14:52:52 -0700 Subject: [PATCH 11/12] kochdocs: compiler docs now under compiler/ --- compiler/docgen.nim | 2 +- tools/kochdocs.nim | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 0085569a429e..f6de4a474d94 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -330,7 +330,7 @@ proc belongsToPackage(conf: ConfigRef; module: PSym): bool = result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId proc externalDep(d: PDoc; module: PSym): string = - if optWholeProject in d.conf.globalOptions: + if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0: let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position) let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt, htmldocsDir, sfMainModule notin module.flags) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index e22303d6f762..21353cf46e53 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -321,8 +321,7 @@ proc buildDoc(nimArgs, destPath: string) = for d in items(doc): var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): - when false: # enable this when bugs fixed - nimArgs2.add " --docroot:@pkg" + nimArgs2.add " --docroot:@pkg" commands[i] = nim & " doc $# --git.url:$# --outdir:$# --index:on $#" % [nimArgs2, gitUrl, destPath, d] i.inc From 6c5a635ab6387d8fff6d5b32620daaa5ae5380c6 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 30 Jan 2020 16:45:49 -0800 Subject: [PATCH 12/12] --docRoot now has smart default: best among @pkg, @path --- compiler/commands.nim | 3 +-- compiler/docgen.nim | 13 +++++++++++-- doc/advopt.txt | 2 ++ tools/kochdocs.nim | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index dc54155f01c6..9b79dc47a1bc 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -414,8 +414,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg of "docroot": - expectArg(conf, switch, arg, pass, info) - conf.docRoot = arg + conf.docRoot = if arg.len == 0: "@default" else: arg of "mainmodule", "m": discard "allow for backwards compatibility, but don't do anything" of "define", "d": diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f6de4a474d94..15ff2e167e74 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -59,15 +59,24 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re let file2 = $file template bail() = result = relativeTo(file, conf.projectPath) + proc nimbleDir(): AbsoluteDir = + getNimbleFile(conf, file2).parentDir.AbsoluteDir case conf.docRoot: + of "@default": # using `@` instead of `$` to avoid shell quoting complications + result = getRelativePathFromConfigPath(conf, file) + let dir = nimbleDir() + if not dir.isEmpty: + let result2 = relativeTo(file, dir) + if not result2.isEmpty and (result.isEmpty or result2.string.len < result.string.len): + result = result2 + if result.isEmpty: bail() of "@pkg": - let dir = getNimbleFile(conf, file2).parentDir.AbsoluteDir + let dir = nimbleDir() if dir.isEmpty: bail() else: result = relativeTo(file, dir) of "@path": result = getRelativePathFromConfigPath(conf, file) if result.isEmpty: bail() - # we could consider a @besteffort mode that would returned the "best" match among @pkg and @path elif conf.docRoot.len > 0: doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError doAssert conf.docRoot.existsDir, conf.docRoot diff --git a/doc/advopt.txt b/doc/advopt.txt index a38c41eda1af..64d7a66e3721 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -71,6 +71,8 @@ Advanced options: generates: docs/sub/main.html if path == @pkg, will use nimble file enclosing dir if path == @path, will use first matching dir in --path + if path == @default (the default and most useful), will use + best match among @pkg,@path. if these are nonexistant, will use project path --docSeeSrcUrl:url activate 'see source' for doc and doc2 commands (see doc.item.seesrc in config/nimdoc.cfg) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 21353cf46e53..0d48bb5c726c 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -321,7 +321,7 @@ proc buildDoc(nimArgs, destPath: string) = for d in items(doc): var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): - nimArgs2.add " --docroot:@pkg" + nimArgs2.add " --docroot" commands[i] = nim & " doc $# --git.url:$# --outdir:$# --index:on $#" % [nimArgs2, gitUrl, destPath, d] i.inc