WIP: Tracking non-.jl source files

Update src/pkgs.jl

Co-authored-by: Kristoffer Carlsson <kcarlsson89@gmail.com>

add docs

add non_julia program

Create non_jl_test.jl

Update types.jl

Update types.jl

simplify

simplify requirements.

update docs

update test

Create new_test.program

test revising

Update test/fake_lang/non_jl_test.jl

Co-authored-by: Tim Holy <tim.holy@gmail.com>

fix typo in tests

test fake_lang

update tests

docs/src/dev_reference.md

@@ -151,3 +151,16 @@ Revise.git_repo
 ```@docs
 Revise.init_worker
 ```
+
+## Teaching Revise about non-julia source codes
+Revise can be made to work for transpilers from non-Julia languages to Julia with a little effort.
+For example, if you wrote a transpiler from C to Julia, you can define a `struct CFile`
+which overrides enough of the common `String` methods (`abspath`,`isabspath`, `joinpath`, `normpath`,`isfile`,`findfirst`, and `String`),
+it will be supported by Revise if you define a method like
+```
+function Revise.parse_source!(mod_exprs_sigs::Revise.ModuleExprsSigs, file::CFile, mod::Module; kwargs...)
+    ex = # julia Expr returned from running transpiler
+    Revise.process_source!(mod_exprs_sigs, ex, file, mod; kwargs...)
+end
+
+```

src/packagedef.jl

@@ -517,6 +517,7 @@ Use the `pkgdata` version if the files are supplied using relative paths.
 function init_watching(pkgdata::PkgData, files=srcfiles(pkgdata))
     udirs = Set{String}()
     for file in files
+        file = String(file)
         dir, basename = splitdir(file)
         dirfull = joinpath(basedir(pkgdata), dir)
         already_watching_dir = haskey(watched_files, dirfull)
@@ -640,16 +641,19 @@ function handle_deletions(pkgdata, file)
     fi = maybe_parse_from_cache!(pkgdata, file)
     maybe_extract_sigs!(fi)
     mexsold = fi.modexsigs
-    filep = normpath(joinpath(basedir(pkgdata), file))
+    idx = fileindex(pkgdata, file)
+    filep = pkgdata.info.files[idx]
+    if isa(filep, AbstractString)
+        filep = normpath(joinpath(basedir(pkgdata), file))
+    end
     topmod = first(keys(mexsold))
-    fileok = file_exists(filep)
+    fileok = file_exists(String(filep))
     mexsnew = fileok ? parse_source(filep, topmod) : ModuleExprsSigs(topmod)
     if mexsnew !== nothing
         delete_missing!(mexsold, mexsnew)
     end
     if !fileok
         @warn("$filep no longer exists, deleted all methods")
-        idx = fileindex(pkgdata, file)
         deleteat!(pkgdata.fileinfos, idx)
         deleteat!(pkgdata.info.files, idx)
         wl = get(watched_files, basedir(pkgdata), nothing)
@@ -872,7 +876,7 @@ it defaults to `Main`.
 
 If this produces many errors, check that you specified `mod` correctly.
 """
-function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
+function track(mod::Module, file; mode=:sigs, kwargs...)
     isfile(file) || error(file, " is not a file")
     # Determine whether we're already tracking this file
     id = PkgId(mod)
@@ -910,13 +914,13 @@ function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
             CodeTracking._pkgfiles[id] = pkgdata.info
         end
         push!(pkgdata, relpath(file, pkgdata)=>FileInfo(fm))
-        init_watching(pkgdata, (file,))
+        init_watching(pkgdata, (String(file),))
         pkgdatas[id] = pkgdata
     end
     return nothing
 end
 
-function track(file::AbstractString; kwargs...)
+function track(file; kwargs...)
     startswith(file, juliadir) && error("use Revise.track(Base) or Revise.track(<stdlib module>)")
     track(Main, file; kwargs...)
 end
@@ -982,7 +986,7 @@ try fixing it with something like `push!(LOAD_PATH, "/path/to/my/private/repos")
 they will not be automatically tracked.
 (See [`Revise.track`](@ref) to set it up manually.)
 """
-function includet(mod::Module, file::AbstractString)
+function includet(mod::Module, file)
     prev = Base.source_path(nothing)
     if prev === nothing
         file = abspath(file)
@@ -1014,7 +1018,7 @@ function includet(mod::Module, file::AbstractString)
     end
     return nothing
 end
-includet(file::AbstractString) = includet(Main, file)
+includet(file) = includet(Main, file)
 
 """
     Revise.silence(pkg)

src/parsing.jl

@@ -7,7 +7,7 @@ if `filename` defines more module(s) then these will all have separate entries i
 
 If parsing `filename` fails, `nothing` is returned.
 """
-parse_source(filename::AbstractString, mod::Module; kwargs...) =
+parse_source(filename, mod::Module; kwargs...) =
     parse_source!(ModuleExprsSigs(mod), filename, mod; kwargs...)
 
 """
@@ -35,7 +35,7 @@ string. `pos` is the 1-based byte offset from which to begin parsing `src`.
 
 See also [`Revise.parse_source`](@ref).
 """
-function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module; mode::Symbol=:sigs)
+function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module; kwargs...)
     startswith(src, "# REVISE: DO NOT PARSE") && return nothing
     ex = Base.parse_input_line(src; filename=filename)
     ex === nothing && return mod_exprs_sigs
@@ -44,6 +44,10 @@ function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, fil
         ln = count(isequal('\n'), SubString(src, 1, min(pos, length(src)))) + 1
         throw(LoadError(filename, ln, ex.args[1]))
     end
+    return process_source!(mod_exprs_sigs, ex, filename, mod; kwargs...)
+end
+
+function process_source!(mod_exprs_sigs::ModuleExprsSigs, @nospecialize(ex), filename, mod::Module; mode::Symbol=:sigs)
     for (mod, ex) in ExprSplitter(mod, ex)
         if mode === :includet
             try

src/types.jl

@@ -147,9 +147,11 @@ Base.PkgId(pkgdata::PkgData) = PkgId(pkgdata.info)
 CodeTracking.basedir(pkgdata::PkgData) = basedir(pkgdata.info)
 CodeTracking.srcfiles(pkgdata::PkgData) = srcfiles(pkgdata.info)
 
-function fileindex(info, file::AbstractString)
+is_same_file(a, b) = String(a) == String(b)
+
+function fileindex(info, file)
     for (i, f) in enumerate(srcfiles(info))
-        f == file && return i
+        is_same_file(f, file) && return i
     end
     return nothing
 end
@@ -168,7 +170,7 @@ function fileinfo(pkgdata::PkgData, file::AbstractString)
 end
 fileinfo(pkgdata::PkgData, i::Integer) = pkgdata.fileinfos[i]
 
-function Base.push!(pkgdata::PkgData, pr::Pair{<:AbstractString,FileInfo})
+function Base.push!(pkgdata::PkgData, pr::Pair{<:Any,FileInfo})
     push!(srcfiles(pkgdata), pr.first)
     push!(pkgdata.fileinfos, pr.second)
     return pkgdata

test/fake_lang/new_test.program

@@ -0,0 +1 @@
+2=x

test/fake_lang/test.program

@@ -0,0 +1,2 @@
+1=x
+2=y

test/runtests.jl

@@ -3978,3 +3978,5 @@ do_test("Base signatures") && @testset "Base signatures" begin
     # Using the extensive repository of code in Base as a testbed
     include("sigtest.jl")
 end
+
+include("non_jl_test.jl")