Tracking non-.jl source files (#680)

Co-authored-by: Tim Holy <tim.holy@gmail.com>
Co-authored-by: Sebastian Pfitzner <pfitzseb@gmail.com>
Co-authored-by: Keno Fischer <keno@juliacomputing.com>

Project.toml

@@ -17,7 +17,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [compat]
-CodeTracking = "1"
+CodeTracking = "1.1"
 JuliaInterpreter = "0.9"
 LoweredCodeUtils = "2.2"
 OrderedCollections = "1"

docs/src/dev_reference.md

@@ -151,3 +151,16 @@ Revise.git_repo
 ```@docs
 Revise.init_worker
 ```
+
+## Teaching Revise about non-julia source codes
+Revise can be made to work for transpilers from non-Julia languages to Julia with a little effort.
+For example, if you wrote a transpiler from C to Julia, you can define a `struct CFile`
+which overrides enough of the common `String` methods (`abspath`,`isabspath`, `joinpath`, `normpath`,`isfile`,`findfirst`, and `String`),
+it will be supported by Revise if you define a method like
+```
+function Revise.parse_source!(mod_exprs_sigs::Revise.ModuleExprsSigs, file::CFile, mod::Module; kwargs...)
+    ex = # julia Expr returned from running transpiler
+    Revise.process_source!(mod_exprs_sigs, ex, file, mod; kwargs...)
+end
+
+```

src/packagedef.jl

@@ -520,6 +520,7 @@ Use the `pkgdata` version if the files are supplied using relative paths.
 function init_watching(pkgdata::PkgData, files=srcfiles(pkgdata))
     udirs = Set{String}()
     for file in files
+        file = String(file)::String
         dir, basename = splitdir(file)
         dirfull = joinpath(basedir(pkgdata), dir)
         already_watching_dir = haskey(watched_files, dirfull)
@@ -643,16 +644,19 @@ function handle_deletions(pkgdata, file)
     fi = maybe_parse_from_cache!(pkgdata, file)
     maybe_extract_sigs!(fi)
     mexsold = fi.modexsigs
-    filep = normpath(joinpath(basedir(pkgdata), file))
+    idx = fileindex(pkgdata, file)
+    filep = pkgdata.info.files[idx]
+    if isa(filep, AbstractString)
+        filep = normpath(joinpath(basedir(pkgdata), file))
+    end
     topmod = first(keys(mexsold))
-    fileok = file_exists(filep)
+    fileok = file_exists(String(filep)::String)
     mexsnew = fileok ? parse_source(filep, topmod) : ModuleExprsSigs(topmod)
     if mexsnew !== nothing
         delete_missing!(mexsold, mexsnew)
     end
     if !fileok
         @warn("$filep no longer exists, deleted all methods")
-        idx = fileindex(pkgdata, file)
         deleteat!(pkgdata.fileinfos, idx)
         deleteat!(pkgdata.info.files, idx)
         wl = get(watched_files, basedir(pkgdata), nothing)
@@ -875,7 +879,7 @@ it defaults to `Main`.
 
 If this produces many errors, check that you specified `mod` correctly.
 """
-function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
+function track(mod::Module, file; mode=:sigs, kwargs...)
     isfile(file) || error(file, " is not a file")
     # Determine whether we're already tracking this file
     id = Base.moduleroot(mod) == Main ? PkgId(mod, string(mod)) : PkgId(mod)  # see #689 for `Main`
@@ -913,13 +917,13 @@ function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
             CodeTracking._pkgfiles[id] = pkgdata.info
         end
         push!(pkgdata, relpath(file, pkgdata)=>FileInfo(fm))
-        init_watching(pkgdata, (file,))
+        init_watching(pkgdata, (String(file)::String,))
         pkgdatas[id] = pkgdata
     end
     return nothing
 end
 
-function track(file::AbstractString; kwargs...)
+function track(file; kwargs...)
     startswith(file, juliadir) && error("use Revise.track(Base) or Revise.track(<stdlib module>)")
     track(Main, file; kwargs...)
 end
@@ -985,7 +989,7 @@ try fixing it with something like `push!(LOAD_PATH, "/path/to/my/private/repos")
 they will not be automatically tracked.
 (See [`Revise.track`](@ref) to set it up manually.)
 """
-function includet(mod::Module, file::AbstractString)
+function includet(mod::Module, file)
     prev = Base.source_path(nothing)
     file = if prev === nothing
         abspath(file)
@@ -1017,7 +1021,7 @@ function includet(mod::Module, file::AbstractString)
     end
     return nothing
 end
-includet(file::AbstractString) = includet(Main, file)
+includet(file) = includet(Main, file)
 
 """
     Revise.silence(pkg)

src/parsing.jl

@@ -7,7 +7,7 @@ if `filename` defines more module(s) then these will all have separate entries i
 
 If parsing `filename` fails, `nothing` is returned.
 """
-parse_source(filename::AbstractString, mod::Module; kwargs...) =
+parse_source(filename, mod::Module; kwargs...) =
     parse_source!(ModuleExprsSigs(mod), filename, mod; kwargs...)
 
 """
@@ -35,7 +35,7 @@ string. `pos` is the 1-based byte offset from which to begin parsing `src`.
 
 See also [`Revise.parse_source`](@ref).
 """
-function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module; mode::Symbol=:sigs)
+function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, filename::AbstractString, mod::Module; kwargs...)
     startswith(src, "# REVISE: DO NOT PARSE") && return nothing
     ex = Base.parse_input_line(src; filename=filename)
     ex === nothing && return mod_exprs_sigs
@@ -44,6 +44,10 @@ function parse_source!(mod_exprs_sigs::ModuleExprsSigs, src::AbstractString, fil
         ln = count(isequal('\n'), SubString(src, 1, min(pos, length(src)))) + 1
         throw(LoadError(filename, ln, ex.args[1]))
     end
+    return process_source!(mod_exprs_sigs, ex, filename, mod; kwargs...)
+end
+
+function process_source!(mod_exprs_sigs::ModuleExprsSigs, ex, filename, mod::Module; mode::Symbol=:sigs)
     for (mod, ex) in ExprSplitter(mod, ex)
         if mode === :includet
             try

src/types.jl

@@ -147,9 +147,11 @@ Base.PkgId(pkgdata::PkgData) = PkgId(pkgdata.info)
 CodeTracking.basedir(pkgdata::PkgData) = basedir(pkgdata.info)
 CodeTracking.srcfiles(pkgdata::PkgData) = srcfiles(pkgdata.info)
 
-function fileindex(info, file::AbstractString)
+is_same_file(a, b) = String(a) == String(b)
+
+function fileindex(info, file)
     for (i, f) in enumerate(srcfiles(info))
-        f == file && return i
+        is_same_file(f, file) && return i
     end
     return nothing
 end
@@ -168,7 +170,7 @@ function fileinfo(pkgdata::PkgData, file::AbstractString)
 end
 fileinfo(pkgdata::PkgData, i::Integer) = pkgdata.fileinfos[i]
 
-function Base.push!(pkgdata::PkgData, pr::Pair{<:AbstractString,FileInfo})
+function Base.push!(pkgdata::PkgData, pr::Pair{<:Any,FileInfo})
     push!(srcfiles(pkgdata), pr.first)
     push!(pkgdata.fileinfos, pr.second)
     return pkgdata

test/fake_lang/new_test.program

@@ -0,0 +1 @@
+2=x

test/fake_lang/test.program

@@ -0,0 +1,2 @@
+1=x
+2=y

test/non_jl_test.jl

@@ -0,0 +1,48 @@
+struct MyFile
+    file::String
+end
+Base.abspath(file::MyFile) = MyFile(Base.abspath(file.file))
+Base.isabspath(file::MyFile) = Base.isabspath(file.file)
+Base.joinpath(str::String, file::MyFile) = MyFile(Base.joinpath(str, file.file))
+Base.normpath(file::MyFile) = MyFile(Base.normpath(file.file))
+Base.isfile(file::MyFile) = Base.isfile(file.file)
+Base.findfirst(str::String, file::MyFile) = Base.findfirst(str, file.file)
+Base.String(file::MyFile) = file.file
+
+function make_module(file::MyFile)
+    exprs = []
+    for line in eachline(file.file)
+       val, name = split(line, '=')
+       push!(exprs, :(function $(Symbol(name))() $val end))
+    end
+    Expr(:toplevel, :(baremodule fake_lang
+       $(exprs...)
+    end), :(using .fake_lang))
+end
+
+function Base.include(mod::Module, file::MyFile)
+    Core.eval(mod, make_module(file))
+end
+Base.include(file::MyFile) = Base.include(Core.Main, file)
+
+using Revise
+function Revise.parse_source!(mod_exprs_sigs::Revise.ModuleExprsSigs, file::MyFile, mod::Module; kwargs...)
+    ex = make_module(file)
+    Revise.process_source!(mod_exprs_sigs, ex, file, mod; kwargs...)
+end
+
+path = joinpath(@__DIR__, "test.program")
+try
+    cp(joinpath(@__DIR__, "fake_lang", "test.program"), path, force=true)
+    m=MyFile(path)
+    includet(m)
+    Revise.revise()
+    @test fake_lang.y() == "2"
+    @test fake_lang.x() == "1"
+    cp(joinpath(@__DIR__, "fake_lang", "new_test.program"), path, force=true)
+    Revise.revise()
+    @test fake_lang.x() == "2"
+    @test_throws MethodError fake_lang.y()
+finally
+    rm(path, force=true)
+end

test/runtests.jl

@@ -3670,3 +3670,5 @@ do_test("Base signatures") && @testset "Base signatures" begin
     # Using the extensive repository of code in Base as a testbed
     include("sigtest.jl")
 end
+
+include("non_jl_test.jl")