From cc23773f42d8cfc25a8cc43721ca4d3423ec6a1d Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jul 2021 19:16:26 +0100 Subject: [PATCH 1/4] Change `index_checkpoint_files` to not filter to JLSO --- Project.toml | 2 +- src/indexing.jl | 4 ++-- test/indexing.jl | 13 +++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 10267c5..13f8791 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = "Invenia Technical Computing Corporation" -version = "0.3.9" +version = "0.4.0" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" diff --git a/src/indexing.jl b/src/indexing.jl index 65e809b..08d9052 100644 --- a/src/indexing.jl +++ b/src/indexing.jl @@ -122,7 +122,7 @@ end """ index_checkpoint_files(dir) -Constructs a index for all the files output by checkpoints located within `dir`. +Constructs a index for all the files located within `dir`. This index tells you their checkpoint_name, checkpoint_path, tags, etc. See [`IndexEntry`](@ref) for full information on what is recorded. @@ -139,7 +139,7 @@ You can also work with it directly, say you wanted to get all checkpoints files 2: https://github.com/JuliaData/DataFrames.jl """ function index_checkpoint_files(dir::AbstractPath) - map(Iterators.filter(==("jlso") ∘ extension, walkpath(dir))) do checkpoint_path + map(Iterators.filter(isfile, walkpath(dir))) do checkpoint_path return IndexEntry(checkpoint_path) end end diff --git a/test/indexing.jl b/test/indexing.jl index 8c989d8..7c44df8 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -19,6 +19,19 @@ end end + @testset "files not saved by Checkpoints.jl" begin + mktempdir(SystemPath) do path + Checkpoints.config("TestPkg.bar", path) + TestPkg.bar([1,2,3]) + other_file = joinpath(path, "date=2021-01-01", "one") + mkpath(dirname(other_file)) + write(other_file, 1) + @show index = index_checkpoint_files(path) + @test length(index) == 2 + @test other_file == only(checkpoint_path(entry) for entry in index if entry.date == "2021-01-01") + end + end + @testset "clashing tags" begin mktempdir() do path Checkpoints.config("TestPkg.bar", path) From 48bc83e8fa797570cd830baf5d741226400eaefe Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jul 2021 19:20:18 +0100 Subject: [PATCH 2/4] Introduce `index_files` function not restricted to JLSO --- Project.toml | 2 +- src/Checkpoints.jl | 2 +- src/indexing.jl | 18 ++++++++++++++++-- test/indexing.jl | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Project.toml b/Project.toml index 13f8791..1645a73 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = "Invenia Technical Computing Corporation" -version = "0.4.0" +version = "0.3.10" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 622ea0e..bf49bec 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -20,7 +20,7 @@ using OrderedCollections export checkpoint, with_checkpoint_tags # creating stuff export enabled_checkpoints # indexing stuff -export IndexEntry, index_checkpoint_files +export IndexEntry, index_checkpoint_files, index_files export checkpoint_name, checkpoint_path, prefixes, tags const LOGGER = getlogger(@__MODULE__) diff --git a/src/indexing.jl b/src/indexing.jl index 08d9052..c9f47f3 100644 --- a/src/indexing.jl +++ b/src/indexing.jl @@ -122,7 +122,7 @@ end """ index_checkpoint_files(dir) -Constructs a index for all the files located within `dir`. +Constructs a index for all the files output by checkpoints located within `dir`. This index tells you their checkpoint_name, checkpoint_path, tags, etc. See [`IndexEntry`](@ref) for full information on what is recorded. @@ -139,9 +139,23 @@ You can also work with it directly, say you wanted to get all checkpoints files 2: https://github.com/JuliaData/DataFrames.jl """ function index_checkpoint_files(dir::AbstractPath) - map(Iterators.filter(isfile, walkpath(dir))) do checkpoint_path + map(Iterators.filter(==("jlso") ∘ extension, walkpath(dir))) do checkpoint_path return IndexEntry(checkpoint_path) end end index_checkpoint_files(dir) = index_checkpoint_files(Path(dir)) + +""" + index_files(dir) + +Constructs a index for all the files located within `dir`. +Same as [`index_checkpoint_files`] except not restricted to files created by Checkpoints.jl. +""" +function index_files(dir::AbstractPath) + map(Iterators.filter(isfile, walkpath(dir))) do path + return IndexEntry(path) + end +end + +index_files(dir) = index_files(Path(dir)) diff --git a/test/indexing.jl b/test/indexing.jl index 7c44df8..e9e74b6 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -26,7 +26,7 @@ other_file = joinpath(path, "date=2021-01-01", "one") mkpath(dirname(other_file)) write(other_file, 1) - @show index = index_checkpoint_files(path) + @show index = index_files(path) @test length(index) == 2 @test other_file == only(checkpoint_path(entry) for entry in index if entry.date == "2021-01-01") end From 6fec26e9c6b7866e301684055e8110207064dc49 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jul 2021 19:21:59 +0100 Subject: [PATCH 3/4] Update test/indexing.jl --- test/indexing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/indexing.jl b/test/indexing.jl index e9e74b6..c8db25f 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -26,7 +26,7 @@ other_file = joinpath(path, "date=2021-01-01", "one") mkpath(dirname(other_file)) write(other_file, 1) - @show index = index_files(path) + index = index_files(path) @test length(index) == 2 @test other_file == only(checkpoint_path(entry) for entry in index if entry.date == "2021-01-01") end From 4d21212fb8830bc1e138c8ff9ef313f042d56c5c Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Thu, 22 Jul 2021 10:46:08 +0100 Subject: [PATCH 4/4] Update test/indexing.jl --- test/indexing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/indexing.jl b/test/indexing.jl index c8db25f..486014f 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -23,7 +23,7 @@ mktempdir(SystemPath) do path Checkpoints.config("TestPkg.bar", path) TestPkg.bar([1,2,3]) - other_file = joinpath(path, "date=2021-01-01", "one") + other_file = joinpath(path, "date=2021-01-01", "other_file.txt") mkpath(dirname(other_file)) write(other_file, 1) index = index_files(path)