Skip to content

Commit

Permalink
add with_checkpoint_tags: tags with dynamic scope (#15)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mzgubic
mzgubic authored May 7, 2021
1 parent a217d11 commit 8b048b6
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 74 deletions.
6 changes: 5 additions & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
name = "Checkpoints"
uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce"
authors = "Invenia Technical Computing Corporation"
version = "0.3.4"
version = "0.3.5"

[deps]
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
ContextVariablesX = "6add18c4-b38d-439d-96f6-d6bc489c04c5"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
JLSO = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11"
Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9"
OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"

[compat]
AWSCore = "0.5.5, 0.6"
Expand All @@ -22,6 +25,7 @@ julia = "1"
[extras]
AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Expand Down
177 changes: 118 additions & 59 deletions docs/Manifest.toml
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,36 @@
# This file is machine-generated - editing it directly is not advised

[[AWSCore]]
deps = ["Base64", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "SymDict", "UUIDs", "XMLDict"]
git-tree-sha1 = "1d3c3af424f774d1b5f040f2f62ce51f2f790b70"
uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
version = "0.6.17"
[[AWS]]
deps = ["Base64", "Compat", "Dates", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "URIs", "UUIDs", "XMLDict"]
git-tree-sha1 = "cb9e9a2880048c07f63c925c04aae6235ace30b0"
uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
version = "1.38.0"

[[AWSS3]]
deps = ["AWSCore", "Base64", "Dates", "EzXML", "FilePathsBase", "HTTP", "LinearAlgebra", "MbedTLS", "Retry", "SymDict", "UUIDs", "XMLDict"]
git-tree-sha1 = "2c0142a8f53a7b3ab299dd5824b895737f846586"
deps = ["AWS", "Base64", "Dates", "EzXML", "FilePathsBase", "HTTP", "MbedTLS", "OrderedCollections", "Retry", "SymDict", "UUIDs", "XMLDict"]
git-tree-sha1 = "fc337c0e58d571b4b760849c8f318c08562af015"
uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
version = "0.6.12"
version = "0.8.3"

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
deps = ["Pkg"]
git-tree-sha1 = "c30985d8821e0cd73870b17b0ed0ce6dc44cb744"
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
version = "1.3.0"

[[BSON]]
git-tree-sha1 = "dd36d7cf3d185eeaaf64db902c15174b22f5dafb"
git-tree-sha1 = "92b8a8479128367aaab2620b8e73dff632f5ae69"
uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
version = "0.2.6"
version = "0.3.3"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Checkpoints]]
deps = ["AWSS3", "DataStructures", "FilePathsBase", "JLSO", "Memento"]
deps = ["AWSS3", "ContextVariablesX", "DataStructures", "Distributed", "FilePathsBase", "JLSO", "Memento", "OrderedCollections"]
path = ".."
uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce"
version = "0.3.2"
version = "0.3.5"

[[CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
Expand All @@ -40,15 +40,21 @@ version = "0.7.0"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "a706ff10f1cd8dab94f59fd09c0e657db8e77ff0"
git-tree-sha1 = "0a817fbe51c976de090aa8c997b7b719b786118d"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.23.0"
version = "3.28.0"

[[ContextVariablesX]]
deps = ["Compat", "Logging", "UUIDs"]
git-tree-sha1 = "fc81d9dd4ffdafb56680f01e6d9db464a6b3689d"
uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5"
version = "0.1.1"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "fb0aa371da91c1ff9dc7fbed6122d3e411420b9c"
git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.8"
version = "0.18.9"

[[Dates]]
deps = ["Printf"]
Expand All @@ -64,16 +70,20 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[DocStringExtensions]]
deps = ["LibGit2", "Markdown", "Pkg", "Test"]
git-tree-sha1 = "50ddf44c53698f5e784bbebb3f4b21c5807401b1"
git-tree-sha1 = "9d4f64f79012636741cf01133158a54b24924c32"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.3"
version = "0.8.4"

[[Documenter]]
deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
git-tree-sha1 = "a4875e0763112d6d017126f3944f4133abb342ae"
uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
version = "0.25.5"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[ExprTools]]
git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
Expand All @@ -86,16 +96,22 @@ uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
version = "1.1.0"

[[FilePathsBase]]
deps = ["Dates", "LinearAlgebra", "Printf", "Test", "UUIDs"]
git-tree-sha1 = "923fd3b942a11712435682eaa95cc8518c428b2c"
deps = ["Dates", "Mmap", "Printf", "Test", "UUIDs"]
git-tree-sha1 = "0f5e8d0cb91a6386ba47bd1527b240bd5725fbae"
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
version = "0.8.0"
version = "0.9.10"

[[GitHub]]
deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal"]
git-tree-sha1 = "a4f61fc1b1724e6eec1d9333eac2d4b01d8fcc8f"
uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
version = "5.4.0"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
git-tree-sha1 = "c7ec02c4c6a039a98a15f955462cd7aea5df4508"
deps = ["Base64", "Dates", "IniFile", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
git-tree-sha1 = "b855bf8247d6e946c75bb30f593bfe7fe591058d"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.8.19"
version = "0.9.8"

[[IOCapture]]
deps = ["Logging"]
Expand All @@ -119,32 +135,39 @@ uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
version = "1.3.0"

[[JLLWrappers]]
git-tree-sha1 = "c70593677bbf2c3ccab4f7500d0f4dacfff7b75c"
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.1.3"
version = "1.3.0"

[[JLSO]]
deps = ["BSON", "CodecZlib", "FilePathsBase", "Memento", "Pkg", "Serialization"]
git-tree-sha1 = "85124b548bf4c2bb59284d353aa09ffc224d761f"
git-tree-sha1 = "7c1a7948ef16218e7e6aa0e9ca7368557d7185c7"
uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11"
version = "2.4.0"
version = "2.5.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.1"

[[LazyJSON]]
deps = ["JSON", "OrderedCollections", "PropertyDicts"]
git-tree-sha1 = "ce08411caa70e0c9e780f142f59debd89a971738"
uuid = "fc18253b-5e1b-504c-a4a2-9ece4944c004"
version = "0.2.2"
[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Printf"]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

Expand Down Expand Up @@ -172,10 +195,8 @@ uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.3"

[[MbedTLS_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "0eef589dd1c26a3ac9d753fe1a8bcad63f956fa6"
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.16.8+1"

[[Memento]]
deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"]
Expand All @@ -192,32 +213,39 @@ git-tree-sha1 = "916b850daad0d46b8c71f65f719c49957e9513ed"
uuid = "78c3b35d-d492-501b-9361-3d52fe80e533"
version = "0.7.1"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[OrderedCollections]]
git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db"
git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.3.2"
version = "1.4.0"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "6370b5b3cf2ce5a3d2b6f7ab2dc10f374e4d7d2b"
git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.0.14"
version = "1.1.0"

[[Pkg]]
deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "ea79e4c9077208cd3bc5d29631a26bc0cff78902"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.1"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[PropertyDicts]]
git-tree-sha1 = "429d887daee312e73842cabe6b122e310b72e25d"
uuid = "f8a19df8-e894-5f55-a973-672c1158cbca"
version = "0.1.0"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets"]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
Expand Down Expand Up @@ -247,6 +275,12 @@ uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SodiumSeal]]
deps = ["Base64", "Libdl", "libsodium_jll"]
git-tree-sha1 = "80cef67d2953e33935b41c6ab0a178b9987b1c99"
uuid = "2133526b-2bfb-4018-ac12-889fb3908a75"
version = "0.1.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
Expand All @@ -267,22 +301,35 @@ git-tree-sha1 = "46badfcc7c6e74535cc7d833a91f4ac4f805f86d"
uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4"
version = "0.3.0"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[TimeZones]]
deps = ["Dates", "EzXML", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"]
git-tree-sha1 = "e8a5ab7e56d23bf147585001d33d969c655d4091"
git-tree-sha1 = "3f6f0be07f33e33bd986a58b4cf2d6c9fd2b7f18"
uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53"
version = "1.5.2"
version = "1.5.4"

[[TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.5"

[[URIs]]
git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
version = "1.3.0"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
Expand All @@ -292,9 +339,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[XML2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "be0db24f70aae7e2b89f2f3092e93b8606d659a6"
git-tree-sha1 = "afd2b541e8fd425cd3b7aa55932a257035ab4a70"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.9.10+3"
version = "2.9.11+0"

[[XMLDict]]
deps = ["EzXML", "IterTools", "OrderedCollections"]
Expand All @@ -303,7 +350,19 @@ uuid = "228000da-037f-5747-90a9-8195ccbf91a5"
version = "0.4.1"

[[Zlib_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6"
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.11+18"

[[libsodium_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "848ab3d00fe39d6fbc2a8641048f8f272af1c51e"
uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8"
version = "1.0.20+0"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
14 changes: 14 additions & 0 deletions docs/src/usage.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,20 @@ Dict{String,Any} with 1 entry:
"data" => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469]
```
#### Tag Context
It is possible to introduce a tag context, such that all the [`checkpoint`](@ref) operations inside the context are tagged with the given tags.
The context is [dynamically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope_vs._dynamic_scope_2), meaning that the tags persist through function calls.
Tag contexts can be nested and can be used inside the package as well as in the application.
For example:
```julia
with_checkpoint_tags(:tag=>1, :othertag=>2) do
bar([1., 2., 3.])
end
```
will result in recording the checkpoint at `"./checkpoints/tag=1/othertag=2/date=2017-01-01/TestPkg/bar.jlso"` _without having to pass `:tag` and `:othertag` directly to bar_.
## Sessions
If you'd like to iteratively checkpoint data (e.g., in a loop) then we recommend using a session.
Expand Down
Loading

0 comments on commit 8b048b6

Please sign in to comment.