From 05cda2bb96b2d5eaf7b8d1923ed701fb30192988 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 10 Oct 2018 15:24:21 -0500 Subject: [PATCH 01/52] Files generated by PkgTemplates --- .gitignore | 4 ++ .gitlab-ci.yml | 40 ++++++++++++++++++++ README.md | 5 +++ REQUIRE | 1 + docs/make.jl | 15 ++++++++ docs/src/assets/invenia.css | 75 +++++++++++++++++++++++++++++++++++++ docs/src/index.md | 5 +++ src/Checkpoints.jl | 5 +++ test/runtests.jl | 5 +++ 9 files changed, 155 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 README.md create mode 100644 REQUIRE create mode 100644 docs/make.jl create mode 100644 docs/src/assets/invenia.css create mode 100644 docs/src/index.md create mode 100644 src/Checkpoints.jl create mode 100644 test/runtests.jl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4637e7a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +# Documenter generated files +/docs/build/ +/docs/site/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..0ffc2df --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,40 @@ +.test_template: &test_definition + # Uncomment below if you would like to run the tests on specific references + # only, such as the branches `master`, `develop`, etc. + # only: + # - master + # - develop + script: + # Substitute `coverage = false` below, if you do not want coverage results. + - /opt/julia/bin/julia -e 'Pkg.clone(pwd()); Pkg.build("Checkpoints"); Pkg.test("Checkpoints", + coverage = true)' + # Comment out below if you do not want coverage results. + - /opt/julia/bin/julia -e 'Pkg.add("Coverage"); cd(Pkg.dir("Checkpoints")); + using Coverage; cl, tl = get_summary(process_folder()); + println("(", cl/tl*100, "%) covered")' + +test:0.5.0: + image: julialang/julia:v0.5.0 + tags: + - docker + <<: *test_definition + +test:0.5.0-dev: + image: julialang/julia:v0.5.0-dev + tags: + - docker + <<: *test_definition + +test:0.6.0-dev: + image: julialang/julia:v0.6.0-dev + tags: + - docker + allow_failure: true + <<: *test_definition + +test:latest: + image: julialang/julia:latest + tags: + - docker + allow_failure: true + <<: *test_definition diff --git a/README.md b/README.md new file mode 100644 index 0000000..c5cb387 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Checkpoints +[![stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![build status](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) +[![coverage](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) diff --git a/REQUIRE b/REQUIRE new file mode 100644 index 0000000..137767a --- /dev/null +++ b/REQUIRE @@ -0,0 +1 @@ +julia 0.6 diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..28d595a --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,15 @@ +using Documenter, Checkpoints + +makedocs( + modules=[Checkpoints], + format=:html, + pages=[ + "Home" => "index.md", + ], + repo="https://gitlab.invenia.ca/invenia/Checkpoints.jl/blob/{commit}{path}#L{line}", + sitename="Checkpoints.jl", + authors="rofinn", + assets=[ + "assets/invenia.css", + ], +) diff --git a/docs/src/assets/invenia.css b/docs/src/assets/invenia.css new file mode 100644 index 0000000..343c6f2 --- /dev/null +++ b/docs/src/assets/invenia.css @@ -0,0 +1,75 @@ +/* Links */ + +a { + color: #4595D1; +} + +a:hover, a:focus { + color: #194E82; +} + +/* Navigation */ + +nav.toc ul a:hover, +nav.toc ul.internal a:hover { + color: #FFFFFF; + background-color: #4595D1; +} + +nav.toc ul .toctext { + color: #FFFFFF; +} + +nav.toc { + box-shadow: none; + color: #FFFFFF; + background-color: #194E82; +} + +nav.toc li.current > .toctext { + color: #FFFFFF; + background-color: #4595D1; + border-top-width: 0px; + border-bottom-width: 0px; +} + +nav.toc ul.internal a { + color: #194E82; + background-color: #FFFFFF; +} + +/* Text */ + +article#docs a.nav-anchor { + color: #194E82; +} + +article#docs blockquote { + font-style: italic; +} + +/* Code */ + +code .hljs-meta { + color: #4595D1; +} + +code .hljs-keyword { + color: #194E82; +} + +pre, code { + font-family: "Liberation Mono", "Consolas", "DejaVu Sans Mono", "Ubuntu Mono", "Courier New", "andale mono", "lucida console", monospace; +} + +/* mkdocs (old) */ + +/*.navbar-default { + background-color: #194E82; +} + +.navbar-default .navbar-nav > .active > a, +.navbar-default .navbar-nav > .active > a:hover, +.navbar-default .navbar-nav > .active > a:focus { + background-color: #4595D1; +}*/ diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..c5cb387 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,5 @@ +# Checkpoints +[![stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![build status](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) +[![coverage](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl new file mode 100644 index 0000000..273ee33 --- /dev/null +++ b/src/Checkpoints.jl @@ -0,0 +1,5 @@ +module Checkpoints + +# Package code goes here. + +end # module diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..2bba8f0 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,5 @@ +using Checkpoints +using Base.Test + +# Write your own tests here. +@test 1 == 2 From 4486e3c62e02e9d85bf5040db2e870e60e0f697d Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 10 Oct 2018 15:33:10 -0500 Subject: [PATCH 02/52] Updated CI settings. --- .gitlab-ci.yml | 186 +++++++++++++++++++++++++++++++++++++++-------- test/runtests.jl | 2 +- 2 files changed, 156 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ffc2df..000d1ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,40 +1,164 @@ -.test_template: &test_definition - # Uncomment below if you would like to run the tests on specific references - # only, such as the branches `master`, `develop`, etc. - # only: - # - master - # - develop +stages: + - test + - coverage + - docs + +.test_shell: &test_shell + artifacts: + name: "$CI_JOB_NAME coverage" + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage/" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install-cred-helper + - ./julia-ci install $JULIA_VERSION script: - # Substitute `coverage = false` below, if you do not want coverage results. - - /opt/julia/bin/julia -e 'Pkg.clone(pwd()); Pkg.build("Checkpoints"); Pkg.test("Checkpoints", - coverage = true)' - # Comment out below if you do not want coverage results. - - /opt/julia/bin/julia -e 'Pkg.add("Coverage"); cd(Pkg.dir("Checkpoints")); - using Coverage; cl, tl = get_summary(process_folder()); - println("(", cl/tl*100, "%) covered")' + - source julia-ci export + - ./julia-ci test + - ./julia-ci coverage + after_script: + - ./julia-ci clean + + +.test_shell_0_6: &test_shell_0_6 + variables: + JULIA_VERSION: "0.6" + ONLINE: "" + <<: *test_shell + +.test_shell_1_0: &test_shell_1_0 + variables: + JULIA_VERSION: "1.0" + ONLINE: "" + <<: *test_shell -test:0.5.0: - image: julialang/julia:v0.5.0 +.test_shell_nightly: &test_shell_nightly + variables: + JULIA_VERSION: "nightly" + ONLINE: "" + allow_failure: true + <<: *test_shell + + +.test_docker: &test_docker + artifacts: + name: coverage + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage/" + variables: + ONLINE: "" + script: + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - julia-coverage $PKG_NAME + +.test_docker_0_6: &test_docker_0_6 + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + <<: *test_docker + + +"0.6 (Mac)": tags: - - docker - <<: *test_definition + - mac + - shell-ci + <<: *test_shell_0_6 -test:0.5.0-dev: - image: julialang/julia:v0.5.0-dev +"0.6 (Linux, 64-bit)": tags: - - docker - <<: *test_definition + - linux + - 64-bit + - docker-ci + <<: *test_docker_0_6 -test:0.6.0-dev: - image: julialang/julia:v0.6.0-dev +"0.6 (Linux, 32-bit)": tags: - - docker - allow_failure: true - <<: *test_definition + - linux + - 32-bit + - shell-ci + <<: *test_shell_0_6 -test:latest: - image: julialang/julia:latest +"1.0 (Mac)": tags: - - docker - allow_failure: true - <<: *test_definition + - mac + - shell-ci + <<: *test_shell_1_0 + +"1.0 (Linux, 64-bit)": + tags: + - linux + - 64-bit + - shell-ci + <<: *test_shell_1_0 + +"1.0 (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_1_0 + +"Nightly (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_nightly + +"Nightly (Linux, 64-bit)": + tags: + - linux + - 64-bit + - shell-ci + <<: *test_shell_nightly + +"Nightly (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_nightly + +"Coverage": + stage: coverage + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + coverage: /Test Coverage (\d+\.\d+%)/ + artifacts: + name: combined_coverage + expire_in: 1 week + paths: + - combined_coverage/ + tags: + - linux + - docker-ci + script: + - genhtml --version + - mkdir $CI_PROJECT_DIR/combined_coverage + - cp -r $CI_PROJECT_DIR/src $CI_PROJECT_DIR/combined_coverage/ + - ls */*.info | xargs -I{} echo '--summary "{}"' | xargs lcov --directory src + - echo "Test Coverage $(genhtml -o $CI_PROJECT_DIR/combined_coverage --no-prefix */coverage.info 2>&1 | grep lines | awk '{print $2}')" + - find $CI_PROJECT_DIR/combined_coverage -type f -name "*.jl" -delete + + +"Documentation": + stage: docs + tags: + - docs + only: + - master + - develop + - docs + - tags # special keyword for all tags + variables: + JULIA_VERSION: "0.6" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION + script: + - source julia-ci export + - julia --depwarn=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.add(\"Documenter\")" + - julia --depwarn=no -e "cd(Pkg.dir(\"$PKG_NAME\", \"docs\")); include(\"make.jl\")" + - julia --depwarn=no -e "const DOCS_DIR = joinpath(\"/mnt\", \"docs\", \"$CI_PROJECT_NAMESPACE\", \"$CI_PROJECT_NAME\", \"$CI_BUILD_REF_NAME\"); mkpath(DOCS_DIR); cp(Pkg.dir(\"$PKG_NAME\", \"docs\", \"build\"), DOCS_DIR; remove_destination=true, follow_symlinks=true)" + after_script: + - ./julia-ci clean diff --git a/test/runtests.jl b/test/runtests.jl index 2bba8f0..8960b6d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,4 +2,4 @@ using Checkpoints using Base.Test # Write your own tests here. -@test 1 == 2 +@test 1 == 1 From 92d71af8e967c6aa0d8a1c779c08f5aa613a3560 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Fri, 12 Oct 2018 16:57:20 -0500 Subject: [PATCH 03/52] Initial work getting a julia serialized object (JSO) file format working. --- REQUIRE | 5 + src/Checkpoints.jl | 96 ++++++++++++++++- src/JSO.jl | 261 +++++++++++++++++++++++++++++++++++++++++++++ test/JSO.jl | 214 +++++++++++++++++++++++++++++++++++++ test/REQUIRE | 4 + test/runtests.jl | 7 +- 6 files changed, 583 insertions(+), 4 deletions(-) create mode 100644 src/JSO.jl create mode 100644 test/JSO.jl create mode 100644 test/REQUIRE diff --git a/REQUIRE b/REQUIRE index 137767a..84adedb 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1 +1,6 @@ julia 0.6 +AWSCore 0.3 +AWSS3 +Compat 0.69.0 +Memento 0.7.0 +Mocking 0.5.2 diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 273ee33..237c48e 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -1,5 +1,99 @@ +""" + Checkpoints + +A very minimal module for defining checkpoints or save location in large codebase with +the ability to configure how those checkpoints save data externally +(similar to how Memento.jl works for logging). +""" module Checkpoints -# Package code goes here. +using AWSCore +using AWSS3 +# using FileIO +using Memento +using Mocking + +using Compat: @__MODULE__ + +export JSO + +const CHECKPOINTS = Dict{String, Function}() +const LOGGER = getlogger(@__MODULE__) + +__init__() = Memento.register(LOGGER) + +include("JSO.jl") + +# function filesave(prefix; ext="jld2") +# function f(label, data) +# parts = split(label, '.') +# parts[end] = string(parts[end], '.', ext) +# path = joinpath(prefix, parts...) +# save(path, data) +# end + +# return f +# end + +# function s3save(config::AWSConfig=AWSCore.aws_config(), bucket, prefix) +# verinfo = sprint(versioninfo, true) +# image = "" + +# # If we're running on AWS batch then store the docker_image +# if haskey(ENV, "AWS_BATCH_JOB_ID") +# job_id = ENV["AWS_BATCH_JOB_ID"] +# response = @mock describe_jobs(Dict("jobs" => [job_id])) + +# if length(response["jobs"]) > 0 +# image = first(response["jobs"])["container"]["image"] +# else +# warn(LOGGER, "No jobs found with id: $job_id.") +# end +# end + +# function f(label, data) +# fileobj = Dict( +# "image" => image, +# "versioninfo" => sprint(verinfo, +# "data" => data, +# ) + +# parts = split(label, '.') +# parts[end] = string(parts[end], '.jso') +# key = join(vcat([prefix], parts), "/") +# s3_put(config, bucket, key, sprint(serialize, fileobj)) +# end + +# return f +# end + +# function s3load(config::AWSConfig=AWSCore.aws_config(), bucket, key) +# obj = s3_get(aws, bucket, key) + +# end + +# function register(labels::String...) +# for l in labels +# if haskey(CHECKPOINTS, l) +# warn(LOGGER, "$l has already registered") +# else +# CHECKPOINTS[l] = (k, v) -> nothing +# end +# end +# end + +# function config(backend::Callable, labels::String...) +# for l in labels +# if haskey(CHECKPOINTS, l) +# CHECKPOINTS[l] = backend +# else +# warn(LOGGER, "$l is not a registered checkpoint label") +# end +# end +# end + +# checkpoint(label::String, x) = CHECKPOINTS[label](label, x) + +# labels() = collect(keys(CHECKPOINTS)) end # module diff --git a/src/JSO.jl b/src/JSO.jl new file mode 100644 index 0000000..24179d7 --- /dev/null +++ b/src/JSO.jl @@ -0,0 +1,261 @@ +""" +A julia serialized object (JSO) file format for storing checkpoint data. + +# Structure +``` +version=1.0 +image=xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest +systeminfo=Julia Version 0.6.4 +--- +var1|5=[0x35, 0x10, 0x01, 0x04, 0x44], +var2|8=[...] +``` +WARNING: The serialized object data is using julia's builtin serialization format which is +not intended for long term storage. As a result, we're storing the serialized object data +in a json file which should also be able to load the docker image and versioninfo to allow +reconstruction. +""" +module JSO + +using AWSCore +using AWSS3 +using Compat +using Memento +using Mocking + +using Compat.Serialization + +const LOGGER = getlogger(@__MODULE__) +const VALID_VERSIONS = (v"1.0", v"2.0") + +# Cache of the versioninfo and image, so we don't compute these every time. +const _CACHE = Dict{Symbol, String}( + :VERSIONINFO => "", + :IMAGE => "", +) + +__init__() = Memento.register(LOGGER) + +struct InvalidFileError <: Exception + msg::String # The msg to display + hpos::Int # Start position in IO + epos::Int # Failure position in IO +end + +struct JSOFile + version::VersionNumber + image::String + systeminfo::String + objects::Dict{String, Vector{UInt8}} +end + +function JSOFile( + data::Dict{String, <:Any}; + image=_image(), + systeminfo=_versioninfo(), + version=v"1.0" +) + _versioncheck(version) + + objects = map(data) do t + varname, vardata = t + io = IOBuffer() + serialize(io, vardata) + return varname => take!(io) + end |> Dict + + return JSOFile(version, image, systeminfo, objects) +end + +JSOFile(data) = JSOFile(Dict("data" => data)) +JSOFile(data::Pair...) = JSOFile(Dict(data...)) + +function Base.:(==)(a::JSOFile, b::JSOFile) + return ( + a.version == b.version && + a.image == b.image && + a.systeminfo == b.systeminfo && + a.objects == b.objects + ) +end + +function Base.write(io::IO, jso::JSOFile) + # Write the header info + header = "version=$(jso.version)\nimage=$(jso.image)\nsysteminfo=$(jso.systeminfo)\n---" + write(io, header) + + for (name, data) in jso.objects + nb = length(data) + write(io, "\n$name|$nb=") + write(io, data) + end +end + +function Base.read(io::IO, ::Type{JSOFile}) + version = v"0.0.0" + img = "" + systeminfo = "" + objects = Dict{String, Vector{UInt8}}() + hpos = position(io) + + version = read_version(io, hpos) + img = read_image(io, hpos) + systeminfo = read_sysinfo(io, hpos) + + ################################ + # Extract each stored variable + ################################ + varname = Compat.readuntil(io, "="; keep=true) + while !isempty(varname) + if !startswith(varname, "\n") + error( + LOGGER, + InvalidFileError( + "Expected newline before variable ($varname).", + hpos, + position(io) + ) + ) + end + + # Strip any whitespace and '=' + varname = strip(varname[1:end-1]) + varname, nb = split(varname, '|') + objects[varname] = read(io, parse(Int, nb)) + + varname = Compat.readuntil(io, "="; keep=true) + end + + return JSOFile(version, img, systeminfo, objects) +end + +function Base.getindex(jso::JSOFile, name::String) + try + return deserialize(IOBuffer(jso.objects[name])) + catch e + warn(LOGGER, e) + return jso.objects[name] + end +end + +# save(io::IO, data) = write(io, JSOFile(data)) +# load(io::IO, data) = read(io, JSOFile(data)) + +######################################### +# Utility function for reading JSO file +######################################## +""" +Extract and validate the format version +""" +function read_version(io, hpos) + str = Compat.readuntil(io, "version="; keep=true) + if isempty(str) + error( + LOGGER, + InvalidFileError("JSO file does not contain 'version='", hpos, position(io)) + ) + end + + if !startswith(str, "version=") + error( + LOGGER, + InvalidFileError("JSO file did not start with 'version='", hpos, position(io)) + ) + end + + str = Compat.readuntil(io, "image="; keep=true) + if isempty(str) + error( + LOGGER, + InvalidFileError("JSO file does not contain 'image='", hpos, position(io)) + ) + end + + tokenized = split(str) + if length(tokenized) == 1 + error( + LOGGER, + InvalidFileError("A version number was not provided", hpos, position(io)) + ) + end + + version = VersionNumber(first(tokenized)) + _versioncheck(version) + + return version +end + +""" +Extract the docker image +""" +function read_image(io, hpos) + str = Compat.readuntil(io, "systeminfo="; keep=true) + if isempty(str) + error( + LOGGER, + InvalidFileError("JSO file does not contain 'systeminfo='", hpos, position(io)) + ) + end + + tokenized = split(str) + if length(tokenized) == 1 + debug(LOGGER, "No docker image specified") + return "" + else + return first(tokenized) + end +end + +""" +Extract the system info +""" +function read_sysinfo(io, hpos) + str = Compat.readuntil(io, "\n---"; keep=true) + if isempty(str) + error( + LOGGER, + InvalidFileError("JSO file missing header separator '---'", hpos, position(io)) + ) + end + + return str[1:end-4] +end + +####################################### +# Functions for lazily evaluating the # +# VERSIONINFO and IMAGE at runtime # +####################################### +function _versioninfo() + if isempty(_CACHE[:VERSIONINFO]) + global _CACHE[:VERSIONINFO] = sprint(versioninfo, true) + end + + return _CACHE[:VERSIONINFO] +end + +function _image() + if isempty(_CACHE[:IMAGE]) && haskey(ENV, "AWS_BATCH_JOB_ID") + job_id = ENV["AWS_BATCH_JOB_ID"] + response = @mock describe_jobs(Dict("jobs" => [job_id])) + + if length(response["jobs"]) > 0 + global _CACHE[:IMAGE] = first(response["jobs"])["container"]["image"] + else + warn(LOGGER, "No jobs found with id: $job_id.") + end + end + + return _CACHE[:IMAGE] +end + +function _versioncheck(version::VersionNumber) + supported = first(VALID_VERSIONS) <= version < last(VALID_VERSIONS) + supported || error(LOGGER, ArgumentError( + string( + "Unsupported version ($version). ", + "Expected a value between ($VALID_VERSIONS)." + ) + )) +end + +end diff --git a/test/JSO.jl b/test/JSO.jl new file mode 100644 index 0000000..ff294d5 --- /dev/null +++ b/test/JSO.jl @@ -0,0 +1,214 @@ +using Compat +using Compat.Test +using Compat.Dates +using Checkpoints +using Checkpoints.JSO: JSOFile, LOGGER, InvalidFileError +using Memento +using Memento.Test + +# To test different types from common external packages +using DataFrames +using Distributions +using TimeZones + +@testset "JSO" begin + # Serialize "Hello World!" on julia 0.5.2 (not supported) + img = JSO._image() + sysinfo = JSO._versioninfo() + hw_5 = UInt8[0x26, 0x15, 0x87, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21] + + datas = Dict( + "String" => "Hello World!", + "Vector" => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], + "Matrix" => [0.400348 0.892196 0.848164; 0.0183529 0.755449 0.397538; 0.870458 0.0441878 0.170899], + "DateTime" => DateTime(2018, 1, 28), + "ZonedDateTime" => ZonedDateTime(2018, 1, 28, tz"America/Chicago"), + "DataFrame" => DataFrame( + :a => collect(1:5), + :b => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], + :c => ["a", "b", "c", "d", "e"], + :d => [true, true, false, false, true], + ), + "Distribution" => Normal(50.2, 4.3), + ) + + @testset "JSOFile" begin + @testset "$k" for (k, v) in datas + jso = JSOFile(v) + io = IOBuffer() + bytes = serialize(io, v) + expected = take!(io) + + @test jso.objects["data"] == expected + end + end + + @testset "reading and writing" begin + @testset "$k" for (k, v) in datas + io = IOBuffer() + orig = JSOFile(v) + write(io, orig) + + seekstart(io) + + result = read(io, JSOFile) + @test result == orig + end + + + @testset "empty io" begin + @test_throws(LOGGER, InvalidFileError, read(IOBuffer(), JSOFile)) + end + + @testset "corrupted start" begin + @test_throws( + LOGGER, + InvalidFileError, + read(IOBuffer("blah\nversion=1.0"), JSOFile) + ) + end + + @testset "empty version" begin + @test_throws( + LOGGER, + InvalidFileError, + read(IOBuffer("version=\nimage="), JSOFile) + ) + end + + @testset "invalid version" begin + jso = JSOFile("Hello World!") + io = IOBuffer() + header = "version=0.0\nimage=\nsysteminfo=$(jso.systeminfo)\n---" + write(io, header) + + for (name, data) in jso.objects + nb = length(data) + write(io, "\n$name|$nb=") + write(io, data) + end + + seekstart(io) + + # read(io, JSOFile) + @test_throws( + LOGGER, + ArgumentError, + read(io, JSOFile) + ) + end + + @testset "missing image" begin + @test_throws( + LOGGER, + InvalidFileError, + read(IOBuffer("version=1.0\nsysteminfo="), JSOFile) + ) + end + + @testset "empty image" begin + jso = JSOFile("Hello World!") + io = IOBuffer() + write(io, jso) + + seekstart(io) + + setlevel!(LOGGER, "debug") + @test_log( + LOGGER, + "debug", + "No docker image specified", + read(io, JSOFile) + ) + end + + @testset "invalid nb" begin + jso = JSOFile("Hello World!") + io = IOBuffer() + header = "version=$(jso.version)\nimage=$(jso.image)\nsysteminfo=$(jso.systeminfo)\n---" + write(io, header) + nb = length(jso.objects["data"]) - 1 + write(io, "\ndata|$nb=") + write(io, jso.objects["data"]) + + seekstart(io) + + # This will cause the parsing our variable names to fail + result = @test_throws( + LOGGER, + InvalidFileError, + read(io, JSOFile) + ) + + io = IOBuffer() + write(io, header) + nb = length(jso.objects["data"]) + 1 + write(io, "\ndata|$nb=") + write(io, jso.objects["data"]) + + seekstart(io) + + result = read(io, JSOFile) + + @test haskey(result.objects, "data") + result["data"] + end + end + + @testset "deserialization" begin + # Test deserialization works + @testset "data - $k" for (k, v) in datas + jso = JSOFile(v) + @test jso["data"] == v + end + + @testset "unsupported julia version" begin + jso = JSOFile(v"1.0", img, sysinfo, Dict("data" => hw_5)) + + # Test failing to deserialize data because of incompatible julia versions + # will will return the raw bytes + result = @test_warn(LOGGER, r"MethodError*", jso["data"]) + @test result == hw_5 + end + + @testset "missing module" begin + # We need to load and use AxisArrays on another process to cause the + # deserialization error + pnum = first(addprocs(1)) + + try + # We need to do this separately because there appears to be a race + # condition on AxisArrays being loaded. + f = @spawnat pnum begin + @eval Main using AxisArrays + end + + fetch(f) + + f = @spawnat pnum begin + io = IOBuffer() + serialize( + io, + AxisArray( + rand(20, 10), + Axis{:time}(14010:10:14200), + Axis{:id}(1:10) + ) + ) + return io + end + + io = fetch(f) + bytes = take!(io) + jso = JSOFile(v"1.0", img, sysinfo, Dict("data" => bytes)) + + # Test failing to deserailize data because of missing modules will + # still return the raw bytes + result = @test_warn(LOGGER, r"UndefVarError*", jso["data"]) + @test result == bytes + finally + rmprocs(pnum) + end + end + end +end diff --git a/test/REQUIRE b/test/REQUIRE new file mode 100644 index 0000000..0415d8c --- /dev/null +++ b/test/REQUIRE @@ -0,0 +1,4 @@ +AxisArrays +DataFrames +Distributions +TimeZones diff --git a/test/runtests.jl b/test/runtests.jl index 8960b6d..b7ea360 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using Checkpoints -using Base.Test +using Compat.Test -# Write your own tests here. -@test 1 == 1 +@testset "Checkpoints" begin + include("JSO.jl") +end From ce0d2311389db0f2f8ee0a3fee066fff6b3800ad Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Sat, 13 Oct 2018 15:42:42 -0500 Subject: [PATCH 04/52] Store JLSO data in BSON. --- REQUIRE | 1 + src/Checkpoints.jl | 4 +- src/JLSO.jl | 167 +++++++++++++++++++++++++++++ src/JSO.jl | 261 --------------------------------------------- test/JLSO.jl | 115 ++++++++++++++++++++ test/JSO.jl | 214 ------------------------------------- test/runtests.jl | 2 +- 7 files changed, 286 insertions(+), 478 deletions(-) create mode 100644 src/JLSO.jl delete mode 100644 src/JSO.jl create mode 100644 test/JLSO.jl delete mode 100644 test/JSO.jl diff --git a/REQUIRE b/REQUIRE index 84adedb..175a0e2 100644 --- a/REQUIRE +++ b/REQUIRE @@ -4,3 +4,4 @@ AWSS3 Compat 0.69.0 Memento 0.7.0 Mocking 0.5.2 +BSON diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 237c48e..307fdfb 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -15,14 +15,14 @@ using Mocking using Compat: @__MODULE__ -export JSO +export JLSO const CHECKPOINTS = Dict{String, Function}() const LOGGER = getlogger(@__MODULE__) __init__() = Memento.register(LOGGER) -include("JSO.jl") +include("JLSO.jl") # function filesave(prefix; ext="jld2") # function f(label, data) diff --git a/src/JLSO.jl b/src/JLSO.jl new file mode 100644 index 0000000..0374fc4 --- /dev/null +++ b/src/JLSO.jl @@ -0,0 +1,167 @@ +""" +A julia serialized object (JLSO) file format for storing checkpoint data. + +# Structure + +The .jlso files are BSON files containing the dictionaries with a specific schema. +NOTE: The raw dictionary should be loadable by any BSON library even if serialized objects +themselves aren't reconstructable. + +Example) +``` +Dict( + "metadata" => Dict( + "format" => v"1.0", + "image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest" + "julia" => v"0.6.4", + "sysinfo" => "Julia Version 0.6.4 ...", + ), + "objects" => Dict( + "var1" => [0x35, 0x10, 0x01, 0x04, 0x44], + "var2" => [...], + ), +) +``` +WARNING: The serialized objects are stored using julia's builtin serialization format which +is not intended for long term storage. As a result, we're storing the serialized object data +in a json file which should also be able to load the docker image and versioninfo to allow +reconstruction. +""" +module JLSO + +using AWSCore +using AWSS3 +using BSON +using Compat +using Memento +using Mocking + +using Compat.Serialization + +const LOGGER = getlogger(@__MODULE__) +const VALID_VERSIONS = (v"1.0", v"2.0") + +# Cache of the versioninfo and image, so we don't compute these every time. +const _CACHE = Dict{Symbol, String}( + :VERSIONINFO => "", + :IMAGE => "", +) + +__init__() = Memento.register(LOGGER) + +struct JLSOFile + format::VersionNumber + image::String + julia::VersionNumber + sysinfo::String + objects::Dict{String, Vector{UInt8}} +end + +function JLSOFile( + data::Dict{String, <:Any}; + image=_image(), + julia=VERSION, + sysinfo=_versioninfo(), + format=v"1.0" +) + _versioncheck(format) + + objects = map(data) do t + varname, vardata = t + io = IOBuffer() + serialize(io, vardata) + return varname => take!(io) + end |> Dict + + return JLSOFile(format, image, julia, sysinfo, objects) +end + +JLSOFile(data) = JLSOFile(Dict("data" => data)) +JLSOFile(data::Pair...) = JLSOFile(Dict(data...)) + +function Base.:(==)(a::JLSOFile, b::JLSOFile) + return ( + a.format == b.format && + a.julia == b.julia && + a.image == b.image && + a.sysinfo == b.sysinfo && + a.objects == b.objects + ) +end + +function Base.write(io::IO, jlso::JLSOFile) + bson( + io, + Dict( + "metadata" => Dict( + "format" => jlso.format, + "image" => jlso.image, + "julia" => jlso.julia, + "sysinfo" => jlso.sysinfo, + ), + "objects" => jlso.objects, + ) + ) +end + +function Base.read(io::IO, ::Type{JLSOFile}) + d = BSON.load(io) + return JLSOFile( + d["metadata"]["format"], + d["metadata"]["image"], + d["metadata"]["julia"], + d["metadata"]["sysinfo"], + d["objects"], + ) +end + +function Base.getindex(jlso::JLSOFile, name::String) + try + return deserialize(IOBuffer(jlso.objects[name])) + catch e + warn(LOGGER, e) + return jlso.objects[name] + end +end + +# save(io::IO, data) = write(io, JLSOFile(data)) +# load(io::IO, data) = read(io, JLSOFile(data)) + +####################################### +# Functions for lazily evaluating the # +# VERSIONINFO and IMAGE at runtime # +####################################### +function _versioninfo() + if isempty(_CACHE[:VERSIONINFO]) + global _CACHE[:VERSIONINFO] = sprint(versioninfo, true) + end + + return _CACHE[:VERSIONINFO] +end + +function _image() + if isempty(_CACHE[:IMAGE]) && haskey(ENV, "AWS_BATCH_JOB_ID") + job_id = ENV["AWS_BATCH_JOB_ID"] + response = @mock describe_jobs(Dict("jobs" => [job_id])) + + if length(response["jobs"]) > 0 + global _CACHE[:IMAGE] = first(response["jobs"])["container"]["image"] + else + warn(LOGGER, "No jobs found with id: $job_id.") + end + end + + return _CACHE[:IMAGE] +end + +function _versioncheck(version::VersionNumber) + supported = first(VALID_VERSIONS) <= version < last(VALID_VERSIONS) + supported || error(LOGGER, ArgumentError( + string( + "Unsupported version ($version). ", + "Expected a value between ($VALID_VERSIONS)." + ) + )) +end + +end diff --git a/src/JSO.jl b/src/JSO.jl deleted file mode 100644 index 24179d7..0000000 --- a/src/JSO.jl +++ /dev/null @@ -1,261 +0,0 @@ -""" -A julia serialized object (JSO) file format for storing checkpoint data. - -# Structure -``` -version=1.0 -image=xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest -systeminfo=Julia Version 0.6.4 ---- -var1|5=[0x35, 0x10, 0x01, 0x04, 0x44], -var2|8=[...] -``` -WARNING: The serialized object data is using julia's builtin serialization format which is -not intended for long term storage. As a result, we're storing the serialized object data -in a json file which should also be able to load the docker image and versioninfo to allow -reconstruction. -""" -module JSO - -using AWSCore -using AWSS3 -using Compat -using Memento -using Mocking - -using Compat.Serialization - -const LOGGER = getlogger(@__MODULE__) -const VALID_VERSIONS = (v"1.0", v"2.0") - -# Cache of the versioninfo and image, so we don't compute these every time. -const _CACHE = Dict{Symbol, String}( - :VERSIONINFO => "", - :IMAGE => "", -) - -__init__() = Memento.register(LOGGER) - -struct InvalidFileError <: Exception - msg::String # The msg to display - hpos::Int # Start position in IO - epos::Int # Failure position in IO -end - -struct JSOFile - version::VersionNumber - image::String - systeminfo::String - objects::Dict{String, Vector{UInt8}} -end - -function JSOFile( - data::Dict{String, <:Any}; - image=_image(), - systeminfo=_versioninfo(), - version=v"1.0" -) - _versioncheck(version) - - objects = map(data) do t - varname, vardata = t - io = IOBuffer() - serialize(io, vardata) - return varname => take!(io) - end |> Dict - - return JSOFile(version, image, systeminfo, objects) -end - -JSOFile(data) = JSOFile(Dict("data" => data)) -JSOFile(data::Pair...) = JSOFile(Dict(data...)) - -function Base.:(==)(a::JSOFile, b::JSOFile) - return ( - a.version == b.version && - a.image == b.image && - a.systeminfo == b.systeminfo && - a.objects == b.objects - ) -end - -function Base.write(io::IO, jso::JSOFile) - # Write the header info - header = "version=$(jso.version)\nimage=$(jso.image)\nsysteminfo=$(jso.systeminfo)\n---" - write(io, header) - - for (name, data) in jso.objects - nb = length(data) - write(io, "\n$name|$nb=") - write(io, data) - end -end - -function Base.read(io::IO, ::Type{JSOFile}) - version = v"0.0.0" - img = "" - systeminfo = "" - objects = Dict{String, Vector{UInt8}}() - hpos = position(io) - - version = read_version(io, hpos) - img = read_image(io, hpos) - systeminfo = read_sysinfo(io, hpos) - - ################################ - # Extract each stored variable - ################################ - varname = Compat.readuntil(io, "="; keep=true) - while !isempty(varname) - if !startswith(varname, "\n") - error( - LOGGER, - InvalidFileError( - "Expected newline before variable ($varname).", - hpos, - position(io) - ) - ) - end - - # Strip any whitespace and '=' - varname = strip(varname[1:end-1]) - varname, nb = split(varname, '|') - objects[varname] = read(io, parse(Int, nb)) - - varname = Compat.readuntil(io, "="; keep=true) - end - - return JSOFile(version, img, systeminfo, objects) -end - -function Base.getindex(jso::JSOFile, name::String) - try - return deserialize(IOBuffer(jso.objects[name])) - catch e - warn(LOGGER, e) - return jso.objects[name] - end -end - -# save(io::IO, data) = write(io, JSOFile(data)) -# load(io::IO, data) = read(io, JSOFile(data)) - -######################################### -# Utility function for reading JSO file -######################################## -""" -Extract and validate the format version -""" -function read_version(io, hpos) - str = Compat.readuntil(io, "version="; keep=true) - if isempty(str) - error( - LOGGER, - InvalidFileError("JSO file does not contain 'version='", hpos, position(io)) - ) - end - - if !startswith(str, "version=") - error( - LOGGER, - InvalidFileError("JSO file did not start with 'version='", hpos, position(io)) - ) - end - - str = Compat.readuntil(io, "image="; keep=true) - if isempty(str) - error( - LOGGER, - InvalidFileError("JSO file does not contain 'image='", hpos, position(io)) - ) - end - - tokenized = split(str) - if length(tokenized) == 1 - error( - LOGGER, - InvalidFileError("A version number was not provided", hpos, position(io)) - ) - end - - version = VersionNumber(first(tokenized)) - _versioncheck(version) - - return version -end - -""" -Extract the docker image -""" -function read_image(io, hpos) - str = Compat.readuntil(io, "systeminfo="; keep=true) - if isempty(str) - error( - LOGGER, - InvalidFileError("JSO file does not contain 'systeminfo='", hpos, position(io)) - ) - end - - tokenized = split(str) - if length(tokenized) == 1 - debug(LOGGER, "No docker image specified") - return "" - else - return first(tokenized) - end -end - -""" -Extract the system info -""" -function read_sysinfo(io, hpos) - str = Compat.readuntil(io, "\n---"; keep=true) - if isempty(str) - error( - LOGGER, - InvalidFileError("JSO file missing header separator '---'", hpos, position(io)) - ) - end - - return str[1:end-4] -end - -####################################### -# Functions for lazily evaluating the # -# VERSIONINFO and IMAGE at runtime # -####################################### -function _versioninfo() - if isempty(_CACHE[:VERSIONINFO]) - global _CACHE[:VERSIONINFO] = sprint(versioninfo, true) - end - - return _CACHE[:VERSIONINFO] -end - -function _image() - if isempty(_CACHE[:IMAGE]) && haskey(ENV, "AWS_BATCH_JOB_ID") - job_id = ENV["AWS_BATCH_JOB_ID"] - response = @mock describe_jobs(Dict("jobs" => [job_id])) - - if length(response["jobs"]) > 0 - global _CACHE[:IMAGE] = first(response["jobs"])["container"]["image"] - else - warn(LOGGER, "No jobs found with id: $job_id.") - end - end - - return _CACHE[:IMAGE] -end - -function _versioncheck(version::VersionNumber) - supported = first(VALID_VERSIONS) <= version < last(VALID_VERSIONS) - supported || error(LOGGER, ArgumentError( - string( - "Unsupported version ($version). ", - "Expected a value between ($VALID_VERSIONS)." - ) - )) -end - -end diff --git a/test/JLSO.jl b/test/JLSO.jl new file mode 100644 index 0000000..2269eb7 --- /dev/null +++ b/test/JLSO.jl @@ -0,0 +1,115 @@ +using Compat +using Compat.Test +using Compat.Dates +using Checkpoints +using Checkpoints.JLSO: JLSOFile, LOGGER +using Memento +using Memento.Test + +# To test different types from common external packages +using DataFrames +using Distributions +using TimeZones + +@testset "JLSO" begin + # Serialize "Hello World!" on julia 0.5.2 (not supported) + img = JLSO._image() + sysinfo = JLSO._versioninfo() + hw_5 = UInt8[0x26, 0x15, 0x87, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21] + + datas = Dict( + "String" => "Hello World!", + "Vector" => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], + "Matrix" => [0.400348 0.892196 0.848164; 0.0183529 0.755449 0.397538; 0.870458 0.0441878 0.170899], + "DateTime" => DateTime(2018, 1, 28), + "ZonedDateTime" => ZonedDateTime(2018, 1, 28, tz"America/Chicago"), + "DataFrame" => DataFrame( + :a => collect(1:5), + :b => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], + :c => ["a", "b", "c", "d", "e"], + :d => [true, true, false, false, true], + ), + "Distribution" => Normal(50.2, 4.3), + ) + + @testset "JLSOFile" begin + @testset "$k" for (k, v) in datas + jlso = JLSOFile(v) + io = IOBuffer() + bytes = serialize(io, v) + expected = take!(io) + + @test jlso.objects["data"] == expected + end + end + + @testset "reading and writing" begin + @testset "$k" for (k, v) in datas + io = IOBuffer() + orig = JLSOFile(v) + write(io, orig) + + seekstart(io) + + result = read(io, JLSOFile) + @test result == orig + end + end + + @testset "deserialization" begin + # Test deserialization works + @testset "data - $k" for (k, v) in datas + jlso = JLSOFile(v) + @test jlso["data"] == v + end + + @testset "unsupported julia version" begin + jlso = JLSOFile(v"1.0", img, VERSION, sysinfo, Dict("data" => hw_5)) + + # Test failing to deserialize data because of incompatible julia versions + # will will return the raw bytes + result = @test_warn(LOGGER, r"MethodError*", jlso["data"]) + @test result == hw_5 + end + + @testset "missing module" begin + # We need to load and use AxisArrays on another process to cause the + # deserialization error + pnum = first(addprocs(1)) + + try + # We need to do this separately because there appears to be a race + # condition on AxisArrays being loaded. + f = @spawnat pnum begin + @eval Main using AxisArrays + end + + fetch(f) + + f = @spawnat pnum begin + io = IOBuffer() + serialize( + io, + AxisArray( + rand(20, 10), + Axis{:time}(14010:10:14200), + Axis{:id}(1:10) + ) + ) + return io + end + + io = fetch(f) + bytes = take!(io) + jlso = JLSOFile(v"1.0", img, VERSION, sysinfo, Dict("data" => bytes)) + + # Test failing to deserailize data because of missing modules will + # still return the raw bytes + result = @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) + @test result == bytes + finally + rmprocs(pnum) + end + end + end +end diff --git a/test/JSO.jl b/test/JSO.jl deleted file mode 100644 index ff294d5..0000000 --- a/test/JSO.jl +++ /dev/null @@ -1,214 +0,0 @@ -using Compat -using Compat.Test -using Compat.Dates -using Checkpoints -using Checkpoints.JSO: JSOFile, LOGGER, InvalidFileError -using Memento -using Memento.Test - -# To test different types from common external packages -using DataFrames -using Distributions -using TimeZones - -@testset "JSO" begin - # Serialize "Hello World!" on julia 0.5.2 (not supported) - img = JSO._image() - sysinfo = JSO._versioninfo() - hw_5 = UInt8[0x26, 0x15, 0x87, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21] - - datas = Dict( - "String" => "Hello World!", - "Vector" => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], - "Matrix" => [0.400348 0.892196 0.848164; 0.0183529 0.755449 0.397538; 0.870458 0.0441878 0.170899], - "DateTime" => DateTime(2018, 1, 28), - "ZonedDateTime" => ZonedDateTime(2018, 1, 28, tz"America/Chicago"), - "DataFrame" => DataFrame( - :a => collect(1:5), - :b => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], - :c => ["a", "b", "c", "d", "e"], - :d => [true, true, false, false, true], - ), - "Distribution" => Normal(50.2, 4.3), - ) - - @testset "JSOFile" begin - @testset "$k" for (k, v) in datas - jso = JSOFile(v) - io = IOBuffer() - bytes = serialize(io, v) - expected = take!(io) - - @test jso.objects["data"] == expected - end - end - - @testset "reading and writing" begin - @testset "$k" for (k, v) in datas - io = IOBuffer() - orig = JSOFile(v) - write(io, orig) - - seekstart(io) - - result = read(io, JSOFile) - @test result == orig - end - - - @testset "empty io" begin - @test_throws(LOGGER, InvalidFileError, read(IOBuffer(), JSOFile)) - end - - @testset "corrupted start" begin - @test_throws( - LOGGER, - InvalidFileError, - read(IOBuffer("blah\nversion=1.0"), JSOFile) - ) - end - - @testset "empty version" begin - @test_throws( - LOGGER, - InvalidFileError, - read(IOBuffer("version=\nimage="), JSOFile) - ) - end - - @testset "invalid version" begin - jso = JSOFile("Hello World!") - io = IOBuffer() - header = "version=0.0\nimage=\nsysteminfo=$(jso.systeminfo)\n---" - write(io, header) - - for (name, data) in jso.objects - nb = length(data) - write(io, "\n$name|$nb=") - write(io, data) - end - - seekstart(io) - - # read(io, JSOFile) - @test_throws( - LOGGER, - ArgumentError, - read(io, JSOFile) - ) - end - - @testset "missing image" begin - @test_throws( - LOGGER, - InvalidFileError, - read(IOBuffer("version=1.0\nsysteminfo="), JSOFile) - ) - end - - @testset "empty image" begin - jso = JSOFile("Hello World!") - io = IOBuffer() - write(io, jso) - - seekstart(io) - - setlevel!(LOGGER, "debug") - @test_log( - LOGGER, - "debug", - "No docker image specified", - read(io, JSOFile) - ) - end - - @testset "invalid nb" begin - jso = JSOFile("Hello World!") - io = IOBuffer() - header = "version=$(jso.version)\nimage=$(jso.image)\nsysteminfo=$(jso.systeminfo)\n---" - write(io, header) - nb = length(jso.objects["data"]) - 1 - write(io, "\ndata|$nb=") - write(io, jso.objects["data"]) - - seekstart(io) - - # This will cause the parsing our variable names to fail - result = @test_throws( - LOGGER, - InvalidFileError, - read(io, JSOFile) - ) - - io = IOBuffer() - write(io, header) - nb = length(jso.objects["data"]) + 1 - write(io, "\ndata|$nb=") - write(io, jso.objects["data"]) - - seekstart(io) - - result = read(io, JSOFile) - - @test haskey(result.objects, "data") - result["data"] - end - end - - @testset "deserialization" begin - # Test deserialization works - @testset "data - $k" for (k, v) in datas - jso = JSOFile(v) - @test jso["data"] == v - end - - @testset "unsupported julia version" begin - jso = JSOFile(v"1.0", img, sysinfo, Dict("data" => hw_5)) - - # Test failing to deserialize data because of incompatible julia versions - # will will return the raw bytes - result = @test_warn(LOGGER, r"MethodError*", jso["data"]) - @test result == hw_5 - end - - @testset "missing module" begin - # We need to load and use AxisArrays on another process to cause the - # deserialization error - pnum = first(addprocs(1)) - - try - # We need to do this separately because there appears to be a race - # condition on AxisArrays being loaded. - f = @spawnat pnum begin - @eval Main using AxisArrays - end - - fetch(f) - - f = @spawnat pnum begin - io = IOBuffer() - serialize( - io, - AxisArray( - rand(20, 10), - Axis{:time}(14010:10:14200), - Axis{:id}(1:10) - ) - ) - return io - end - - io = fetch(f) - bytes = take!(io) - jso = JSOFile(v"1.0", img, sysinfo, Dict("data" => bytes)) - - # Test failing to deserailize data because of missing modules will - # still return the raw bytes - result = @test_warn(LOGGER, r"UndefVarError*", jso["data"]) - @test result == bytes - finally - rmprocs(pnum) - end - end - end -end diff --git a/test/runtests.jl b/test/runtests.jl index b7ea360..24819b2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,5 +2,5 @@ using Checkpoints using Compat.Test @testset "Checkpoints" begin - include("JSO.jl") + include("JLSO.jl") end From 5e9812e353d7c283b7520b410c6ab0d7d9560c7c Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Mon, 15 Oct 2018 11:32:45 -0500 Subject: [PATCH 05/52] 1.0 updates. --- src/JLSO.jl | 21 +++++++++++++++------ test/JLSO.jl | 12 +++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/JLSO.jl b/src/JLSO.jl index 0374fc4..2529113 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -33,6 +33,8 @@ using AWSCore using AWSS3 using BSON using Compat +using Compat.InteractiveUtils +using Compat.Serialization using Memento using Mocking @@ -66,12 +68,13 @@ function JLSOFile( ) _versioncheck(format) - objects = map(data) do t - varname, vardata = t + objects = Dict{String, Vector{UInt8}}() + + for (key, val) in data io = IOBuffer() - serialize(io, vardata) - return varname => take!(io) - end |> Dict + serialize(io, val) + objects[key] = take!(io) + end return JLSOFile(format, image, julia, sysinfo, objects) end @@ -133,7 +136,13 @@ end ####################################### function _versioninfo() if isempty(_CACHE[:VERSIONINFO]) - global _CACHE[:VERSIONINFO] = sprint(versioninfo, true) + global _CACHE[:VERSIONINFO] = if VERSION < v"0.7.0" + sprint(versioninfo, true) + else + io = IOBuffer() + versioninfo(io; verbose=true) + String(take!(io)) + end end return _CACHE[:VERSIONINFO] diff --git a/test/JLSO.jl b/test/JLSO.jl index 2269eb7..ea8f358 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -1,6 +1,9 @@ using Compat using Compat.Test using Compat.Dates +using Compat.Distributed +using Compat.InteractiveUtils +using Compat.Serialization using Checkpoints using Checkpoints.JLSO: JLSOFile, LOGGER using Memento @@ -81,6 +84,8 @@ using TimeZones # We need to do this separately because there appears to be a race # condition on AxisArrays being loaded. f = @spawnat pnum begin + @eval Main using Compat + @eval Main using Compat.Serialization @eval Main using AxisArrays end @@ -105,7 +110,12 @@ using TimeZones # Test failing to deserailize data because of missing modules will # still return the raw bytes - result = @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) + result = if VERSION < v"0.7.0" + @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) + else + @test_warn(LOGGER, r"KeyError*", jlso["data"]) + end + @test result == bytes finally rmprocs(pnum) From b60ba7835672b84a3ada7da9721e66049738f048 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 16 Oct 2018 00:32:18 -0500 Subject: [PATCH 06/52] Support both base and bson serialized objects (speed vs backwards compat). --- src/JLSO.jl | 167 +++++++++++++++++++++++++++++++++++++---------- test/JLSO.jl | 118 ++++++++++++++++++++++++--------- test/runtests.jl | 12 ++++ 3 files changed, 231 insertions(+), 66 deletions(-) diff --git a/src/JLSO.jl b/src/JLSO.jl index 2529113..9a76a4a 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -11,10 +11,14 @@ Example) ``` Dict( "metadata" => Dict( - "format" => v"1.0", - "image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest" + "version" => v"1.0", "julia" => v"0.6.4", - "sysinfo" => "Julia Version 0.6.4 ...", + "format" => :bson, # Could also be :serialize + "image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest" + "pkgs" => Dict( + "AxisArrays" => v"0.2.1", + ... + ) ), "objects" => Dict( "var1" => [0x35, 0x10, 0x01, 0x04, 0x44], @@ -33,61 +37,109 @@ using AWSCore using AWSS3 using BSON using Compat -using Compat.InteractiveUtils +using Compat.Pkg using Compat.Serialization using Memento using Mocking using Compat.Serialization +using Compat: Nothing const LOGGER = getlogger(@__MODULE__) const VALID_VERSIONS = (v"1.0", v"2.0") # Cache of the versioninfo and image, so we don't compute these every time. -const _CACHE = Dict{Symbol, String}( - :VERSIONINFO => "", +const _CACHE = Dict( + :PKGS => Dict{String, VersionNumber}(), :IMAGE => "", ) __init__() = Memento.register(LOGGER) struct JLSOFile - format::VersionNumber - image::String + version::VersionNumber julia::VersionNumber - sysinfo::String + format::Symbol + image::String + pkgs::Dict{String, VersionNumber} objects::Dict{String, Vector{UInt8}} end +""" + JLSOFile(data; image="", julia=$VERSION, version=v"1.0, format=:serialize) + +Stores the information needed to write a .jlso file. + +# Arguments + +- `data` - The objects to be stored in the file. + +# Keywords + +- `image=""` - The docker image URI that was used to generate the file +- `julia=$VERSION` - The julia version used to write the file +- `version=v"1.0"` - The file schema version +- `format=:bson` - The format to use for serializing individual objects. While `:bson` is + recommended for longer term object storage, `:serialize` tends to be the faster choice + for adhoc serialization. +""" function JLSOFile( data::Dict{String, <:Any}; - image=_image(), + version=v"1.0", julia=VERSION, - sysinfo=_versioninfo(), - format=v"1.0" + format=:bson, + image=_image(), ) - _versioncheck(format) + _versioncheck(version) objects = Dict{String, Vector{UInt8}}() for (key, val) in data io = IOBuffer() - serialize(io, val) + + if format === :bson + bson(io, Dict(key => val)) + elseif format === :serialize + serialize(io, val) + else + error(LOGGER, ArgumentError("Unsupported format $format")) + end + objects[key] = take!(io) end - return JLSOFile(format, image, julia, sysinfo, objects) + return JLSOFile(version, julia, format, image, _pkgs(), objects) end -JLSOFile(data) = JLSOFile(Dict("data" => data)) -JLSOFile(data::Pair...) = JLSOFile(Dict(data...)) +JLSOFile(data; kwargs...) = JLSOFile(Dict("data" => data); kwargs...) +JLSOFile(data::Pair...; kwargs...) = JLSOFile(Dict(data...); kwargs...) + +function Base.show(io::IO, jlso::JLSOFile) + if get(io, :compat, false) + print(io, jlso) + else + variables = join(names(jlso), ", ") + kwargs = join( + [ + "version=v\"$(jlso.version)\"", + "julia=v\"$(jlso.julia)\"", + "format=:$(jlso.format)", + "image=\"$(jlso.image)\"", + ], + ", " + ) + + print(io, "JLSOFile([$variables]; $kwargs)") + end +end function Base.:(==)(a::JLSOFile, b::JLSOFile) return ( - a.format == b.format && + a.version == b.version && a.julia == b.julia && a.image == b.image && - a.sysinfo == b.sysinfo && + a.pkgs == b.pkgs && + a.format == b.format && a.objects == b.objects ) end @@ -97,10 +149,11 @@ function Base.write(io::IO, jlso::JLSOFile) io, Dict( "metadata" => Dict( + "version" => jlso.version, + "julia" => jlso.julia, "format" => jlso.format, "image" => jlso.image, - "julia" => jlso.julia, - "sysinfo" => jlso.sysinfo, + "pkgs" => jlso.pkgs, ), "objects" => jlso.objects, ) @@ -110,42 +163,86 @@ end function Base.read(io::IO, ::Type{JLSOFile}) d = BSON.load(io) return JLSOFile( + d["metadata"]["version"], + d["metadata"]["julia"], d["metadata"]["format"], d["metadata"]["image"], - d["metadata"]["julia"], - d["metadata"]["sysinfo"], + d["metadata"]["pkgs"], d["objects"], ) end +Base.names(jlso::JLSOFile) = collect(keys(jlso.objects)) + +# TODO: Include a more detail summary method for displaying version information. + +""" + getindex(jlso, name) + +Returns the deserialized object with the specified name. +""" function Base.getindex(jlso::JLSOFile, name::String) try - return deserialize(IOBuffer(jlso.objects[name])) + if jlso.format === :bson + BSON.load(IOBuffer(jlso.objects[name]))[name] + elseif jlso.format === :serialize + deserialize(IOBuffer(jlso.objects[name])) + else + error(LOGGER, ArgumentError("Unsupported format $(jlso.format)")) + end catch e warn(LOGGER, e) return jlso.objects[name] end end -# save(io::IO, data) = write(io, JLSOFile(data)) -# load(io::IO, data) = read(io, JLSOFile(data)) +""" + save(io, data) + save(path, data) + +Creates a JLSOFile with the specified data and kwargs and writes it back to the io. +""" +save(io::IO, data; kwargs...) = write(io, JLSOFile(data; kwargs...)) +save(io::IO, data::Pair...; kwargs...) = save(io, Dict(data...); kwargs...) +save(path::String, args...; kwargs...) = open(io -> save(io, args...; kwargs...), path, "w") + +""" + load(io, objects...) -> Dict{String, Any} + load(path, objects...) -> Dict{String, Any} + +Load the JLSOFile from the io and deserialize the specified objects. +If no object names are specified then all objects in the file are returned. +""" +load(path::String, args...) = open(io -> load(io, args...), path) +function load(io::IO, objects::String...) + jlso = read(io, JLSOFile) + objects = isempty(objects) ? names(jlso) : objects + result = Dict{String, Any}() + + for o in objects + result[o] = jlso[o] + end + + return result +end + + ####################################### # Functions for lazily evaluating the # # VERSIONINFO and IMAGE at runtime # ####################################### -function _versioninfo() - if isempty(_CACHE[:VERSIONINFO]) - global _CACHE[:VERSIONINFO] = if VERSION < v"0.7.0" - sprint(versioninfo, true) - else - io = IOBuffer() - versioninfo(io; verbose=true) - String(take!(io)) +function _pkgs() + if isempty(_CACHE[:PKGS]) + for (pkg, ver) in Pkg.installed() + # BSON can't handle Void types + if ver !== nothing + global _CACHE[:PKGS][pkg] = ver + end end end - return _CACHE[:VERSIONINFO] + return _CACHE[:PKGS] end function _image() diff --git a/test/JLSO.jl b/test/JLSO.jl index ea8f358..6c87b09 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -1,3 +1,4 @@ +using BSON using Compat using Compat.Test using Compat.Dates @@ -12,12 +13,13 @@ using Memento.Test # To test different types from common external packages using DataFrames using Distributions +using Nullables # Needed for loading BSON encoded ZonedDateTimes on 1.0 using TimeZones @testset "JLSO" begin # Serialize "Hello World!" on julia 0.5.2 (not supported) img = JLSO._image() - sysinfo = JLSO._versioninfo() + pkgs = JLSO._pkgs() hw_5 = UInt8[0x26, 0x15, 0x87, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21] datas = Dict( @@ -36,20 +38,30 @@ using TimeZones ) @testset "JLSOFile" begin - @testset "$k" for (k, v) in datas - jlso = JLSOFile(v) + @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas + jlso = JLSOFile(k => v; format=fmt) io = IOBuffer() - bytes = serialize(io, v) + bytes = fmt === :bson ? bson(io, Dict(k => v)) : serialize(io, v) expected = take!(io) - @test jlso.objects["data"] == expected + @test jlso.objects[k] == expected end end + @testset "show" begin + jlso = JLSOFile(datas["String"]) + expected = string( + "JLSOFile([data]; version=v\"1.0.0\", julia=v\"$VERSION\", ", + "format=:bson, image=\"\")" + ) + @test sprint(show, jlso; context=:compact => true) == expected + @test sprint(show, jlso) == sprint(print, jlso) + end + @testset "reading and writing" begin - @testset "$k" for (k, v) in datas + @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas io = IOBuffer() - orig = JLSOFile(v) + orig = JLSOFile(v; format=fmt) write(io, orig) seekstart(io) @@ -61,18 +73,20 @@ using TimeZones @testset "deserialization" begin # Test deserialization works - @testset "data - $k" for (k, v) in datas - jlso = JLSOFile(v) + @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas + jlso = JLSOFile(v; format=fmt) @test jlso["data"] == v end @testset "unsupported julia version" begin - jlso = JLSOFile(v"1.0", img, VERSION, sysinfo, Dict("data" => hw_5)) + jlso = JLSOFile(v"1.0", VERSION, :serialize, img, pkgs, Dict("data" => hw_5)) # Test failing to deserialize data because of incompatible julia versions # will will return the raw bytes result = @test_warn(LOGGER, r"MethodError*", jlso["data"]) @test result == hw_5 + + # TODO: Test that BSON works across julia versions using external files? end @testset "missing module" begin @@ -86,40 +100,82 @@ using TimeZones f = @spawnat pnum begin @eval Main using Compat @eval Main using Compat.Serialization + @eval Main using BSON @eval Main using AxisArrays end fetch(f) - f = @spawnat pnum begin - io = IOBuffer() - serialize( - io, - AxisArray( - rand(20, 10), - Axis{:time}(14010:10:14200), - Axis{:id}(1:10) + @testset "serialize" begin + f = @spawnat pnum begin + io = IOBuffer() + serialize( + io, + AxisArray( + rand(20, 10), + Axis{:time}(14010:10:14200), + Axis{:id}(1:10) + ) ) - ) - return io - end + return io + end - io = fetch(f) - bytes = take!(io) - jlso = JLSOFile(v"1.0", img, VERSION, sysinfo, Dict("data" => bytes)) + io = fetch(f) + bytes = take!(io) - # Test failing to deserailize data because of missing modules will - # still return the raw bytes - result = if VERSION < v"0.7.0" - @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) - else - @test_warn(LOGGER, r"KeyError*", jlso["data"]) + jlso = JLSOFile(v"1.0", VERSION, :serialize, img, pkgs, Dict("data" => bytes)) + + # Test failing to deserailize data because of missing modules will + # still return the raw bytes + result = if VERSION < v"0.7.0" + @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) + else + @test_warn(LOGGER, r"KeyError*", jlso["data"]) + end + + @test result == bytes end - @test result == bytes + @testset "bson" begin + f = @spawnat pnum begin + io = IOBuffer() + bson( + io, + Dict( + "data" => AxisArray( + rand(20, 10), + Axis{:time}(14010:10:14200), + Axis{:id}(1:10) + ) + ) + ) + return io + end + + io = fetch(f) + bytes = take!(io) + + jlso = JLSOFile(v"1.0", VERSION, :bson, img, pkgs, Dict("data" => bytes)) + + # Test failing to deserailize data because of missing modules will + # still return the raw bytes + result = @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) + + @test result == bytes + end finally rmprocs(pnum) end end end + + @testset "saving and loading" begin + mktempdir() do path + @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas + JLSO.save("$path/$fmt-$k.jlso", k => v; format=fmt) + result = JLSO.load("$path/$fmt-$k.jlso") + @test result[k] == v + end + end + end end diff --git a/test/runtests.jl b/test/runtests.jl index 24819b2..e12e817 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,18 @@ using Checkpoints using Compat.Test +# The method based on keyword arguments is not in Compat, so to avoid +# deprecation warnings on 0.7 we need this little definition. +if VERSION < v"0.7.0-DEV.4524" + function sprint(f::Function, args...; context=nothing) + if context !== nothing + Base.sprint((io, args...) -> f(IOContext(io, context), args...), args...) + else + Base.sprint(f, args...) + end + end +end + @testset "Checkpoints" begin include("JLSO.jl") end From a91b28a3608e898dbbe5f9c5bc4fc3c1de382f8e Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 16 Oct 2018 12:15:42 -0500 Subject: [PATCH 07/52] Make `:serialize` the default. --- src/JLSO.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/JLSO.jl b/src/JLSO.jl index 9a76a4a..8c4c8c0 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -79,7 +79,7 @@ Stores the information needed to write a .jlso file. - `image=""` - The docker image URI that was used to generate the file - `julia=$VERSION` - The julia version used to write the file - `version=v"1.0"` - The file schema version -- `format=:bson` - The format to use for serializing individual objects. While `:bson` is +- `format=:serialize` - The format to use for serializing individual objects. While `:bson` is recommended for longer term object storage, `:serialize` tends to be the faster choice for adhoc serialization. """ @@ -87,7 +87,7 @@ function JLSOFile( data::Dict{String, <:Any}; version=v"1.0", julia=VERSION, - format=:bson, + format=:serialize, image=_image(), ) _versioncheck(version) From b61124cbe35fcc10cdd0625f3e8be1cbe1499bf3 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 16 Oct 2018 12:57:10 -0500 Subject: [PATCH 08/52] Fix show test. --- test/JLSO.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/JLSO.jl b/test/JLSO.jl index 6c87b09..63bb47e 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -52,7 +52,7 @@ using TimeZones jlso = JLSOFile(datas["String"]) expected = string( "JLSOFile([data]; version=v\"1.0.0\", julia=v\"$VERSION\", ", - "format=:bson, image=\"\")" + "format=:serialize, image=\"\")" ) @test sprint(show, jlso; context=:compact => true) == expected @test sprint(show, jlso) == sprint(print, jlso) From 8c17053d487a5924d0ff5c7584e7d6e8af04edc9 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 16 Oct 2018 22:54:09 -0500 Subject: [PATCH 09/52] Initial checkpointing API for local and remote (S3) JLSO files. --- REQUIRE | 1 + src/Checkpoints.jl | 145 +++++++++++++++++++++++---------------------- src/JLSO.jl | 1 + test/JLSO.jl | 17 ++++++ test/runtests.jl | 63 ++++++++++++++++++++ test/testmod.jl | 23 +++++++ 6 files changed, 178 insertions(+), 72 deletions(-) create mode 100644 test/testmod.jl diff --git a/REQUIRE b/REQUIRE index 175a0e2..c2a4506 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,6 @@ julia 0.6 AWSCore 0.3 +AWSSDK 0.2.0 AWSS3 Compat 0.69.0 Memento 0.7.0 diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 307fdfb..d5d2c76 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -9,10 +9,11 @@ module Checkpoints using AWSCore using AWSS3 -# using FileIO using Memento using Mocking +using AWSCore: AWSConfig +using AWSS3: s3_put using Compat: @__MODULE__ export JLSO @@ -24,76 +25,76 @@ __init__() = Memento.register(LOGGER) include("JLSO.jl") -# function filesave(prefix; ext="jld2") -# function f(label, data) -# parts = split(label, '.') -# parts[end] = string(parts[end], '.', ext) -# path = joinpath(prefix, parts...) -# save(path, data) -# end - -# return f -# end - -# function s3save(config::AWSConfig=AWSCore.aws_config(), bucket, prefix) -# verinfo = sprint(versioninfo, true) -# image = "" - -# # If we're running on AWS batch then store the docker_image -# if haskey(ENV, "AWS_BATCH_JOB_ID") -# job_id = ENV["AWS_BATCH_JOB_ID"] -# response = @mock describe_jobs(Dict("jobs" => [job_id])) - -# if length(response["jobs"]) > 0 -# image = first(response["jobs"])["container"]["image"] -# else -# warn(LOGGER, "No jobs found with id: $job_id.") -# end -# end - -# function f(label, data) -# fileobj = Dict( -# "image" => image, -# "versioninfo" => sprint(verinfo, -# "data" => data, -# ) - -# parts = split(label, '.') -# parts[end] = string(parts[end], '.jso') -# key = join(vcat([prefix], parts), "/") -# s3_put(config, bucket, key, sprint(serialize, fileobj)) -# end - -# return f -# end - -# function s3load(config::AWSConfig=AWSCore.aws_config(), bucket, key) -# obj = s3_get(aws, bucket, key) - -# end - -# function register(labels::String...) -# for l in labels -# if haskey(CHECKPOINTS, l) -# warn(LOGGER, "$l has already registered") -# else -# CHECKPOINTS[l] = (k, v) -> nothing -# end -# end -# end - -# function config(backend::Callable, labels::String...) -# for l in labels -# if haskey(CHECKPOINTS, l) -# CHECKPOINTS[l] = backend -# else -# warn(LOGGER, "$l is not a registered checkpoint label") -# end -# end -# end - -# checkpoint(label::String, x) = CHECKPOINTS[label](label, x) - -# labels() = collect(keys(CHECKPOINTS)) +function saver(prefix::String; kwargs...) + function f(label, data) + parts = split(label, '.') + parts[end] = string(parts[end], ".jlso") + parent = joinpath(prefix, parts[1:end-1]...) + + # Make the parent path if doesn't already exist + mkpath(parent) + + # Save the file to disk + path = joinpath(parent, parts[end]) + JLSO.save(path, data; kwargs...) + end + + return f +end + +function saver(config::AWSConfig, bucket::String, prefix::String; kwargs...) + function f(label, data) + parts = split(label, ".") + parts[end] = string(parts[end], ".jlso") + key = join(vcat([prefix], parts), "/") + + # Serialize the data to an IOBuffer + io = IOBuffer() + JLSO.save(io, data; kwargs...) + + # Upload the serialized object (Vector{UInt8}) + @mock s3_put(config, bucket, key, take!(io)) + end + + return f +end + +# TODO: Migrate the `saver` into a Saver/Loader API to allow the same settings to be used +# for both saving and loading objects. + +function register(labels::Vector{String}) + for l in labels + if haskey(CHECKPOINTS, l) + warn(LOGGER, "$l has already registered") + else + CHECKPOINTS[l] = (k, v) -> nothing + end + end +end + +function register(prefix::Union{Module, String}, labels::Vector{String}) + register(map(l -> join([prefix, l], "."), labels)) +end + +function config(backend::Function, labels::Vector{String}) + for l in labels + if haskey(CHECKPOINTS, l) + CHECKPOINTS[l] = backend + else + warn(LOGGER, "$l is not a registered checkpoint label") + end + end +end + +function config(backend::Function, prefix::Union{Module, String}) + config(backend, filter(l -> startswith(l, prefix), labels())) +end + +checkpoint(label::String, x) = CHECKPOINTS[label](label, x) +function checkpoint(prefix::Union{Module, String}, label::String, x) + checkpoint(join([prefix, label], "."), x) +end + +labels() = collect(keys(CHECKPOINTS)) end # module diff --git a/src/JLSO.jl b/src/JLSO.jl index 8c4c8c0..077b0e4 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -35,6 +35,7 @@ module JLSO using AWSCore using AWSS3 +using AWSSDK.Batch: describe_jobs using BSON using Compat using Compat.Pkg diff --git a/test/JLSO.jl b/test/JLSO.jl index 63bb47e..b69bfbb 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -1,4 +1,5 @@ using BSON +using AWSSDK.Batch: describe_jobs using Compat using Compat.Test using Compat.Dates @@ -16,6 +17,10 @@ using Distributions using Nullables # Needed for loading BSON encoded ZonedDateTimes on 1.0 using TimeZones +const DESCRIBE_JOBS_RESP = Dict( + "jobs" => [Dict("container" => Dict("image" => "busybox"))] +) + @testset "JLSO" begin # Serialize "Hello World!" on julia 0.5.2 (not supported) img = JLSO._image() @@ -38,6 +43,18 @@ using TimeZones ) @testset "JLSOFile" begin + patch = @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + + withenv("AWS_BATCH_JOB_ID" => 1) do + apply(patch) do + jlso = JLSOFile("I'm a batch job.") + @test jlso.image == "busybox" + end + end + + # Reset the cached image for future tests + JLSO._CACHE[:IMAGE] = "" + @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas jlso = JLSOFile(k => v; format=fmt) io = IOBuffer() diff --git a/test/runtests.jl b/test/runtests.jl index e12e817..02c577d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,12 @@ +using Mocking +Mocking.enable(force=true) + using Checkpoints using Compat.Test +using AWSCore + +using AWSCore: AWSConfig +using AWSS3: s3_put # The method based on keyword arguments is not in Compat, so to avoid # deprecation warnings on 0.7 we need this little definition. @@ -13,6 +20,62 @@ if VERSION < v"0.7.0-DEV.4524" end end + @testset "Checkpoints" begin include("JLSO.jl") + + include("testmod.jl") + + x = reshape(collect(1:100), 10, 10) + y = reshape(collect(101:200), 10, 10) + + @testset "Local saver" begin + mktempdir() do path + Checkpoints.config(Checkpoints.saver(path), "TestModule.foo") + + TestModule.foo(x, y) + + mod_path = joinpath(path, "TestModule") + @test isdir(mod_path) + + foo_path = joinpath(path, "TestModule", "foo") + bar_path = joinpath(path, "TestModule", "bar") + @test isdir(foo_path) + @test !isdir(bar_path) + + x_path = joinpath(path, "TestModule", "foo", "x.jlso") + y_path = joinpath(path, "TestModule", "foo", "y.jlso") + @test isfile(x_path) + @test isfile(y_path) + + @test JLSO.load(x_path)["data"] == x + @test JLSO.load(y_path)["data"] == y + end + end + + @testset "S3 saver" begin + objects = Dict{String, Vector{UInt8}}() + + s3_put_patch = @patch function s3_put(config::AWSConfig, bucket, prefix, data) + objects[joinpath(bucket, prefix)] = data + end + + config = AWSCore.aws_config() + bucket = "mybucket" + prefix = joinpath("mybackrun", string(DateTime(2017, 1, 1, 8, 50, 32))) + Checkpoints.config( + Checkpoints.saver(config, bucket, prefix), + "TestModule.foo" + ) + + apply(s3_put_patch) do + TestModule.foo(x, y) + + io = IOBuffer(objects[joinpath(bucket, prefix, "TestModule/foo/x.jlso")]) + @test JLSO.load(io)["data"] == x + + io = IOBuffer(objects[joinpath(bucket, prefix, "TestModule/foo/y.jlso")]) + @test JLSO.load(io)["data"] == y + end + end end diff --git a/test/testmod.jl b/test/testmod.jl new file mode 100644 index 0000000..d64c501 --- /dev/null +++ b/test/testmod.jl @@ -0,0 +1,23 @@ +module TestModule + +using Checkpoints +using Checkpoints: register, checkpoint +using Compat: @__MODULE__ + +const MODULE = @__MODULE__() + +__init__() = register(MODULE, ["foo.x", "foo.y", "bar.a"]) + +function foo(x::Matrix, y::Matrix) + checkpoint(MODULE, "foo.x", x) + checkpoint(MODULE, "foo.y", y) + return x * y +end + + +function bar(a::Vector) + checkpoint(MODULE, "bar.a", a) + return a * a' +end + +end From 25cbb61119334b01c3ad3bf474f91ebf0c1680d0 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 17 Oct 2018 11:37:04 -0500 Subject: [PATCH 10/52] Tests passing on 1.0 --- test/testmod.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testmod.jl b/test/testmod.jl index d64c501..7a7d8a4 100644 --- a/test/testmod.jl +++ b/test/testmod.jl @@ -2,9 +2,9 @@ module TestModule using Checkpoints using Checkpoints: register, checkpoint -using Compat: @__MODULE__ -const MODULE = @__MODULE__() +# We aren't using `@__MODULE__` because that would return TestModule on 0.6 and Main.TestModule on 0.7 +const MODULE = "TestModule" __init__() = register(MODULE, ["foo.x", "foo.y", "bar.a"]) From b232869fb2a113e10ba32a443b45c64a838d0061 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 17 Oct 2018 16:09:44 -0500 Subject: [PATCH 11/52] Cleaned up API and added documentation. --- docs/make.jl | 4 +- docs/src/api.md | 19 ++++ docs/src/usage.md | 223 +++++++++++++++++++++++++++++++++++++++++++++ src/Checkpoints.jl | 53 +++++++++-- test/runtests.jl | 41 ++++----- test/testmod.jl | 23 ----- test/testpkg.jl | 23 +++++ 7 files changed, 327 insertions(+), 59 deletions(-) create mode 100644 docs/src/api.md create mode 100644 docs/src/usage.md delete mode 100644 test/testmod.jl create mode 100644 test/testpkg.jl diff --git a/docs/make.jl b/docs/make.jl index 28d595a..aa5cfe8 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -5,10 +5,12 @@ makedocs( format=:html, pages=[ "Home" => "index.md", + "Usage" => "usage.md", + "API" => "api.md", ], repo="https://gitlab.invenia.ca/invenia/Checkpoints.jl/blob/{commit}{path}#L{line}", sitename="Checkpoints.jl", - authors="rofinn", + authors="Rory Finnegan", assets=[ "assets/invenia.css", ], diff --git a/docs/src/api.md b/docs/src/api.md new file mode 100644 index 0000000..13b92a6 --- /dev/null +++ b/docs/src/api.md @@ -0,0 +1,19 @@ +# API + +## Checkpoints + +```@autodocs +Modules = [Checkpoints] +Public = true +Private = true +Pages = ["Checkpoints.jl"] +``` + +## Julia Serialized Objects (JLSO) + +```@autodocs +Modules = [Checkpoints.JLSO] +Public = true +Private = true +Pages = ["JLSO.jl"] +``` diff --git a/docs/src/usage.md b/docs/src/usage.md new file mode 100644 index 0000000..97b4c51 --- /dev/null +++ b/docs/src/usage.md @@ -0,0 +1,223 @@ +# Usage + + +## Checkpointing + +Let's begin by creating a module (or package) that contains data we may want to save +(or checkpoint). + +```julia +julia> module TestPkg + + using Checkpoints: register, checkpoint + + # We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 + const MODULE = "TestPkg" + + __init__() = register(MODULE, ["foo", "bar"]) + + function foo(x::Matrix, y::Matrix) + # Save multiple variables to 1 foo.jlso file by passing in pairs of variables + checkpoint(MODULE, "foo", "x" => x, "y" => y) + return x * y + end + + + function bar(a::Vector) + # Save a single value for bar.jlso. The object name in that file defaults to "data". + checkpoint(MODULE, "bar", a) + return a * a' + end + + end +TestPkg +``` + +Now we get a list of all available checkpoints outside our module. +```julia +julia> using Checkpoints + +julia> Checkpoints.available() +2-element Array{String,1}: + "TestPkg.bar" + "TestPkg.foo" +``` + +We can run our functions in `TestPkg` normally without saving any data. +```julia +julia> TestPkg.foo(rand(5, 5), rand(5, 5)) +5×5 Array{Float64,2}: + 0.968095 1.18687 1.55126 0.393847 0.854391 + 0.839788 1.1527 1.36785 0.361546 0.818136 + 1.44853 1.5996 2.17535 0.567696 1.31739 + 1.08267 1.74522 2.28862 0.673888 1.35935 + 0.755876 1.62275 2.24326 0.727734 1.13352 +``` + +To start checkpointing the arguments `x` and `y` to `TestPkg.foo` we first need to define a +backend saving function. The `Checkpoints.saver(prefix)` and `Checkpoints.saver(config, bucket, prefix)` +methods are provided as an easy way to generate backend functions that save to either the +local filesystem or an S3 bucket. +```julia +julia> f = Checkpoints.saver("checkpoints") +(::f) (generic function with 1 method) +``` + +Now we just need to assign the backend function `f` to our checkpoints. In our case, +all checkpoints with the prefix `"TestPkg.foo"`. +```julia +julia> Checkpoints.config(f, "TestPkg.foo") +``` + +To confirm that our checkpoint works will assign our expected `x` and `y values to local +variables. +```julia +julia> x = rand(5, 5) +5×5 Array{Float64,2}: + 0.605955 0.314332 0.666603 0.997074 0.106063 + 0.691509 0.438608 0.121533 0.931504 0.127145 + 0.704745 0.640941 0.237085 0.333055 0.648672 + 0.911475 0.410938 0.0143505 0.257862 0.0238969 + 0.956029 0.593267 0.0334345 0.374615 0.0301007 + +julia> y = rand(5, 5) +5×5 Array{Float64,2}: + 0.245635 0.211488 0.122208 0.917927 0.736712 + 0.556079 0.837774 0.0845954 0.812386 0.478323 + 0.661403 0.307322 0.015631 0.150063 0.765874 + 0.949725 0.332881 0.667242 0.468574 0.223302 + 0.723806 0.682948 0.511228 0.635479 0.879735 +``` + +Finally, rerun `TestPkg.foo` and inspect the generated file +```julia +julia> TestPkg.foo(x, y) +5×5 Array{Float64,2}: + 1.78824 1.0007 0.830574 1.44622 1.42326 + 1.47084 0.947963 0.81005 1.52659 1.13218 + 1.47216 1.31275 0.697899 1.77145 1.65238 + 0.724089 0.643606 0.33065 1.30867 0.95765 + 0.964419 0.854747 0.432891 1.55921 1.12383 + +julia> isfile("checkpoints/TestPkg/foo.jlso") +true + +julia> using Checkpoints.JLSO + +julia> d = JLSO.load("checkpoints/TestPkg/foo.jlso") +Dict{String,Any} with 2 entries: + "x" => [0.605955 0.314332 … 0.997074 0.106063; 0.691509 0.438608 … 0.931504 0.127145; … ; 0.911475 0.410938 … 0.257862 0.0238969; 0.956029… + "y" => [0.245635 0.211488 … 0.917927 0.736712; 0.556079 0.837774 … 0.812386 0.478323; … ; 0.949725 0.332881 … 0.468574 0.223302; 0.723806 … + +julia> d["x"] +5×5 Array{Float64,2}: + 0.605955 0.314332 0.666603 0.997074 0.106063 + 0.691509 0.438608 0.121533 0.931504 0.127145 + 0.704745 0.640941 0.237085 0.333055 0.648672 + 0.911475 0.410938 0.0143505 0.257862 0.0238969 + 0.956029 0.593267 0.0334345 0.374615 0.0301007 + +``` + +As we can see, the value of `x` was successfully saved to `checkpoints/MyPkg/foo.jlso`. + +## Load Failures + +What if I can't `load` my .jlso files? + +If you're julia environment doesn't match the one used to save .jlso file +(e.g., different julia version or missing packages) then you may get errors. + +```julia +julia> using Checkpoints, Checkpoints.JLSO +[ Info: Recompiling stale cache file /Users/rory/.playground/share/checkpoints/depot/compiled/v1.0/Checkpoints/E2USV.ji for Checkpoints [08085054-0ffc-5852-afcc-fc6ba29efde0] + +julia> d = JLSO.load("checkpoints/TestPkg/foo.jlso") +[warn | Checkpoints.JLSO]: EOFError: read end of file +[warn | Checkpoints.JLSO]: EOFError: read end of file +Dict{String,Any} with 2 entries: + "x" => UInt8[0x15, 0x00, 0x0e, 0x14, 0x02, 0xca, 0xca, 0x32, 0x20, 0x7b … 0x98, 0x3f, 0xc6, 0xc9, 0x58, 0xc8, 0xb7, 0xd2, 0x9e, 0x3f] + "y" => UInt8[0x15, 0x00, 0x0e, 0x14, 0x02, 0xca, 0xca, 0xfe, 0x60, 0xe0 … 0xcc, 0x3f, 0x9f, 0xb0, 0xc4, 0x03, 0xca, 0x26, 0xec, 0x3f] +``` + +In this case, we should try manually loading the `JLSO.JLSOFile` and inspect the metadata +saved with the file. + +```julia +julia> jlso = open("checkpoints/TestPkg/foo.jlso") do io + read(io, JLSO.JLSOFile) + end +JLSOFile([x, y]; version=v"1.0.0", julia=v"0.6.4", format=:serialize, image="") + +julia> VERSION +v"1.0.0" +``` + +As we can see, our .jlso file was saved in julia v0.6.4 and we're trying to load in on julia v1.0. +If you still have difficulty loading the file when the julia versions match then you may want to inspect +the package versions installed when saving the file. + +```julia +julia> jlso.pkgs +Dict{String,VersionNumber} with 60 entries: + "Coverage" => v"0.6.0" + "HTTP" => v"0.6.15" + "LegacyStrings" => v"0.4.0" + "Nullables" => v"0.0.8" + "AxisArrays" => v"0.2.1" + "Compat" => v"1.2.0" + "DataStructures" => v"0.8.4" + "CategoricalArrays" => v"0.3.14" + "Calculus" => v"0.4.1" + "DeepDiffs" => v"1.1.0" + "StatsFuns" => v"0.6.1" + "JLD2" => v"0.0.6" + "DataFrames" => v"0.11.7" + "SpecialFunctions" => v"0.6.0" + "TranscodingStreams" => v"0.5.4" + "Blosc" => v"0.5.1" + "Distributions" => v"0.15.0" + "SHA" => v"0.5.7" + "Missings" => v"0.2.10" + "SymDict" => v"0.2.1" + "CodecZlib" => v"0.4.4" + "HDF5" => v"0.9.5" + "AWSCore" => v"0.3.9" + "Retry" => v"0.2.0" + "MbedTLS" => v"0.5.13" + "FileIO" => v"0.9.1" + "Mocking" => v"0.5.7" + "TimeZones" => v"0.8.0" + "BSON" => v"0.1.4" + "PDMats" => v"0.8.0" + "BenchmarkTools" => v"0.3.2" + "SortingAlgorithms" => v"0.2.1" + "WeakRefStrings" => v"0.4.7" + "Memento" => v"0.10.0" + "Syslogs" => v"0.2.0" + "JSON" => v"0.17.2" + "StatsBase" => v"0.23.1" + "DocStringExtensions" => v"0.4.6" + "Checkpoints" => v"0.0.0-" + "QuadGK" => v"0.3.0" + "BinDeps" => v"0.8.10" + "RangeArrays" => v"0.3.1" + "Parameters" => v"0.9.2" + "Reexport" => v"0.1.0" + "CMakeWrapper" => v"0.1.0" + "URIParser" => v"0.3.1" + "XMLDict" => v"0.1.3" + "Documenter" => v"0.19.6" + "IntervalSets" => v"0.3.0" + "DataStreams" => v"0.3.6" + "JLD" => v"0.8.3" + "Rmath" => v"0.4.0" + "BinaryProvider" => v"0.3.3" + "IterTools" => v"0.2.1" + "IniFile" => v"0.4.0" + "AWSSDK" => v"0.3.1" + "NamedTuples" => v"4.0.2" + "AWSS3" => v"0.3.7" + "Homebrew" => v"0.6.4" + "LightXML" => v"0.7.0" +``` diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index d5d2c76..15cd3e2 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -25,6 +25,14 @@ __init__() = Memento.register(LOGGER) include("JLSO.jl") +""" + saver(prefix::String; kwargs...) -> Function + saver(config::AWSConfig, bucket::String, prefix::String; kwargs...) -> Function + +Generates a function that will dynamically saving variables to organized JLSO files locally +or on S3. Labels with '.' separators will be used to form subdirectories +(e.g., "Foo.bar.x" will be saved to "\$prefix/Foo/bar/x.jlso") +""" function saver(prefix::String; kwargs...) function f(label, data) parts = split(label, '.') @@ -62,6 +70,11 @@ end # TODO: Migrate the `saver` into a Saver/Loader API to allow the same settings to be used # for both saving and loading objects. +""" + register([prefix], labels) + +Registers a checkpoint that may be configured at a later time. +""" function register(labels::Vector{String}) for l in labels if haskey(CHECKPOINTS, l) @@ -76,25 +89,45 @@ function register(prefix::Union{Module, String}, labels::Vector{String}) register(map(l -> join([prefix, l], "."), labels)) end +""" + config(backend::Function, labels::Vector{String}) + config(backend::Function, prefix::String) + +Configures the specified checkpoints with the backend saving function. +The backend function is expected to take the checkpoint label and data to be saved. +The data is a `Dict` mapping variable names and values. +""" function config(backend::Function, labels::Vector{String}) for l in labels - if haskey(CHECKPOINTS, l) - CHECKPOINTS[l] = backend - else - warn(LOGGER, "$l is not a registered checkpoint label") - end + haskey(CHECKPOINTS, l) || warn(LOGGER, "$l is not a registered checkpoint label") + CHECKPOINTS[l] = backend end end function config(backend::Function, prefix::Union{Module, String}) - config(backend, filter(l -> startswith(l, prefix), labels())) + config(backend, filter(l -> startswith(l, prefix), available())) end -checkpoint(label::String, x) = CHECKPOINTS[label](label, x) -function checkpoint(prefix::Union{Module, String}, label::String, x) - checkpoint(join([prefix, label], "."), x) +""" + checkpoint([prefix], label, data) + checkpoint([prefix], label, data::Pair...) + checkpoint([prefix], label, data::Dict) + +Defines a data checkpoint with a specified `label` and values `data`. +By default checkpoints are no-ops and need to be configured with a backend funciton. +""" +checkpoint(label::String, data::Dict) = CHECKPOINTS[label](label, data) +checkpoint(label::String, data::Pair...) = checkpoint(label, Dict(data...)) +checkpoint(label::String, data) = checkpoint(label, Dict("data" => data)) +function checkpoint(prefix::Union{Module, String}, label::String, args...) + checkpoint(join([prefix, label], "."), args...) end -labels() = collect(keys(CHECKPOINTS)) +""" + available() -> Vector{String} + +Returns a vector of all available (registered) checkpoints. +""" +available() = collect(keys(CHECKPOINTS)) end # module diff --git a/test/runtests.jl b/test/runtests.jl index 02c577d..7063b4e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,32 +24,29 @@ end @testset "Checkpoints" begin include("JLSO.jl") - include("testmod.jl") + include("testpkg.jl") x = reshape(collect(1:100), 10, 10) y = reshape(collect(101:200), 10, 10) + a = collect(1:10) @testset "Local saver" begin mktempdir() do path - Checkpoints.config(Checkpoints.saver(path), "TestModule.foo") + Checkpoints.config(Checkpoints.saver(path), "TestPkg.foo") - TestModule.foo(x, y) + TestPkg.foo(x, y) - mod_path = joinpath(path, "TestModule") + mod_path = joinpath(path, "TestPkg") @test isdir(mod_path) - foo_path = joinpath(path, "TestModule", "foo") - bar_path = joinpath(path, "TestModule", "bar") - @test isdir(foo_path) - @test !isdir(bar_path) + foo_path = joinpath(path, "TestPkg", "foo.jlso") + bar_path = joinpath(path, "TestPkg", "bar.jlso") + @test isfile(foo_path) + @test !isfile(bar_path) - x_path = joinpath(path, "TestModule", "foo", "x.jlso") - y_path = joinpath(path, "TestModule", "foo", "y.jlso") - @test isfile(x_path) - @test isfile(y_path) - - @test JLSO.load(x_path)["data"] == x - @test JLSO.load(y_path)["data"] == y + data = JLSO.load(foo_path) + @test data["x"] == x + @test data["y"] == y end end @@ -63,19 +60,13 @@ end config = AWSCore.aws_config() bucket = "mybucket" prefix = joinpath("mybackrun", string(DateTime(2017, 1, 1, 8, 50, 32))) - Checkpoints.config( - Checkpoints.saver(config, bucket, prefix), - "TestModule.foo" - ) + Checkpoints.config(Checkpoints.saver(config, bucket, prefix), "TestPkg.bar") apply(s3_put_patch) do - TestModule.foo(x, y) - - io = IOBuffer(objects[joinpath(bucket, prefix, "TestModule/foo/x.jlso")]) - @test JLSO.load(io)["data"] == x + TestPkg.bar(a) - io = IOBuffer(objects[joinpath(bucket, prefix, "TestModule/foo/y.jlso")]) - @test JLSO.load(io)["data"] == y + io = IOBuffer(objects[joinpath(bucket, prefix, "TestPkg/bar.jlso")]) + @test JLSO.load(io)["data"] == a end end end diff --git a/test/testmod.jl b/test/testmod.jl deleted file mode 100644 index 7a7d8a4..0000000 --- a/test/testmod.jl +++ /dev/null @@ -1,23 +0,0 @@ -module TestModule - -using Checkpoints -using Checkpoints: register, checkpoint - -# We aren't using `@__MODULE__` because that would return TestModule on 0.6 and Main.TestModule on 0.7 -const MODULE = "TestModule" - -__init__() = register(MODULE, ["foo.x", "foo.y", "bar.a"]) - -function foo(x::Matrix, y::Matrix) - checkpoint(MODULE, "foo.x", x) - checkpoint(MODULE, "foo.y", y) - return x * y -end - - -function bar(a::Vector) - checkpoint(MODULE, "bar.a", a) - return a * a' -end - -end diff --git a/test/testpkg.jl b/test/testpkg.jl new file mode 100644 index 0000000..96e42b2 --- /dev/null +++ b/test/testpkg.jl @@ -0,0 +1,23 @@ +module TestPkg + +using Checkpoints: register, checkpoint + +# We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 +const MODULE = "TestPkg" + +__init__() = register(MODULE, ["foo", "bar"]) + +function foo(x::Matrix, y::Matrix) + # Save multiple variables to 1 foo.jlso file by passing in pairs of variables + checkpoint(MODULE, "foo", "x" => x, "y" => y) + return x * y +end + + +function bar(a::Vector) + # Save a single value for bar.jlso. The object name in that file defaults to "data". + checkpoint(MODULE, "bar", a) + return a * a' +end + +end From 8daeede39f2ee8204ab1ffd7cc24fcd79cc2e06b Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Thu, 25 Oct 2018 15:50:44 -0500 Subject: [PATCH 12/52] Added checkpoints session. - Added a Session type which can keep track and stage multiple checkpoints before writing (or committing) to the file. - Add tagging to checkpoints to allow the prefix path to be modified at execution time. --- REQUIRE | 1 + docs/src/usage.md | 144 ++++++++++++++++++++++++++++++++++++++------- src/Checkpoints.jl | 142 ++++++++++++++++++++------------------------ src/JLSO.jl | 36 ++++++++---- src/handler.jl | 80 +++++++++++++++++++++++++ src/session.jl | 62 +++++++++++++++++++ test/runtests.jl | 74 ++++++++++++++++++++--- test/testpkg.jl | 27 +++++++-- 8 files changed, 443 insertions(+), 123 deletions(-) create mode 100644 src/handler.jl create mode 100644 src/session.jl diff --git a/REQUIRE b/REQUIRE index c2a4506..5812ea4 100644 --- a/REQUIRE +++ b/REQUIRE @@ -3,6 +3,7 @@ AWSCore 0.3 AWSSDK 0.2.0 AWSS3 Compat 0.69.0 +DataStructures Memento 0.7.0 Mocking 0.5.2 BSON diff --git a/docs/src/usage.md b/docs/src/usage.md index 97b4c51..818236d 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -1,20 +1,17 @@ # Usage - -## Checkpointing - Let's begin by creating a module (or package) that contains data we may want to save (or checkpoint). ```julia julia> module TestPkg - using Checkpoints: register, checkpoint + using Checkpoints: register, checkpoint, Session # We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 const MODULE = "TestPkg" - __init__() = register(MODULE, ["foo", "bar"]) + __init__() = register(MODULE, ["foo", "bar", "baz]) function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables @@ -22,17 +19,29 @@ julia> module TestPkg return x * y end - function bar(a::Vector) - # Save a single value for bar.jlso. The object name in that file defaults to "data". - checkpoint(MODULE, "bar", a) + # Save a single value for bar.jlso. The object name in that file defaults to "date". + # Any kwargs passed to checkpoint will be appended to the handler path passed to config. + # In this case the path would be `/date=2017-01-01/TestPkg/bar.jlso` + checkpoint(MODULE, "bar", a; date="2017-01-01") return a * a' end - end + function baz(data::Dict) + # Check that saving multiple values to a Session works. + Session(MODULE, "baz") do s + for (k, v) in data + checkpoint(s, k => v) + end + end + end + end + TestPkg ``` +## Basic Checkpointing + Now we get a list of all available checkpoints outside our module. ```julia julia> using Checkpoints @@ -41,9 +50,28 @@ julia> Checkpoints.available() 2-element Array{String,1}: "TestPkg.bar" "TestPkg.foo" + "TestPkg.baz" ``` -We can run our functions in `TestPkg` normally without saving any data. +Let's start by looking at `TestPkg.foo` + +#### Package + +As a reference, here is the sample code for `TestPkg.foo` that we'll be calling. + +```julia +... +function foo(x::Matrix, y::Matrix) + # Save multiple variables to 1 foo.jlso file by passing in pairs of variables + checkpoint(MODULE, "foo", "x" => x, "y" => y) + return x * y +end +... +``` + +#### Application + +We can run our function `TestPkg.foo` normally without saving any data. ```julia julia> TestPkg.foo(rand(5, 5), rand(5, 5)) 5×5 Array{Float64,2}: @@ -54,22 +82,13 @@ julia> TestPkg.foo(rand(5, 5), rand(5, 5)) 0.755876 1.62275 2.24326 0.727734 1.13352 ``` -To start checkpointing the arguments `x` and `y` to `TestPkg.foo` we first need to define a -backend saving function. The `Checkpoints.saver(prefix)` and `Checkpoints.saver(config, bucket, prefix)` -methods are provided as an easy way to generate backend functions that save to either the -local filesystem or an S3 bucket. -```julia -julia> f = Checkpoints.saver("checkpoints") -(::f) (generic function with 1 method) -``` - -Now we just need to assign the backend function `f` to our checkpoints. In our case, +Now we just need to assign a backend handler for our checkpoints. In our case, all checkpoints with the prefix `"TestPkg.foo"`. ```julia -julia> Checkpoints.config(f, "TestPkg.foo") +julia> Checkpoints.config("TestPkg.foo", "./checkpoints") ``` -To confirm that our checkpoint works will assign our expected `x` and `y values to local +To confirm that our checkpoints by assigning our expected `x` and `y values to local variables. ```julia julia> x = rand(5, 5) @@ -121,6 +140,87 @@ julia> d["x"] As we can see, the value of `x` was successfully saved to `checkpoints/MyPkg/foo.jlso`. +## Tags + +Tags can be used to append the handler path at runtime. + +#### Package + +As a reference, here is the sample code for `TestPkg.bar` that we'll be calling. + +```julia +... +function bar(a::Vector) + # Save a single value for bar.jlso. The object name in that file defaults to "date". + # Any kwargs passed to checkpoint will be appended to the handler path passed to config. + # In this case the path would be `/date=2017-01-01/TestPkg/bar.jlso` + checkpoint(MODULE, "bar", a; date="2017-01-01") + return a * a' +end +... +``` + +#### Application + +```julia + +julia> a = rand(10) +10-element Array{Float64,1}: + 0.166881 + 0.817174 + 0.413097 + 0.955415 + 0.139473 + 0.49518 + 0.416731 + 0.431096 + 0.126912 + 0.600469 + +julia> Checkpoints.config("TestPkg.bar", "./checkpoints") + +julia> JLSO.load("./checkpoints/date=2017-01-01/TestPkg/bar.jlso") +Dict{String,Any} with 1 entry: + "data" => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469] +``` + +## Sessions + +If you'd like to iteratively checkpoint data (e.g., in a loop) then we recommend using a session. + +#### Package + +As a reference, here is the sample code for `TestPkg.baz` that we'll be calling. + +```julia +... +function baz(data::Dict) + # Check that saving multiple values to a Session works. + Session(MODULE, "baz") do s + for (k, v) in data + checkpoint(s, k => v) + end + end +end +... +``` + +#### Application + +```julia +julia> d = Dict("x" => rand(10), "y" => rand(10)) +Dict{String,Array{Float64,1}} with 2 entries: + "x" => [0.517666, 0.976474, 0.961658, 0.0933946, 0.877478, 0.428836, 0.0623459, 0.548001, 0.437111, 0.0783503] + "y" => [0.0623591, 0.0441436, 0.28578, 0.289995, 0.999642, 0.26299, 0.965148, 0.899285, 0.292166, 0.595886] + +julia> TestPkg.baz(d) + +julia> JLSO.load("./checkpoints/TestPkg/baz.jlso") +Dict{String,Any} with 2 entries: + "x" => [0.517666, 0.976474, 0.961658, 0.0933946, 0.877478, 0.428836, 0.0623459, 0.548001, 0.437111, 0.0783503] + "y" => [0.0623591, 0.0441436, 0.28578, 0.289995, 0.999642, 0.26299, 0.965148, 0.899285, 0.292166, 0.595886] +``` + ## Load Failures What if I can't `load` my .jlso files? diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 15cd3e2..82a6067 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -9,78 +9,104 @@ module Checkpoints using AWSCore using AWSS3 +using AWSTools.S3 using Memento using Mocking +using FilePathsBase -using AWSCore: AWSConfig -using AWSS3: s3_put -using Compat: @__MODULE__ +using Compat: @__MODULE__, Nothing, undef +using DataStructures: DefaultDict -export JLSO +export JLSO, checkpoint -const CHECKPOINTS = Dict{String, Function}() const LOGGER = getlogger(@__MODULE__) __init__() = Memento.register(LOGGER) include("JLSO.jl") +include("handler.jl") + +const CHECKPOINTS = Dict{String, Union{Nothing, Handler}}() + +include("session.jl") """ - saver(prefix::String; kwargs...) -> Function - saver(config::AWSConfig, bucket::String, prefix::String; kwargs...) -> Function + available() -> Vector{String} -Generates a function that will dynamically saving variables to organized JLSO files locally -or on S3. Labels with '.' separators will be used to form subdirectories -(e.g., "Foo.bar.x" will be saved to "\$prefix/Foo/bar/x.jlso") +Returns a vector of all available (registered) checkpoints. """ -function saver(prefix::String; kwargs...) - function f(label, data) - parts = split(label, '.') - parts[end] = string(parts[end], ".jlso") - parent = joinpath(prefix, parts[1:end-1]...) - - # Make the parent path if doesn't already exist - mkpath(parent) - - # Save the file to disk - path = joinpath(parent, parts[end]) - JLSO.save(path, data; kwargs...) - end +available() = collect(keys(CHECKPOINTS)) + +""" + checkpoint([prefix], name, data) + checkpoint([prefix], name, data::Pair...; tags...) + checkpoint([prefix], name, data::Dict; tags...) + +Defines a data checkpoint with a specified `label` and values `data`. +By default checkpoints are no-ops and need to be explicitly configured. + + checkpoint(session, data; tags...) + checkpoint(handler, name, data::Dict; tags...) - return f +Alternatively, you can also checkpoint with to a session which stages the data to be +commited later by `commit!(session)`. +Explicitly calling checkpoint on a handler is generally not advised, but is an option. +""" +function checkpoint(name::String, data::Dict; tags...) + checkpoint(CHECKPOINTS[name], name, data; tags...) end -function saver(config::AWSConfig, bucket::String, prefix::String; kwargs...) - function f(label, data) - parts = split(label, ".") - parts[end] = string(parts[end], ".jlso") - key = join(vcat([prefix], parts), "/") +checkpoint(name::String, data::Pair...; tags...) = checkpoint(name, Dict(data...); tags...) + +checkpoint(name::String, data; tags...) = checkpoint(name, Dict("data" => data); tags...) - # Serialize the data to an IOBuffer - io = IOBuffer() - JLSO.save(io, data; kwargs...) +function checkpoint(prefix::Union{Module, String}, name::String, args...; kwargs...) + checkpoint("$prefix.$name", args...; kwargs...) +end - # Upload the serialized object (Vector{UInt8}) - @mock s3_put(config, bucket, key, take!(io)) +""" + config(handler::Handler, labels::Vector{String}) + config(handler::Handler, prefix::String) + config(labels::Vector{String}, args...; kwargs...) + config(prefix::String, args...; kwargs...) + +Configures the specified checkpoints with a `Handler`. +If the first argument is not a `Handler` then all `args` and `kwargs` are passed to a +`Handler` constructor for you. +""" +function config(handler::Handler, names::Vector{String}) + for n in names + haskey(CHECKPOINTS, n) || warn(LOGGER, "$n is not a registered checkpoint") + CHECKPOINTS[n] = handler end +end - return f +function config(handler::Handler, prefix::Union{Module, String}) + config(handler, filter(l -> startswith(l, prefix), available())) +end + +function config(names::Vector{String}, args...; kwargs...) + config(Handler(args...; kwargs...), names) +end + +function config(prefix::Union{Module, String}, args...; kwargs...) + config(Handler(args...; kwargs...), prefix) end -# TODO: Migrate the `saver` into a Saver/Loader API to allow the same settings to be used -# for both saving and loading objects. """ register([prefix], labels) Registers a checkpoint that may be configured at a later time. """ +function register end + function register(labels::Vector{String}) for l in labels if haskey(CHECKPOINTS, l) warn(LOGGER, "$l has already registered") else - CHECKPOINTS[l] = (k, v) -> nothing + CHECKPOINTS[l] = nothing end end end @@ -89,45 +115,5 @@ function register(prefix::Union{Module, String}, labels::Vector{String}) register(map(l -> join([prefix, l], "."), labels)) end -""" - config(backend::Function, labels::Vector{String}) - config(backend::Function, prefix::String) - -Configures the specified checkpoints with the backend saving function. -The backend function is expected to take the checkpoint label and data to be saved. -The data is a `Dict` mapping variable names and values. -""" -function config(backend::Function, labels::Vector{String}) - for l in labels - haskey(CHECKPOINTS, l) || warn(LOGGER, "$l is not a registered checkpoint label") - CHECKPOINTS[l] = backend - end -end - -function config(backend::Function, prefix::Union{Module, String}) - config(backend, filter(l -> startswith(l, prefix), available())) -end - -""" - checkpoint([prefix], label, data) - checkpoint([prefix], label, data::Pair...) - checkpoint([prefix], label, data::Dict) - -Defines a data checkpoint with a specified `label` and values `data`. -By default checkpoints are no-ops and need to be configured with a backend funciton. -""" -checkpoint(label::String, data::Dict) = CHECKPOINTS[label](label, data) -checkpoint(label::String, data::Pair...) = checkpoint(label, Dict(data...)) -checkpoint(label::String, data) = checkpoint(label, Dict("data" => data)) -function checkpoint(prefix::Union{Module, String}, label::String, args...) - checkpoint(join([prefix, label], "."), args...) -end - -""" - available() -> Vector{String} - -Returns a vector of all available (registered) checkpoints. -""" -available() = collect(keys(CHECKPOINTS)) end # module diff --git a/src/JLSO.jl b/src/JLSO.jl index 077b0e4..386b75f 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -46,6 +46,8 @@ using Mocking using Compat.Serialization using Compat: Nothing +export JLSOFile + const LOGGER = getlogger(@__MODULE__) const VALID_VERSIONS = (v"1.0", v"2.0") @@ -94,22 +96,13 @@ function JLSOFile( _versioncheck(version) objects = Dict{String, Vector{UInt8}}() + jlso = JLSOFile(version, julia, format, image, _pkgs(), objects) for (key, val) in data - io = IOBuffer() - - if format === :bson - bson(io, Dict(key => val)) - elseif format === :serialize - serialize(io, val) - else - error(LOGGER, ArgumentError("Unsupported format $format")) - end - - objects[key] = take!(io) + jlso[key] = val end - return JLSOFile(version, julia, format, image, _pkgs(), objects) + return jlso end JLSOFile(data; kwargs...) = JLSOFile(Dict("data" => data); kwargs...) @@ -197,6 +190,25 @@ function Base.getindex(jlso::JLSOFile, name::String) end end +""" + setindex!(jlso, value, name) + +Adds the object to the file and serializes it. +""" +function Base.setindex!(jlso::JLSOFile, value, name::String) + io = IOBuffer() + + if jlso.format === :bson + bson(io, Dict(name => value)) + elseif jlso.format === :serialize + serialize(io, value) + else + error(LOGGER, ArgumentError("Unsupported format $(jlso.format)")) + end + + jlso.objects[name] = take!(io) +end + """ save(io, data) save(path, data) diff --git a/src/handler.jl b/src/handler.jl new file mode 100644 index 0000000..b6a0e2b --- /dev/null +++ b/src/handler.jl @@ -0,0 +1,80 @@ +struct Handler{P<:AbstractPath} + path::P + settings # Could be Vector or Pairs on 0.6 or 1.0 respectively +end + +""" + Handler(path::Union{String, AbstractPath}; kwargs...) + Handler(bucket::String, prefix::String; kwargs...) + +Handles iteratively saving JLSO file to the specified path location. +FilePath are used to abstract away differences between paths on S3 or locally. +""" +Handler(path::AbstractPath; kwargs...) = Handler(path, kwargs) +Handler(path::String; kwargs...) = Handler(Path(path), kwargs) +Handler(bucket::String, prefix::String; kwargs...) = Handler(S3Path(bucket, prefix), kwargs) + +""" + path(handler, name; tags...) + +Determines the path to save to based on the handlers path prefix, name and tags. +Tags are used to dynamically prefix the named file with the handler's path. +Names with a '.' separators will be used to form subdirectories +(e.g., "Foo.bar.x" will be saved to "\$prefix/Foo/bar/x.jlso"). +""" +function path(handler::Handler{P}, name::String; tags...) where P + # Build up a path prefix based on the tags passed in. + prefix = Vector{String}(undef, length(tags)) + for (i, t) in enumerate(tags) + prefix[i] = string(first(t), "=", last(t)) + end + + # Split up the name by '.' and add the jlso extension + parts = split(name, '.') + parts[end] = string(parts[end], ".jlso") + + return join(handler.path, prefix..., parts...) +end + +""" + stage!(handler::Handler, jlso::JLSOFIle, data::Dict) + +Update the JLSOFile with the new data. +""" +function stage!(handler::Handler, jlso::JLSO.JLSOFile, data::Dict) + for (k, v) in data + jlso[k] = v + end + + return jlso +end + +""" + commit!(handler, path, jlso) + +Write the JLSOFile to the path as bytes. +""" +function commit!(handler::Handler{P}, path::P, jlso::JLSO.JLSOFile) where P <: AbstractPath + io = IOBuffer() + write(io, jlso) + bytes = take!(io) + + # FilePathsBase should probably default to a no-op? + if P <: Union{PosixPath, WindowsPath} && hasparent(path) + mkdir(parent(path); recursive=true, exist_ok=true) + end + + write(path, bytes) +end + +function checkpoint(handler::Handler, name::String, data::Dict; tags...) + jlso = JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) + p = path(handler, name; tags...) + stage!(handler, jlso, data) + commit!(handler, p, jlso) +end + +#= +Define our no-op conditions just to be safe +=# +checkpoint(handler::Nothing, args...; kwargs...) = nothing diff --git a/src/session.jl b/src/session.jl new file mode 100644 index 0000000..7223f61 --- /dev/null +++ b/src/session.jl @@ -0,0 +1,62 @@ +struct Session{H<:Union{Nothing, Handler}} + name::String + handler::H + objects::DefaultDict +end + +function Session(name::String) + # Create our objects dictionary which defaults to returning + # an empty JLSOFile + handler = CHECKPOINTS[name] + + objects = DefaultDict{AbstractPath, JLSO.JLSOFile}() do + JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) + end + + Session{typeof(handler)}(name, handler, objects) +end + +Session(prefix::Union{Module, String}, name::String) = Session(join([prefix, name], ".")) + +function Session(f::Function, args...) + session = Session(args...) + f(session) + commit!(session) +end + +function Session(f::Function, names::Vector{String}) + sessions = Session.(names) + f(sessions...) + commit!.(sessions) +end + +function Session(f::Function, prefix::Union{Module, String}, names::Vector{String}) + Session(f, map(n -> "$prefix.$n", names)) +end + +""" + commit!(session) + +Write all staged JLSOFiles to the respective paths. +""" +function commit!(session::Session) + # No-ops skip when handler is nothing + session.handler === nothing && return nothing + + for (p, jlso) in session.objects + commit!(session.handler, p, jlso) + end +end + +function checkpoint(session::Session, data::Dict; tags...) + # No-ops skip when handler is nothing + session.handler === nothing && return nothing + + p = path(session.handler, session.name; tags...) + jlso = session.objects[p] + session.objects[p] = stage!(session.handler, jlso, data) +end + +checkpoint(s::Session, data::Pair...; tags...) = checkpoint(s, Dict(data...); tags...) + +checkpoint(s::Session, data; tags...) = checkpoint(s, Dict("data" => data); tags...) diff --git a/test/runtests.jl b/test/runtests.jl index 7063b4e..a734e96 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -30,11 +30,12 @@ end y = reshape(collect(101:200), 10, 10) a = collect(1:10) - @testset "Local saver" begin + @testset "Local handler" begin mktempdir() do path - Checkpoints.config(Checkpoints.saver(path), "TestPkg.foo") + Checkpoints.config("TestPkg.foo", path) TestPkg.foo(x, y) + TestPkg.bar(a) mod_path = joinpath(path, "TestPkg") @test isdir(mod_path) @@ -50,7 +51,7 @@ end end end - @testset "S3 saver" begin + @testset "S3 handler" begin objects = Dict{String, Vector{UInt8}}() s3_put_patch = @patch function s3_put(config::AWSConfig, bucket, prefix, data) @@ -59,14 +60,73 @@ end config = AWSCore.aws_config() bucket = "mybucket" - prefix = joinpath("mybackrun", string(DateTime(2017, 1, 1, 8, 50, 32))) - Checkpoints.config(Checkpoints.saver(config, bucket, prefix), "TestPkg.bar") + prefix = joinpath("mybackrun") + Checkpoints.config("TestPkg.bar", bucket, prefix) apply(s3_put_patch) do TestPkg.bar(a) - - io = IOBuffer(objects[joinpath(bucket, prefix, "TestPkg/bar.jlso")]) + expected_path = joinpath(bucket, prefix, "date=2017-01-01", "TestPkg/bar.jlso") + io = IOBuffer(objects[expected_path]) @test JLSO.load(io)["data"] == a end end + + @testset "Sessions" begin + @testset "No-op" begin + mktempdir() do path + d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + + TestPkg.baz(d) + + mod_path = joinpath(path, "TestPkg") + baz_path = joinpath(path, "TestPkg", "baz.jlso") + @test !isfile(baz_path) + end + end + @testset "Single" begin + mktempdir() do path + d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + Checkpoints.config("TestPkg.baz", path) + + TestPkg.baz(d) + + mod_path = joinpath(path, "TestPkg") + @test isdir(mod_path) + + baz_path = joinpath(path, "TestPkg", "baz.jlso") + @test isfile(baz_path) + + data = JLSO.load(baz_path) + for (k, v) in data + @test v == d[k] + end + end + end + @testset "Multi" begin + mktempdir() do path + a = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + b = rand(10) + Checkpoints.config("TestPkg.qux" , path) + + TestPkg.qux(a, b) + + mod_path = joinpath(path, "TestPkg") + @test isdir(mod_path) + + qux_a_path = joinpath(path, "TestPkg", "qux_a.jlso") + @test isfile(qux_a_path) + + qux_b_path = joinpath(path, "TestPkg", "qux_b.jlso") + @test isfile(qux_b_path) + + data = JLSO.load(qux_a_path) + for (k, v) in data + @test v == a[k] + end + + data = JLSO.load(qux_b_path) + @test data["data"] == b + end + end + end end diff --git a/test/testpkg.jl b/test/testpkg.jl index 96e42b2..44760a2 100644 --- a/test/testpkg.jl +++ b/test/testpkg.jl @@ -1,11 +1,11 @@ module TestPkg -using Checkpoints: register, checkpoint +using Checkpoints: register, checkpoint, Session # We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 const MODULE = "TestPkg" -__init__() = register(MODULE, ["foo", "bar"]) +__init__() = register(MODULE, ["foo", "bar", "baz", "qux_a", "qux_b"]) function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables @@ -13,11 +13,30 @@ function foo(x::Matrix, y::Matrix) return x * y end - function bar(a::Vector) # Save a single value for bar.jlso. The object name in that file defaults to "data". - checkpoint(MODULE, "bar", a) + checkpoint(MODULE, "bar", a; date="2017-01-01") return a * a' end +function baz(data::Dict) + # Check that saving multiple values to a Session works. + Session(MODULE, "baz") do s + for (k, v) in data + checkpoint(s, k => v) + end + end +end + +function qux(a::Dict, b::Vector) + # Check that saving multiple values to multiple Sessions also works. + Session(MODULE, ["qux_a", "qux_b"]) do sa, sb + for (k, v) in a + checkpoint(sa, k => v) + end + + checkpoint(sb, b) + end +end + end From f280cc29dcfbf406cab193205052edf0a2904bcd Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Thu, 25 Oct 2018 16:31:33 -0500 Subject: [PATCH 13/52] Update requirements. --- REQUIRE | 3 ++- src/Checkpoints.jl | 2 -- src/JLSO.jl | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/REQUIRE b/REQUIRE index 5812ea4..e8d1ad4 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,9 +1,10 @@ julia 0.6 AWSCore 0.3 AWSSDK 0.2.0 -AWSS3 +AWSTools 0.6.0 Compat 0.69.0 DataStructures +FilePathsBase 0.4.0 Memento 0.7.0 Mocking 0.5.2 BSON diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 82a6067..08ae735 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -7,8 +7,6 @@ the ability to configure how those checkpoints save data externally """ module Checkpoints -using AWSCore -using AWSS3 using AWSTools.S3 using Memento using Mocking diff --git a/src/JLSO.jl b/src/JLSO.jl index 386b75f..2bf3054 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -34,7 +34,6 @@ reconstruction. module JLSO using AWSCore -using AWSS3 using AWSSDK.Batch: describe_jobs using BSON using Compat From 6fe48fa4a9506ab522e95358b14fd0c6e4a042b8 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Thu, 25 Oct 2018 16:39:50 -0500 Subject: [PATCH 14/52] Compat.Random --- test/JLSO.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/JLSO.jl b/test/JLSO.jl index b69bfbb..27606e6 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -5,6 +5,7 @@ using Compat.Test using Compat.Dates using Compat.Distributed using Compat.InteractiveUtils +using Compat.Random using Compat.Serialization using Checkpoints using Checkpoints.JLSO: JLSOFile, LOGGER From d6646c8de7a412a6c1dd0f7bd7b381d96d202a46 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Thu, 25 Oct 2018 17:26:56 -0500 Subject: [PATCH 15/52] Fixed documentation typo. --- docs/src/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/usage.md b/docs/src/usage.md index 818236d..26806b9 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -88,7 +88,7 @@ all checkpoints with the prefix `"TestPkg.foo"`. julia> Checkpoints.config("TestPkg.foo", "./checkpoints") ``` -To confirm that our checkpoints by assigning our expected `x` and `y values to local +To confirm that our checkpoints work let's assign our expected `x` and `y values to local variables. ```julia julia> x = rand(5, 5) From b9a121d2e30998fc190d80a4cc1055ac997db15e Mon Sep 17 00:00:00 2001 From: morris25 Date: Fri, 26 Oct 2018 09:20:39 -0500 Subject: [PATCH 16/52] Updates gitlab-ci and makedoc kwargs --- .gitlab-ci.yml | 20 +++++++++----------- docs/make.jl | 4 +++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000d1ec..f617ec9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ stages: - julia-coverage $PKG_NAME .test_docker_0_6: &test_docker_0_6 - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 <<: *test_docker @@ -121,7 +121,7 @@ stages: "Coverage": stage: coverage - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 coverage: /Test Coverage (\d+\.\d+%)/ artifacts: name: combined_coverage @@ -131,13 +131,13 @@ stages: tags: - linux - docker-ci + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci script: - - genhtml --version - - mkdir $CI_PROJECT_DIR/combined_coverage - - cp -r $CI_PROJECT_DIR/src $CI_PROJECT_DIR/combined_coverage/ - - ls */*.info | xargs -I{} echo '--summary "{}"' | xargs lcov --directory src - - echo "Test Coverage $(genhtml -o $CI_PROJECT_DIR/combined_coverage --no-prefix */coverage.info 2>&1 | grep lines | awk '{print $2}')" - - find $CI_PROJECT_DIR/combined_coverage -type f -name "*.jl" -delete + - ./julia-ci publish-coverage combined_coverage + after_script: + - ./julia-ci clean "Documentation": @@ -157,8 +157,6 @@ stages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - julia --depwarn=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.add(\"Documenter\")" - - julia --depwarn=no -e "cd(Pkg.dir(\"$PKG_NAME\", \"docs\")); include(\"make.jl\")" - - julia --depwarn=no -e "const DOCS_DIR = joinpath(\"/mnt\", \"docs\", \"$CI_PROJECT_NAMESPACE\", \"$CI_PROJECT_NAME\", \"$CI_BUILD_REF_NAME\"); mkpath(DOCS_DIR); cp(Pkg.dir(\"$PKG_NAME\", \"docs\", \"build\"), DOCS_DIR; remove_destination=true, follow_symlinks=true)" + - ./julia-ci docs after_script: - ./julia-ci clean diff --git a/docs/make.jl b/docs/make.jl index aa5cfe8..cb4183f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -13,5 +13,7 @@ makedocs( authors="Rory Finnegan", assets=[ "assets/invenia.css", - ], + ], + strict = true, + checkdocs = :none, ) From c8834d032c251b74d7190d78bd1322172355f22f Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Wed, 21 Nov 2018 10:49:00 +0000 Subject: [PATCH 17/52] fix typo in example --- docs/src/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/usage.md b/docs/src/usage.md index 26806b9..559858f 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -11,7 +11,7 @@ julia> module TestPkg # We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 const MODULE = "TestPkg" - __init__() = register(MODULE, ["foo", "bar", "baz]) + __init__() = register(MODULE, ["foo", "bar", "baz"]) function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables From 30e17ff11340f9f8581bbea096ce3be72749176f Mon Sep 17 00:00:00 2001 From: morris25 Date: Mon, 3 Dec 2018 13:01:17 -0600 Subject: [PATCH 18/52] Use Pages for docs --- .gitlab-ci.yml | 12 +++++------- README.md | 3 +-- docs/src/index.md | 3 +-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f617ec9..59f29d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,14 +140,9 @@ stages: - ./julia-ci clean -"Documentation": +pages: stage: docs - tags: - - docs only: - - master - - develop - - docs - tags # special keyword for all tags variables: JULIA_VERSION: "0.6" @@ -157,6 +152,9 @@ stages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - ./julia-ci docs + - ./julia-ci publish-docs public after_script: - ./julia-ci clean + artifacts: + paths: + - public/ diff --git a/README.md b/README.md index c5cb387..4281fd3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ # Checkpoints -[![stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) -[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://invenia.pages.invenia.ca/Checkpoints.jl/) [![build status](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) [![coverage](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) diff --git a/docs/src/index.md b/docs/src/index.md index c5cb387..4281fd3 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,5 +1,4 @@ # Checkpoints -[![stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) -[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/Checkpoints.jl/master) +[![latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://invenia.pages.invenia.ca/Checkpoints.jl/) [![build status](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) [![coverage](https://gitlab.invenia.ca/invenia/Checkpoints.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/Checkpoints.jl/commits/master) From 46624ccf86c86d35957e5c727083aa6771e550b6 Mon Sep 17 00:00:00 2001 From: morris25 Date: Tue, 4 Dec 2018 14:07:35 -0600 Subject: [PATCH 19/52] Deploy docs on master --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59f29d7..98b90ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,7 +143,7 @@ stages: pages: stage: docs only: - - tags # special keyword for all tags + - master variables: JULIA_VERSION: "0.6" before_script: From 64c49cca226716d61bf42bc1e1fc948d2f572431 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Thu, 6 Dec 2018 15:45:04 -0600 Subject: [PATCH 20/52] log more things --- src/Checkpoints.jl | 1 + src/handler.jl | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 08ae735..c3e6698 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -75,6 +75,7 @@ If the first argument is not a `Handler` then all `args` and `kwargs` are passed function config(handler::Handler, names::Vector{String}) for n in names haskey(CHECKPOINTS, n) || warn(LOGGER, "$n is not a registered checkpoint") + debug(LOGGER, "Checkpoint $n set to use $(handler)") CHECKPOINTS[n] = handler end end diff --git a/src/handler.jl b/src/handler.jl index b6a0e2b..322792a 100644 --- a/src/handler.jl +++ b/src/handler.jl @@ -68,6 +68,7 @@ function commit!(handler::Handler{P}, path::P, jlso::JLSO.JLSOFile) where P <: A end function checkpoint(handler::Handler, name::String, data::Dict; tags...) + debug(LOGGER, "Checkpoint $name triggerred, with tags: $(join(tags, ", ")).") jlso = JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) p = path(handler, name; tags...) stage!(handler, jlso, data) @@ -77,4 +78,7 @@ end #= Define our no-op conditions just to be safe =# -checkpoint(handler::Nothing, args...; kwargs...) = nothing +function checkpoint(handler::Nothing, name::String, data::Dict; tags...) + debug(LOGGER, "Checkpoint $name triggerred, but no handler has been set.") + nothing +end From 77a593db220da1e63873f9652b86137cf3358d45 Mon Sep 17 00:00:00 2001 From: morris25 Date: Thu, 13 Dec 2018 14:54:55 -0600 Subject: [PATCH 21/52] Use gitlab-ci templates --- .gitlab-ci.yml | 122 ++++++++----------------------------------------- 1 file changed, 18 insertions(+), 104 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98b90ca..4d9e85a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,160 +1,74 @@ +include: + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/test_templates.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/coverage.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/documentation.yml + stages: - test - coverage - - docs - -.test_shell: &test_shell - artifacts: - name: "$CI_JOB_NAME coverage" - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage/" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install-cred-helper - - ./julia-ci install $JULIA_VERSION - script: - - source julia-ci export - - ./julia-ci test - - ./julia-ci coverage - after_script: - - ./julia-ci clean - - -.test_shell_0_6: &test_shell_0_6 - variables: - JULIA_VERSION: "0.6" - ONLINE: "" - <<: *test_shell - -.test_shell_1_0: &test_shell_1_0 - variables: - JULIA_VERSION: "1.0" - ONLINE: "" - <<: *test_shell - -.test_shell_nightly: &test_shell_nightly - variables: - JULIA_VERSION: "nightly" - ONLINE: "" - allow_failure: true - <<: *test_shell - + - documentation -.test_docker: &test_docker - artifacts: - name: coverage - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage/" - variables: - ONLINE: "" - script: - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - julia-coverage $PKG_NAME - -.test_docker_0_6: &test_docker_0_6 - image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - <<: *test_docker +variables: + JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + ONLINE: "" "0.6 (Mac)": tags: - mac - shell-ci - <<: *test_shell_0_6 + extends: .test_shell_0_6 "0.6 (Linux, 64-bit)": tags: - linux - 64-bit - docker-ci - <<: *test_docker_0_6 + extends: .test_docker "0.6 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_0_6 + extends: .test_shell_0_6 "1.0 (Mac)": tags: - mac - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "1.0 (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "1.0 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "Nightly (Mac)": tags: - mac - shell-ci - <<: *test_shell_nightly + extends: .test_shell_nightly "Nightly (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_nightly + extends: .test_shell_nightly "Nightly (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_nightly - -"Coverage": - stage: coverage - image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - coverage: /Test Coverage (\d+\.\d+%)/ - artifacts: - name: combined_coverage - expire_in: 1 week - paths: - - combined_coverage/ - tags: - - linux - - docker-ci - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - script: - - ./julia-ci publish-coverage combined_coverage - after_script: - - ./julia-ci clean - - -pages: - stage: docs - only: - - master - variables: - JULIA_VERSION: "0.6" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION - script: - - source julia-ci export - - ./julia-ci publish-docs public - after_script: - - ./julia-ci clean - artifacts: - paths: - - public/ + extends: .test_shell_nightly From 13c5df8a470be40978a64f0098253b70d90f82af Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Mon, 18 Feb 2019 16:57:53 -0800 Subject: [PATCH 22/52] Drop support for Julia 0.6 Summary of changes: * Raise minimum Julia version to 0.7 * Replace 0.6 on CI with 1.1 * Modernize documentation setup * Remove dependency on Compat * Remove unnecessary VERSION checks --- .gitignore | 6 ++ .gitlab-ci.yml | 46 ++++---- Project.toml | 38 +++++++ REQUIRE | 3 +- docs/Manifest.toml | 256 +++++++++++++++++++++++++++++++++++++++++++++ docs/Project.toml | 6 ++ docs/make.jl | 2 +- src/Checkpoints.jl | 1 - src/JLSO.jl | 8 +- test/JLSO.jl | 22 ++-- test/runtests.jl | 15 +-- 11 files changed, 339 insertions(+), 64 deletions(-) create mode 100644 Project.toml create mode 100644 docs/Manifest.toml create mode 100644 docs/Project.toml diff --git a/.gitignore b/.gitignore index 4637e7a..ff8cca9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ .DS_Store +# Coverage generated files +*.jl.cov +*.jl.*.cov +*.jl.mem # Documenter generated files /docs/build/ /docs/site/ +# Pkg generated files +/Manifest.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d9e85a..44ac62b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,57 +1,51 @@ -include: - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/test_templates.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/coverage.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/documentation.yml - stages: + - setup - test - - coverage - - documentation - -variables: - JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - ONLINE: "" + - teardown +include: + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml -"0.6 (Mac)": +"1.0 (Mac)": tags: - mac - shell-ci - extends: .test_shell_0_6 + extends: .test_shell_1_0 -"0.6 (Linux, 64-bit)": +"1.0 (Linux, 64-bit)": tags: - linux - 64-bit - docker-ci - extends: .test_docker + extends: .test_docker_1_0 -"0.6 (Linux, 32-bit)": +"1.0 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - extends: .test_shell_0_6 + extends: .test_shell_1_0 -"1.0 (Mac)": +"1.1 (Mac)": tags: - mac - shell-ci - extends: .test_shell_1_0 + extends: .test_shell_1_1 -"1.0 (Linux, 64-bit)": +"1.1 (Linux, 64-bit)": tags: - linux - 64-bit - - shell-ci - extends: .test_shell_1_0 + - docker-ci + extends: .test_docker_1_1 -"1.0 (Linux, 32-bit)": +"1.1 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - extends: .test_shell_1_0 + extends: .test_shell_1_1 "Nightly (Mac)": tags: @@ -63,8 +57,8 @@ variables: tags: - linux - 64-bit - - shell-ci - extends: .test_shell_nightly + - docker-ci + extends: .test_docker_nightly "Nightly (Linux, 32-bit)": tags: diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..3770169 --- /dev/null +++ b/Project.toml @@ -0,0 +1,38 @@ +name = "Checkpoints" +uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" + +[deps] +AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" +AWSSDK = "0d499d91-6ae5-5d63-9313-12987b87d5ad" +AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" +BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" +Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[compat] +AWSCore = ">=0.3.0" +AWSSDK = ">=0.2.0" +AWSTools = ">=0.6.0" +FilePathsBase = ">=0.4.0" +Memento = ">=0.7.0" +Mocking = ">=0.5.2" +julia = "0.7, 1.0" + +[extras] +AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" +AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Nullables = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" + +[targets] +test = ["AWSS3", "AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Nullables", "Random", "TimeZones"] diff --git a/REQUIRE b/REQUIRE index e8d1ad4..22b75fd 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,8 +1,7 @@ -julia 0.6 +julia 0.7 AWSCore 0.3 AWSSDK 0.2.0 AWSTools 0.6.0 -Compat 0.69.0 DataStructures FilePathsBase 0.4.0 Memento 0.7.0 diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 0000000..304ba22 --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,256 @@ +# This file is machine-generated - editing it directly is not advised + +[[AWSCore]] +deps = ["Base64", "DataStructures", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Retry", "Sockets", "SymDict", "Test", "XMLDict"] +git-tree-sha1 = "7750fad931afd52b84a8f014617daecc74235e92" +uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" +version = "0.5.5" + +[[AWSS3]] +deps = ["AWSCore", "Base64", "DataStructures", "Dates", "EzXML", "HTTP", "MbedTLS", "Retry", "SymDict", "Test", "XMLDict"] +git-tree-sha1 = "4b7387c999f8c589542d1a449669d68a86583eca" +uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" +version = "0.5.0" + +[[AWSSDK]] +deps = ["AWSCore"] +git-tree-sha1 = "89c95eab2bc1128fe247c31e9774d47eb7ef0d40" +uuid = "0d499d91-6ae5-5d63-9313-12987b87d5ad" +version = "0.4.0" + +[[AWSTools]] +deps = ["AWSCore", "AWSS3", "Base64", "Dates", "FilePathsBase", "MbedTLS", "Memento", "Mocking", "OrderedCollections", "Retry", "Test", "UUIDs", "XMLDict"] +git-tree-sha1 = "42b01ca3ae5743647e80af6be24df496f66f7768" +uuid = "83bcdc74-1232-581c-948a-f29122bf8259" +version = "1.0.0" + +[[BSON]] +deps = ["Test"] +git-tree-sha1 = "922b43e731601b73f0e53f1cc94ea719b94f673a" +uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" +version = "0.2.1" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BinaryProvider]] +deps = ["Libdl", "Pkg", "SHA", "Test"] +git-tree-sha1 = "055eb2690182ebc31087859c3dd8598371d3ef9e" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.5.3" + +[[Checkpoints]] +deps = ["AWSCore", "AWSSDK", "AWSTools", "BSON", "DataStructures", "FilePathsBase", "Memento", "Mocking", "Pkg", "Serialization"] +path = ".." +uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" +version = "0.1.0+" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "49269e311ffe11ac5b334681d212329002a9832a" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "1.5.1" + +[[DataStructures]] +deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"] +git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.15.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.6.0" + +[[Documenter]] +deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] +git-tree-sha1 = "f5dd8036d6e6673d8f0f9b60cd844ebb89aec591" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.21.4" + +[[EzXML]] +deps = ["BinaryProvider", "Libdl", "Printf", "Test"] +git-tree-sha1 = "5623d1486bfaadd815f5c4ca501adda02b5337f1" +uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" +version = "0.9.0" + +[[FilePathsBase]] +deps = ["Compat"] +git-tree-sha1 = "230d04188b262b0795144a5907b495fbc3ce8131" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.4.0" + +[[HTTP]] +deps = ["Base64", "Dates", "Distributed", "IniFile", "MbedTLS", "Random", "Sockets", "Test"] +git-tree-sha1 = "25db0e3f27bd5715814ca7e4ad22025fdcf5cc6e" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.8.0" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[IterTools]] +deps = ["SparseArrays", "Test"] +git-tree-sha1 = "79246285c43602384e6f1943b3554042a3712056" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.1.1" + +[[JSON]] +deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] +git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.20.0" + +[[LazyJSON]] +deps = ["DataStructures", "JSON", "Mmap", "Test"] +git-tree-sha1 = "f7bbbaebd2863861cb922efb63154b759cabadcc" +uuid = "fc18253b-5e1b-504c-a4a2-9ece4944c004" +version = "0.1.0" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS]] +deps = ["BinaryProvider", "Dates", "Libdl", "Random", "Sockets", "Test"] +git-tree-sha1 = "40b4a9149f0967714991328b8155c9ff5f91e755" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "0.6.7" + +[[Memento]] +deps = ["Compat", "JSON", "Nullables", "Syslogs", "TimeZones"] +git-tree-sha1 = "c4ade3575bcc2c180cfa14cf8b3b63eebca51629" +uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +version = "0.10.0" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Mocking]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "4bf69aaf823b119b034e091e16b18311aa191663" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.5.7" + +[[Nullables]] +deps = ["Compat"] +git-tree-sha1 = "ae1a63457e14554df2159b0b028f48536125092d" +uuid = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" +version = "0.0.8" + +[[OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.0.2" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Retry]] +deps = ["Test"] +git-tree-sha1 = "56bfdfca33e70883e96fd398548ebd4d405b41fe" +uuid = "20febd7b-183b-5ae2-ac4a-720e7ce64774" +version = "0.4.0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[SymDict]] +deps = ["Test"] +git-tree-sha1 = "0108ccdaea3ef69d9680eeafc8d5ad198b896ec8" +uuid = "2da68c74-98d7-5633-99d6-8493888d7b1e" +version = "0.3.0" + +[[Syslogs]] +deps = ["Compat", "Nullables"] +git-tree-sha1 = "d3e512a044cc8873c741d88758f8e1888c7c47d3" +uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4" +version = "0.2.0" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TimeZones]] +deps = ["Compat", "EzXML", "Mocking", "Nullables"] +git-tree-sha1 = "5437144a2bbb5b661783ad34b0d19d5696845b25" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "0.8.5" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[XMLDict]] +deps = ["DataStructures", "EzXML", "IterTools", "Test"] +git-tree-sha1 = "77a40486f4e5c81c57867d056933022bc4c5fe02" +uuid = "228000da-037f-5747-90a9-8195ccbf91a5" +version = "0.3.0" diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..1514f3e --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,6 @@ +[deps] +Checkpoints = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "~0.21" diff --git a/docs/make.jl b/docs/make.jl index cb4183f..936bf3d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,7 +2,7 @@ using Documenter, Checkpoints makedocs( modules=[Checkpoints], - format=:html, + format=Documenter.HTML(prettyurls=(get(ENV, "CI", nothing) == "true")), pages=[ "Home" => "index.md", "Usage" => "usage.md", diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index c3e6698..d83a1fb 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -12,7 +12,6 @@ using Memento using Mocking using FilePathsBase -using Compat: @__MODULE__, Nothing, undef using DataStructures: DefaultDict export JLSO, checkpoint diff --git a/src/JLSO.jl b/src/JLSO.jl index 2bf3054..15d95f4 100644 --- a/src/JLSO.jl +++ b/src/JLSO.jl @@ -36,15 +36,11 @@ module JLSO using AWSCore using AWSSDK.Batch: describe_jobs using BSON -using Compat -using Compat.Pkg -using Compat.Serialization +using Pkg +using Serialization using Memento using Mocking -using Compat.Serialization -using Compat: Nothing - export JLSOFile const LOGGER = getlogger(@__MODULE__) diff --git a/test/JLSO.jl b/test/JLSO.jl index 27606e6..a29cdf6 100644 --- a/test/JLSO.jl +++ b/test/JLSO.jl @@ -1,12 +1,11 @@ using BSON using AWSSDK.Batch: describe_jobs -using Compat -using Compat.Test -using Compat.Dates -using Compat.Distributed -using Compat.InteractiveUtils -using Compat.Random -using Compat.Serialization +using Test +using Dates +using Distributed +using InteractiveUtils +using Random +using Serialization using Checkpoints using Checkpoints.JLSO: JLSOFile, LOGGER using Memento @@ -116,8 +115,7 @@ const DESCRIBE_JOBS_RESP = Dict( # We need to do this separately because there appears to be a race # condition on AxisArrays being loaded. f = @spawnat pnum begin - @eval Main using Compat - @eval Main using Compat.Serialization + @eval Main using Serialization @eval Main using BSON @eval Main using AxisArrays end @@ -145,11 +143,7 @@ const DESCRIBE_JOBS_RESP = Dict( # Test failing to deserailize data because of missing modules will # still return the raw bytes - result = if VERSION < v"0.7.0" - @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) - else - @test_warn(LOGGER, r"KeyError*", jlso["data"]) - end + result = @test_warn(LOGGER, r"KeyError*", jlso["data"]) @test result == bytes end diff --git a/test/runtests.jl b/test/runtests.jl index a734e96..319ef3a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,25 +2,12 @@ using Mocking Mocking.enable(force=true) using Checkpoints -using Compat.Test +using Test using AWSCore using AWSCore: AWSConfig using AWSS3: s3_put -# The method based on keyword arguments is not in Compat, so to avoid -# deprecation warnings on 0.7 we need this little definition. -if VERSION < v"0.7.0-DEV.4524" - function sprint(f::Function, args...; context=nothing) - if context !== nothing - Base.sprint((io, args...) -> f(IOContext(io, context), args...), args...) - else - Base.sprint(f, args...) - end - end -end - - @testset "Checkpoints" begin include("JLSO.jl") From 48cf82dae7e8111e0732085b2d8810979eab6653 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Thu, 21 Feb 2019 12:54:21 -0800 Subject: [PATCH 23/52] Set minimum AWSCore version to 0.5.5 --- Project.toml | 2 +- REQUIRE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 3770169..5e3a3a5 100644 --- a/Project.toml +++ b/Project.toml @@ -14,7 +14,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [compat] -AWSCore = ">=0.3.0" +AWSCore = ">=0.5.5" AWSSDK = ">=0.2.0" AWSTools = ">=0.6.0" FilePathsBase = ">=0.4.0" diff --git a/REQUIRE b/REQUIRE index 22b75fd..8d91fb2 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,5 @@ julia 0.7 -AWSCore 0.3 +AWSCore 0.5.5 AWSSDK 0.2.0 AWSTools 0.6.0 DataStructures From 041acb2cbced89eb256aaf0325fd93542a875b82 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Thu, 21 Feb 2019 15:15:41 -0800 Subject: [PATCH 24/52] Add Test to Project.toml [extras] --- Project.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5e3a3a5..ccc9976 100644 --- a/Project.toml +++ b/Project.toml @@ -32,7 +32,8 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Nullables = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" [targets] -test = ["AWSS3", "AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Nullables", "Random", "TimeZones"] +test = ["AWSS3", "AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Nullables", "Random", "Test", "TimeZones"] From d363649020df72682f45df7a0eea6bd9892547a2 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Thu, 21 Feb 2019 15:55:06 -0800 Subject: [PATCH 25/52] Set minimum Julia version to 1.0 We require AWSCore 0.5.5 but that only supports 1.0, which makes METADATA consistency checks fail for 0.7. --- Project.toml | 2 +- REQUIRE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 5e3a3a5..65f1e86 100644 --- a/Project.toml +++ b/Project.toml @@ -20,7 +20,7 @@ AWSTools = ">=0.6.0" FilePathsBase = ">=0.4.0" Memento = ">=0.7.0" Mocking = ">=0.5.2" -julia = "0.7, 1.0" +julia = "^1.0" [extras] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" diff --git a/REQUIRE b/REQUIRE index 8d91fb2..907995e 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,4 @@ -julia 0.7 +julia 1.0 AWSCore 0.5.5 AWSSDK 0.2.0 AWSTools 0.6.0 From e58373b61806f1d70a885f4366199973bcd1c609 Mon Sep 17 00:00:00 2001 From: Fernando Chorney Date: Mon, 25 Feb 2019 10:21:33 -0600 Subject: [PATCH 26/52] Replace JLSO code with JLSO Package --- Project.toml | 20 +--- REQUIRE | 5 +- src/Checkpoints.jl | 5 +- src/JLSO.jl | 281 --------------------------------------------- test/JLSO.jl | 193 ------------------------------- test/REQUIRE | 4 - test/runtests.jl | 4 +- 7 files changed, 9 insertions(+), 503 deletions(-) delete mode 100644 src/JLSO.jl delete mode 100644 test/JLSO.jl delete mode 100644 test/REQUIRE diff --git a/Project.toml b/Project.toml index cc380b8..92df879 100644 --- a/Project.toml +++ b/Project.toml @@ -2,20 +2,14 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" [deps] -AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" -AWSSDK = "0d499d91-6ae5-5d63-9313-12987b87d5ad" AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" -BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" +JLSO = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" -Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [compat] AWSCore = ">=0.5.5" -AWSSDK = ">=0.2.0" AWSTools = ">=0.6.0" FilePathsBase = ">=0.4.0" Memento = ">=0.7.0" @@ -23,17 +17,11 @@ Mocking = ">=0.5.2" julia = "^1.0" [extras] +AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" -AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" -Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -Nullables = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" +Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" [targets] -test = ["AWSS3", "AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Nullables", "Random", "Test", "TimeZones"] +test = ["AWSCore", "AWSS3", "Mocking", "Random", "Test"] diff --git a/REQUIRE b/REQUIRE index 907995e..77796f8 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,9 +1,6 @@ julia 1.0 -AWSCore 0.5.5 -AWSSDK 0.2.0 AWSTools 0.6.0 DataStructures FilePathsBase 0.4.0 +JLSO Memento 0.7.0 -Mocking 0.5.2 -BSON diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index d83a1fb..d90ca8a 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -9,18 +9,17 @@ module Checkpoints using AWSTools.S3 using Memento -using Mocking using FilePathsBase +using JLSO using DataStructures: DefaultDict -export JLSO, checkpoint +export checkpoint const LOGGER = getlogger(@__MODULE__) __init__() = Memento.register(LOGGER) -include("JLSO.jl") include("handler.jl") const CHECKPOINTS = Dict{String, Union{Nothing, Handler}}() diff --git a/src/JLSO.jl b/src/JLSO.jl deleted file mode 100644 index 15d95f4..0000000 --- a/src/JLSO.jl +++ /dev/null @@ -1,281 +0,0 @@ -""" -A julia serialized object (JLSO) file format for storing checkpoint data. - -# Structure - -The .jlso files are BSON files containing the dictionaries with a specific schema. -NOTE: The raw dictionary should be loadable by any BSON library even if serialized objects -themselves aren't reconstructable. - -Example) -``` -Dict( - "metadata" => Dict( - "version" => v"1.0", - "julia" => v"0.6.4", - "format" => :bson, # Could also be :serialize - "image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest" - "pkgs" => Dict( - "AxisArrays" => v"0.2.1", - ... - ) - ), - "objects" => Dict( - "var1" => [0x35, 0x10, 0x01, 0x04, 0x44], - "var2" => [...], - ), -) -``` -WARNING: The serialized objects are stored using julia's builtin serialization format which -is not intended for long term storage. As a result, we're storing the serialized object data -in a json file which should also be able to load the docker image and versioninfo to allow -reconstruction. -""" -module JLSO - -using AWSCore -using AWSSDK.Batch: describe_jobs -using BSON -using Pkg -using Serialization -using Memento -using Mocking - -export JLSOFile - -const LOGGER = getlogger(@__MODULE__) -const VALID_VERSIONS = (v"1.0", v"2.0") - -# Cache of the versioninfo and image, so we don't compute these every time. -const _CACHE = Dict( - :PKGS => Dict{String, VersionNumber}(), - :IMAGE => "", -) - -__init__() = Memento.register(LOGGER) - -struct JLSOFile - version::VersionNumber - julia::VersionNumber - format::Symbol - image::String - pkgs::Dict{String, VersionNumber} - objects::Dict{String, Vector{UInt8}} -end - -""" - JLSOFile(data; image="", julia=$VERSION, version=v"1.0, format=:serialize) - -Stores the information needed to write a .jlso file. - -# Arguments - -- `data` - The objects to be stored in the file. - -# Keywords - -- `image=""` - The docker image URI that was used to generate the file -- `julia=$VERSION` - The julia version used to write the file -- `version=v"1.0"` - The file schema version -- `format=:serialize` - The format to use for serializing individual objects. While `:bson` is - recommended for longer term object storage, `:serialize` tends to be the faster choice - for adhoc serialization. -""" -function JLSOFile( - data::Dict{String, <:Any}; - version=v"1.0", - julia=VERSION, - format=:serialize, - image=_image(), -) - _versioncheck(version) - - objects = Dict{String, Vector{UInt8}}() - jlso = JLSOFile(version, julia, format, image, _pkgs(), objects) - - for (key, val) in data - jlso[key] = val - end - - return jlso -end - -JLSOFile(data; kwargs...) = JLSOFile(Dict("data" => data); kwargs...) -JLSOFile(data::Pair...; kwargs...) = JLSOFile(Dict(data...); kwargs...) - -function Base.show(io::IO, jlso::JLSOFile) - if get(io, :compat, false) - print(io, jlso) - else - variables = join(names(jlso), ", ") - kwargs = join( - [ - "version=v\"$(jlso.version)\"", - "julia=v\"$(jlso.julia)\"", - "format=:$(jlso.format)", - "image=\"$(jlso.image)\"", - ], - ", " - ) - - print(io, "JLSOFile([$variables]; $kwargs)") - end -end - -function Base.:(==)(a::JLSOFile, b::JLSOFile) - return ( - a.version == b.version && - a.julia == b.julia && - a.image == b.image && - a.pkgs == b.pkgs && - a.format == b.format && - a.objects == b.objects - ) -end - -function Base.write(io::IO, jlso::JLSOFile) - bson( - io, - Dict( - "metadata" => Dict( - "version" => jlso.version, - "julia" => jlso.julia, - "format" => jlso.format, - "image" => jlso.image, - "pkgs" => jlso.pkgs, - ), - "objects" => jlso.objects, - ) - ) -end - -function Base.read(io::IO, ::Type{JLSOFile}) - d = BSON.load(io) - return JLSOFile( - d["metadata"]["version"], - d["metadata"]["julia"], - d["metadata"]["format"], - d["metadata"]["image"], - d["metadata"]["pkgs"], - d["objects"], - ) -end - -Base.names(jlso::JLSOFile) = collect(keys(jlso.objects)) - -# TODO: Include a more detail summary method for displaying version information. - -""" - getindex(jlso, name) - -Returns the deserialized object with the specified name. -""" -function Base.getindex(jlso::JLSOFile, name::String) - try - if jlso.format === :bson - BSON.load(IOBuffer(jlso.objects[name]))[name] - elseif jlso.format === :serialize - deserialize(IOBuffer(jlso.objects[name])) - else - error(LOGGER, ArgumentError("Unsupported format $(jlso.format)")) - end - catch e - warn(LOGGER, e) - return jlso.objects[name] - end -end - -""" - setindex!(jlso, value, name) - -Adds the object to the file and serializes it. -""" -function Base.setindex!(jlso::JLSOFile, value, name::String) - io = IOBuffer() - - if jlso.format === :bson - bson(io, Dict(name => value)) - elseif jlso.format === :serialize - serialize(io, value) - else - error(LOGGER, ArgumentError("Unsupported format $(jlso.format)")) - end - - jlso.objects[name] = take!(io) -end - -""" - save(io, data) - save(path, data) - -Creates a JLSOFile with the specified data and kwargs and writes it back to the io. -""" -save(io::IO, data; kwargs...) = write(io, JLSOFile(data; kwargs...)) -save(io::IO, data::Pair...; kwargs...) = save(io, Dict(data...); kwargs...) -save(path::String, args...; kwargs...) = open(io -> save(io, args...; kwargs...), path, "w") - -""" - load(io, objects...) -> Dict{String, Any} - load(path, objects...) -> Dict{String, Any} - -Load the JLSOFile from the io and deserialize the specified objects. -If no object names are specified then all objects in the file are returned. -""" -load(path::String, args...) = open(io -> load(io, args...), path) -function load(io::IO, objects::String...) - jlso = read(io, JLSOFile) - objects = isempty(objects) ? names(jlso) : objects - result = Dict{String, Any}() - - for o in objects - result[o] = jlso[o] - end - - return result -end - - - -####################################### -# Functions for lazily evaluating the # -# VERSIONINFO and IMAGE at runtime # -####################################### -function _pkgs() - if isempty(_CACHE[:PKGS]) - for (pkg, ver) in Pkg.installed() - # BSON can't handle Void types - if ver !== nothing - global _CACHE[:PKGS][pkg] = ver - end - end - end - - return _CACHE[:PKGS] -end - -function _image() - if isempty(_CACHE[:IMAGE]) && haskey(ENV, "AWS_BATCH_JOB_ID") - job_id = ENV["AWS_BATCH_JOB_ID"] - response = @mock describe_jobs(Dict("jobs" => [job_id])) - - if length(response["jobs"]) > 0 - global _CACHE[:IMAGE] = first(response["jobs"])["container"]["image"] - else - warn(LOGGER, "No jobs found with id: $job_id.") - end - end - - return _CACHE[:IMAGE] -end - -function _versioncheck(version::VersionNumber) - supported = first(VALID_VERSIONS) <= version < last(VALID_VERSIONS) - supported || error(LOGGER, ArgumentError( - string( - "Unsupported version ($version). ", - "Expected a value between ($VALID_VERSIONS)." - ) - )) -end - -end diff --git a/test/JLSO.jl b/test/JLSO.jl deleted file mode 100644 index a29cdf6..0000000 --- a/test/JLSO.jl +++ /dev/null @@ -1,193 +0,0 @@ -using BSON -using AWSSDK.Batch: describe_jobs -using Test -using Dates -using Distributed -using InteractiveUtils -using Random -using Serialization -using Checkpoints -using Checkpoints.JLSO: JLSOFile, LOGGER -using Memento -using Memento.Test - -# To test different types from common external packages -using DataFrames -using Distributions -using Nullables # Needed for loading BSON encoded ZonedDateTimes on 1.0 -using TimeZones - -const DESCRIBE_JOBS_RESP = Dict( - "jobs" => [Dict("container" => Dict("image" => "busybox"))] -) - -@testset "JLSO" begin - # Serialize "Hello World!" on julia 0.5.2 (not supported) - img = JLSO._image() - pkgs = JLSO._pkgs() - hw_5 = UInt8[0x26, 0x15, 0x87, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21] - - datas = Dict( - "String" => "Hello World!", - "Vector" => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], - "Matrix" => [0.400348 0.892196 0.848164; 0.0183529 0.755449 0.397538; 0.870458 0.0441878 0.170899], - "DateTime" => DateTime(2018, 1, 28), - "ZonedDateTime" => ZonedDateTime(2018, 1, 28, tz"America/Chicago"), - "DataFrame" => DataFrame( - :a => collect(1:5), - :b => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], - :c => ["a", "b", "c", "d", "e"], - :d => [true, true, false, false, true], - ), - "Distribution" => Normal(50.2, 4.3), - ) - - @testset "JLSOFile" begin - patch = @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - - withenv("AWS_BATCH_JOB_ID" => 1) do - apply(patch) do - jlso = JLSOFile("I'm a batch job.") - @test jlso.image == "busybox" - end - end - - # Reset the cached image for future tests - JLSO._CACHE[:IMAGE] = "" - - @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas - jlso = JLSOFile(k => v; format=fmt) - io = IOBuffer() - bytes = fmt === :bson ? bson(io, Dict(k => v)) : serialize(io, v) - expected = take!(io) - - @test jlso.objects[k] == expected - end - end - - @testset "show" begin - jlso = JLSOFile(datas["String"]) - expected = string( - "JLSOFile([data]; version=v\"1.0.0\", julia=v\"$VERSION\", ", - "format=:serialize, image=\"\")" - ) - @test sprint(show, jlso; context=:compact => true) == expected - @test sprint(show, jlso) == sprint(print, jlso) - end - - @testset "reading and writing" begin - @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas - io = IOBuffer() - orig = JLSOFile(v; format=fmt) - write(io, orig) - - seekstart(io) - - result = read(io, JLSOFile) - @test result == orig - end - end - - @testset "deserialization" begin - # Test deserialization works - @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas - jlso = JLSOFile(v; format=fmt) - @test jlso["data"] == v - end - - @testset "unsupported julia version" begin - jlso = JLSOFile(v"1.0", VERSION, :serialize, img, pkgs, Dict("data" => hw_5)) - - # Test failing to deserialize data because of incompatible julia versions - # will will return the raw bytes - result = @test_warn(LOGGER, r"MethodError*", jlso["data"]) - @test result == hw_5 - - # TODO: Test that BSON works across julia versions using external files? - end - - @testset "missing module" begin - # We need to load and use AxisArrays on another process to cause the - # deserialization error - pnum = first(addprocs(1)) - - try - # We need to do this separately because there appears to be a race - # condition on AxisArrays being loaded. - f = @spawnat pnum begin - @eval Main using Serialization - @eval Main using BSON - @eval Main using AxisArrays - end - - fetch(f) - - @testset "serialize" begin - f = @spawnat pnum begin - io = IOBuffer() - serialize( - io, - AxisArray( - rand(20, 10), - Axis{:time}(14010:10:14200), - Axis{:id}(1:10) - ) - ) - return io - end - - io = fetch(f) - bytes = take!(io) - - jlso = JLSOFile(v"1.0", VERSION, :serialize, img, pkgs, Dict("data" => bytes)) - - # Test failing to deserailize data because of missing modules will - # still return the raw bytes - result = @test_warn(LOGGER, r"KeyError*", jlso["data"]) - - @test result == bytes - end - - @testset "bson" begin - f = @spawnat pnum begin - io = IOBuffer() - bson( - io, - Dict( - "data" => AxisArray( - rand(20, 10), - Axis{:time}(14010:10:14200), - Axis{:id}(1:10) - ) - ) - ) - return io - end - - io = fetch(f) - bytes = take!(io) - - jlso = JLSOFile(v"1.0", VERSION, :bson, img, pkgs, Dict("data" => bytes)) - - # Test failing to deserailize data because of missing modules will - # still return the raw bytes - result = @test_warn(LOGGER, r"UndefVarError*", jlso["data"]) - - @test result == bytes - end - finally - rmprocs(pnum) - end - end - end - - @testset "saving and loading" begin - mktempdir() do path - @testset "$fmt - $k" for fmt in (:bson, :serialize), (k, v) in datas - JLSO.save("$path/$fmt-$k.jlso", k => v; format=fmt) - result = JLSO.load("$path/$fmt-$k.jlso") - @test result[k] == v - end - end - end -end diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index 0415d8c..0000000 --- a/test/REQUIRE +++ /dev/null @@ -1,4 +0,0 @@ -AxisArrays -DataFrames -Distributions -TimeZones diff --git a/test/runtests.jl b/test/runtests.jl index 319ef3a..b0bebb9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,13 +4,13 @@ Mocking.enable(force=true) using Checkpoints using Test using AWSCore +using JLSO +using Random using AWSCore: AWSConfig using AWSS3: s3_put @testset "Checkpoints" begin - include("JLSO.jl") - include("testpkg.jl") x = reshape(collect(1:100), 10, 10) From c4197d328b5e23028e51bc990bda354212383487 Mon Sep 17 00:00:00 2001 From: Fernando Chorney Date: Mon, 25 Feb 2019 10:28:32 -0600 Subject: [PATCH 27/52] Clean up Docs --- docs/Manifest.toml | 26 ++++++++++++-------------- docs/src/api.md | 9 --------- docs/src/usage.md | 8 ++++---- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 304ba22..851a513 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,10 +1,8 @@ -# This file is machine-generated - editing it directly is not advised - [[AWSCore]] deps = ["Base64", "DataStructures", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Retry", "Sockets", "SymDict", "Test", "XMLDict"] -git-tree-sha1 = "7750fad931afd52b84a8f014617daecc74235e92" +git-tree-sha1 = "d9b2ada3bf289c504f765fec863cacea5aeb0541" uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" -version = "0.5.5" +version = "0.6.0" [[AWSS3]] deps = ["AWSCore", "Base64", "DataStructures", "Dates", "EzXML", "HTTP", "MbedTLS", "Retry", "SymDict", "Test", "XMLDict"] @@ -12,12 +10,6 @@ git-tree-sha1 = "4b7387c999f8c589542d1a449669d68a86583eca" uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" version = "0.5.0" -[[AWSSDK]] -deps = ["AWSCore"] -git-tree-sha1 = "89c95eab2bc1128fe247c31e9774d47eb7ef0d40" -uuid = "0d499d91-6ae5-5d63-9313-12987b87d5ad" -version = "0.4.0" - [[AWSTools]] deps = ["AWSCore", "AWSS3", "Base64", "Dates", "FilePathsBase", "MbedTLS", "Memento", "Mocking", "OrderedCollections", "Retry", "Test", "UUIDs", "XMLDict"] git-tree-sha1 = "42b01ca3ae5743647e80af6be24df496f66f7768" @@ -40,7 +32,7 @@ uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" version = "0.5.3" [[Checkpoints]] -deps = ["AWSCore", "AWSSDK", "AWSTools", "BSON", "DataStructures", "FilePathsBase", "Memento", "Mocking", "Pkg", "Serialization"] +deps = ["AWSTools", "DataStructures", "FilePathsBase", "JLSO", "Memento"] path = ".." uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" version = "0.1.0+" @@ -66,7 +58,7 @@ deps = ["Mmap"] uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" [[Distributed]] -deps = ["Random", "Serialization", "Sockets"] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[DocStringExtensions]] @@ -106,7 +98,7 @@ uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" version = "0.5.0" [[InteractiveUtils]] -deps = ["Markdown"] +deps = ["LinearAlgebra", "Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[IterTools]] @@ -115,6 +107,12 @@ git-tree-sha1 = "79246285c43602384e6f1943b3554042a3712056" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" version = "1.1.1" +[[JLSO]] +deps = ["BSON", "Dates", "Distributed", "InteractiveUtils", "Memento", "Pkg", "Random", "Serialization", "Test"] +git-tree-sha1 = "df7ec1f6de5b7a66d409023b349da45ff462a442" +uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" +version = "1.0.0" + [[JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" @@ -243,7 +241,7 @@ uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" version = "0.8.5" [[UUIDs]] -deps = ["Random", "SHA"] +deps = ["Random"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[Unicode]] diff --git a/docs/src/api.md b/docs/src/api.md index 13b92a6..f45abda 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -8,12 +8,3 @@ Public = true Private = true Pages = ["Checkpoints.jl"] ``` - -## Julia Serialized Objects (JLSO) - -```@autodocs -Modules = [Checkpoints.JLSO] -Public = true -Private = true -Pages = ["JLSO.jl"] -``` diff --git a/docs/src/usage.md b/docs/src/usage.md index 559858f..e054648 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -121,7 +121,7 @@ julia> TestPkg.foo(x, y) julia> isfile("checkpoints/TestPkg/foo.jlso") true -julia> using Checkpoints.JLSO +julia> using JLSO julia> d = JLSO.load("checkpoints/TestPkg/foo.jlso") Dict{String,Any} with 2 entries: @@ -229,12 +229,12 @@ If you're julia environment doesn't match the one used to save .jlso file (e.g., different julia version or missing packages) then you may get errors. ```julia -julia> using Checkpoints, Checkpoints.JLSO +julia> using Checkpoints, JLSO [ Info: Recompiling stale cache file /Users/rory/.playground/share/checkpoints/depot/compiled/v1.0/Checkpoints/E2USV.ji for Checkpoints [08085054-0ffc-5852-afcc-fc6ba29efde0] julia> d = JLSO.load("checkpoints/TestPkg/foo.jlso") -[warn | Checkpoints.JLSO]: EOFError: read end of file -[warn | Checkpoints.JLSO]: EOFError: read end of file +[warn | JLSO]: EOFError: read end of file +[warn | JLSO]: EOFError: read end of file Dict{String,Any} with 2 entries: "x" => UInt8[0x15, 0x00, 0x0e, 0x14, 0x02, 0xca, 0xca, 0x32, 0x20, 0x7b … 0x98, 0x3f, 0xc6, 0xc9, 0x58, 0xc8, 0xb7, 0xd2, 0x9e, 0x3f] "y" => UInt8[0x15, 0x00, 0x0e, 0x14, 0x02, 0xca, 0xca, 0xfe, 0x60, 0xe0 … 0xcc, 0x3f, 0x9f, 0xb0, 0xc4, 0x03, 0xca, 0x26, 0xec, 0x3f] From dd3ddab5f5f6a30d8a3d6435baef265109cd3b7b Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Fri, 29 Mar 2019 21:01:49 +0000 Subject: [PATCH 28/52] Fixing typeo --- src/handler.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handler.jl b/src/handler.jl index 322792a..635ed40 100644 --- a/src/handler.jl +++ b/src/handler.jl @@ -68,7 +68,7 @@ function commit!(handler::Handler{P}, path::P, jlso::JLSO.JLSOFile) where P <: A end function checkpoint(handler::Handler, name::String, data::Dict; tags...) - debug(LOGGER, "Checkpoint $name triggerred, with tags: $(join(tags, ", ")).") + debug(LOGGER, "Checkpoint $name triggered, with tags: $(join(tags, ", ")).") jlso = JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) p = path(handler, name; tags...) stage!(handler, jlso, data) @@ -79,6 +79,6 @@ end Define our no-op conditions just to be safe =# function checkpoint(handler::Nothing, name::String, data::Dict; tags...) - debug(LOGGER, "Checkpoint $name triggerred, but no handler has been set.") + debug(LOGGER, "Checkpoint $name triggered, but no handler has been set.") nothing end From f0cf1e52d04ed5890238c90e44424cdee9edf777 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Wed, 5 Jun 2019 19:55:48 +0000 Subject: [PATCH 29/52] Move to semver bounds --- Project.toml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Project.toml b/Project.toml index 92df879..aadda49 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" +version = "0.2.1" +authors = ["Invenia Technical Computing Corporation"] [deps] AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" @@ -9,12 +11,13 @@ JLSO = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" [compat] -AWSCore = ">=0.5.5" -AWSTools = ">=0.6.0" -FilePathsBase = ">=0.4.0" -Memento = ">=0.7.0" -Mocking = ">=0.5.2" -julia = "^1.0" +AWSCore = "0.5.5, 0.6" +AWSTools = "1.1.2" +FilePathsBase = 0.4" +Memento = "0.10, 0.11, 0.12" +Mocking = "0.5.2" +JLSO = "1" +julia = "1.0" [extras] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" From 5f9e58cec72698cc2dfb404ce3ab3e049c902b47 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Wed, 5 Jun 2019 19:59:15 +0000 Subject: [PATCH 30/52] fix typo --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index aadda49..280c23b 100644 --- a/Project.toml +++ b/Project.toml @@ -13,7 +13,7 @@ Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" [compat] AWSCore = "0.5.5, 0.6" AWSTools = "1.1.2" -FilePathsBase = 0.4" +FilePathsBase = "0.4" Memento = "0.10, 0.11, 0.12" Mocking = "0.5.2" JLSO = "1" From bd6dc65d583fc1b8c8625a8e08329bee5f0dcce4 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Thu, 22 Aug 2019 14:25:15 -0500 Subject: [PATCH 31/52] Switch to using AWSS3.S3Path rather than AWSTools. --- .gitlab-ci.yml | 49 ++++++++++++++++++++++++++++++++++++++++++++++ Project.toml | 12 ++++++------ src/Checkpoints.jl | 2 +- src/handler.jl | 12 +++++------- test/runtests.jl | 44 +++++++++++++++++++++-------------------- 5 files changed, 84 insertions(+), 35 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44ac62b..5ab897a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,32 @@ include: - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml +.awscli_install: &awscli_install + | + # Install AWS CLI + [[ $CI_DISPOSABLE_ENVIRONMENT == "true" ]] && SUDO="" || SUDO="sudo" + if [[ -x "$(command -v yum)" ]]; then + # Note: git and gcc is needed to install cloudspy + ${SUDO} yum -y install python3 python3-devel git gcc + elif [[ -x "$(command -v apt-get)" ]]; then + ${SUDO} apt-get -y update + ${SUDO} apt-get -y install python3 python3-dev python3-venv git + elif [[ -x "$(command -v brew)" ]]; then + brew install python || (brew upgrade python && brew cleanup python) + fi + + python3 -m venv venv + source venv/bin/activate + pip install awscli + python --version + aws --version + +.cloudspy_install: &cloudspy_install + | + # Install Cloudspy in the env + pip install git+https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.invenia.ca/infrastructure/cloudspy.git#egg=cloudspy + + "1.0 (Mac)": tags: - mac @@ -66,3 +92,26 @@ include: - 32-bit - shell-ci extends: .test_shell_nightly + +"Online Tests": + tags: + - linux + - docker-build + - shell-ci + variables: + LIVE: "true" # Runs the online S3 tests against AWS + AWS_DEFAULT_REGION: us-east-1 + before_script: + - *awscli_install + - *cloudspy_install + - curl -sS -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/$CI_HELPER_BRANCH/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION + script: + - source julia-ci export + - export PATH="$PATH:/usr/local/bin" + - eval $(aws-stack-outputs gitlab-ci-runners) # Exports the test bucket to work in + - printenv + - ./julia-ci test + - ./julia-ci coverage + extends: .test_shell_1_0 diff --git a/Project.toml b/Project.toml index 280c23b..558a800 100644 --- a/Project.toml +++ b/Project.toml @@ -1,10 +1,10 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" -version = "0.2.1" authors = ["Invenia Technical Computing Corporation"] +version = "0.2.1" [deps] -AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" +AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" JLSO = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" @@ -12,11 +12,11 @@ Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" [compat] AWSCore = "0.5.5, 0.6" -AWSTools = "1.1.2" -FilePathsBase = "0.4" -Memento = "0.10, 0.11, 0.12" -Mocking = "0.5.2" +AWSS3 = "0.6" +FilePathsBase = "0.6" JLSO = "1" +Memento = "0.10, 0.11, 0.12" +Mocking = "0.5.2, 0.6" julia = "1.0" [extras] diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index d90ca8a..4afb7ab 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -7,7 +7,7 @@ the ability to configure how those checkpoints save data externally """ module Checkpoints -using AWSTools.S3 +using AWSS3 using Memento using FilePathsBase using JLSO diff --git a/src/handler.jl b/src/handler.jl index 322792a..d4a12b0 100644 --- a/src/handler.jl +++ b/src/handler.jl @@ -12,7 +12,7 @@ FilePath are used to abstract away differences between paths on S3 or locally. """ Handler(path::AbstractPath; kwargs...) = Handler(path, kwargs) Handler(path::String; kwargs...) = Handler(Path(path), kwargs) -Handler(bucket::String, prefix::String; kwargs...) = Handler(S3Path(bucket, prefix), kwargs) +Handler(bucket::String, prefix::String; kwargs...) = Handler(S3Path("s3://$bucket/$prefix"), kwargs) """ path(handler, name; tags...) @@ -55,15 +55,13 @@ end Write the JLSOFile to the path as bytes. """ function commit!(handler::Handler{P}, path::P, jlso::JLSO.JLSOFile) where P <: AbstractPath + # NOTE: This is only necessary because FilePathsBase.FileBuffer needs to support + # write(::FileBuffer, ::UInt8) + # https://github.com/rofinn/FilePathsBase.jl/issues/45 io = IOBuffer() write(io, jlso) bytes = take!(io) - - # FilePathsBase should probably default to a no-op? - if P <: Union{PosixPath, WindowsPath} && hasparent(path) - mkdir(parent(path); recursive=true, exist_ok=true) - end - + mkdir(parent(path); recursive=true, exist_ok=true) write(path, bytes) end diff --git a/test/runtests.jl b/test/runtests.jl index b0bebb9..8083d83 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,14 +1,12 @@ -using Mocking -Mocking.enable(force=true) - using Checkpoints using Test using AWSCore +using FilePathsBase using JLSO using Random using AWSCore: AWSConfig -using AWSS3: s3_put +using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @testset "Checkpoints" begin include("testpkg.jl") @@ -38,23 +36,27 @@ using AWSS3: s3_put end end - @testset "S3 handler" begin - objects = Dict{String, Vector{UInt8}}() - - s3_put_patch = @patch function s3_put(config::AWSConfig, bucket, prefix, data) - objects[joinpath(bucket, prefix)] = data - end - - config = AWSCore.aws_config() - bucket = "mybucket" - prefix = joinpath("mybackrun") - Checkpoints.config("TestPkg.bar", bucket, prefix) - - apply(s3_put_patch) do - TestPkg.bar(a) - expected_path = joinpath(bucket, prefix, "date=2017-01-01", "TestPkg/bar.jlso") - io = IOBuffer(objects[expected_path]) - @test JLSO.load(io)["data"] == a + if parse(Bool, get(ENV, "LIVE", "false")) + @testset "S3 handler" begin + config = AWSCore.aws_config() + prefix = "Checkpoints.jl/" + @show ENV + bucket = get( + ENV, + "TestBucketAndPrefix", + string(aws_account_number(config), "-tests") + ) + bucket in s3_list_buckets(config) || s3_create_bucket(config, bucket) + + mkdir(Path("s3://$bucket/$prefix"); recursive=true, exist_ok=true) + + mktmpdir(Path("s3://$bucket/Checkpoints.jl/")) do fp + Checkpoints.config("TestPkg.bar", fp) + + TestPkg.bar(a) + expected_path = fp / "date=2017-01-01" / "TestPkg/bar.jlso" + @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a + end end end From 9a80e5c01633241aaa9c6c934263e4d98eb53593 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Fri, 23 Aug 2019 16:01:04 +0000 Subject: [PATCH 32/52] Apply suggestion to test/runtests.jl --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8083d83..ea6d3bd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,7 +36,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket end end - if parse(Bool, get(ENV, "LIVE", "false")) + if get(ENV, "LIVE", "false") == "true" @testset "S3 handler" begin config = AWSCore.aws_config() prefix = "Checkpoints.jl/" From 5522d362b2207d966a1ec1b9f591157338a8b4e4 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Fri, 23 Aug 2019 16:01:09 +0000 Subject: [PATCH 33/52] Apply suggestion to test/runtests.jl --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index ea6d3bd..d9354bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -40,7 +40,6 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @testset "S3 handler" begin config = AWSCore.aws_config() prefix = "Checkpoints.jl/" - @show ENV bucket = get( ENV, "TestBucketAndPrefix", From 29577746d2a4f22ed1c076ff8a42df7f875b9f7f Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Fri, 23 Aug 2019 11:16:10 -0500 Subject: [PATCH 34/52] Use gitlab-ci-helper common functions. --- .gitlab-ci.yml | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ab897a..44fb6fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,32 +6,7 @@ stages: include: - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml - -.awscli_install: &awscli_install - | - # Install AWS CLI - [[ $CI_DISPOSABLE_ENVIRONMENT == "true" ]] && SUDO="" || SUDO="sudo" - if [[ -x "$(command -v yum)" ]]; then - # Note: git and gcc is needed to install cloudspy - ${SUDO} yum -y install python3 python3-devel git gcc - elif [[ -x "$(command -v apt-get)" ]]; then - ${SUDO} apt-get -y update - ${SUDO} apt-get -y install python3 python3-dev python3-venv git - elif [[ -x "$(command -v brew)" ]]; then - brew install python || (brew upgrade python && brew cleanup python) - fi - - python3 -m venv venv - source venv/bin/activate - pip install awscli - python --version - aws --version - -.cloudspy_install: &cloudspy_install - | - # Install Cloudspy in the env - pip install git+https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.invenia.ca/infrastructure/cloudspy.git#egg=cloudspy - + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/common-functions.yml "1.0 (Mac)": tags: @@ -100,10 +75,10 @@ include: - shell-ci variables: LIVE: "true" # Runs the online S3 tests against AWS - AWS_DEFAULT_REGION: us-east-1 before_script: - - *awscli_install - - *cloudspy_install + - echo "$common_functions" > common && source common + - install_awscli + - install_cloudspy - curl -sS -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/$CI_HELPER_BRANCH/julia-ci - chmod +x julia-ci - ./julia-ci install $JULIA_VERSION @@ -111,7 +86,6 @@ include: - source julia-ci export - export PATH="$PATH:/usr/local/bin" - eval $(aws-stack-outputs gitlab-ci-runners) # Exports the test bucket to work in - - printenv - ./julia-ci test - ./julia-ci coverage extends: .test_shell_1_0 From fd34a6aff765b5dcc41543bd079632296f074aef Mon Sep 17 00:00:00 2001 From: Mary Jo Ramos Date: Wed, 4 Sep 2019 21:40:41 +0000 Subject: [PATCH 35/52] Update .gitlab-ci.yml to use job matrix --- .gitlab-ci.yml | 79 ++++++-------------------------------------------- Project.toml | 2 +- REQUIRE | 6 ---- 3 files changed, 10 insertions(+), 77 deletions(-) delete mode 100644 REQUIRE diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44fb6fc..c86f715 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,72 +1,13 @@ -stages: - - setup - - test - - teardown - +--- include: - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/common-functions.yml - -"1.0 (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_1_0 - -"1.0 (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_1_0 - -"1.0 (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_1_0 - -"1.1 (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_1_1 + - project: infrastructure/gitlab-ci-helper + file: /templates/julia.yml -"1.1 (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_1_1 - -"1.1 (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_1_1 - -"Nightly (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_nightly - -"Nightly (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_nightly - -"Nightly (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_nightly +.setup: &setup + | + echo "$common_functions" > common && source common + install_awscli + install_cloudspy "Online Tests": tags: @@ -76,9 +17,7 @@ include: variables: LIVE: "true" # Runs the online S3 tests against AWS before_script: - - echo "$common_functions" > common && source common - - install_awscli - - install_cloudspy + - *setup - curl -sS -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/$CI_HELPER_BRANCH/julia-ci - chmod +x julia-ci - ./julia-ci install $JULIA_VERSION diff --git a/Project.toml b/Project.toml index 558a800..03f7fda 100644 --- a/Project.toml +++ b/Project.toml @@ -17,7 +17,7 @@ FilePathsBase = "0.6" JLSO = "1" Memento = "0.10, 0.11, 0.12" Mocking = "0.5.2, 0.6" -julia = "1.0" +julia = "1" [extras] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" diff --git a/REQUIRE b/REQUIRE deleted file mode 100644 index 77796f8..0000000 --- a/REQUIRE +++ /dev/null @@ -1,6 +0,0 @@ -julia 1.0 -AWSTools 0.6.0 -DataStructures -FilePathsBase 0.4.0 -JLSO -Memento 0.7.0 From cfcb52b264d4709105404944714e09f92597fc05 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 4 Sep 2019 21:38:36 +0000 Subject: [PATCH 36/52] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 03f7fda..2b64219 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = ["Invenia Technical Computing Corporation"] -version = "0.2.1" +version = "0.2.2" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" From 7f2f5698abb2aa87c2c66d25a9e002757dace736 Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Mon, 9 Sep 2019 12:07:52 +0100 Subject: [PATCH 37/52] drop Mocking dep --- Project.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 2b64219..52e5f04 100644 --- a/Project.toml +++ b/Project.toml @@ -16,15 +16,13 @@ AWSS3 = "0.6" FilePathsBase = "0.6" JLSO = "1" Memento = "0.10, 0.11, 0.12" -Mocking = "0.5.2, 0.6" julia = "1" [extras] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" -Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["AWSCore", "AWSS3", "Mocking", "Random", "Test"] +test = ["AWSCore", "AWSS3", "Random", "Test"] From 75ba7f8d3bd03bb6189761144a5ffac6992c584d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Sep 2019 09:55:41 -0500 Subject: [PATCH 38/52] Set project version to 0.2.3 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 52e5f04..a93812d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = ["Invenia Technical Computing Corporation"] -version = "0.2.2" +version = "0.2.3" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" From 69aac1eff1d597d931e4616af0c7ddcbfa1704ba Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 12 Nov 2019 17:43:09 -0600 Subject: [PATCH 39/52] Use symbol keys (vs strings) for JLSO.load output on v1.2. --- test/runtests.jl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d9354bf..3c8d817 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,8 +31,8 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @test !isfile(bar_path) data = JLSO.load(foo_path) - @test data["x"] == x - @test data["y"] == y + @test data[:x] == x + @test data[:y] == y end end @@ -73,7 +73,10 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket end @testset "Single" begin mktempdir() do path - d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + d = Dict(zip( + map(x -> Symbol(randstring(4)), 1:10), + map(x -> rand(10), 1:10) + )) Checkpoints.config("TestPkg.baz", path) TestPkg.baz(d) @@ -92,7 +95,10 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket end @testset "Multi" begin mktempdir() do path - a = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + a = Dict(zip( + map(x -> Symbol(randstring(4)), 1:10), + map(x -> rand(10), 1:10) + )) b = rand(10) Checkpoints.config("TestPkg.qux" , path) @@ -113,7 +119,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket end data = JLSO.load(qux_b_path) - @test data["data"] == b + @test data[:data] == b end end end From e373965956c28bdb618823af0628426573bb43b1 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 12 Nov 2019 17:55:19 -0600 Subject: [PATCH 40/52] Update docs and live tests as well. --- docs/src/usage.md | 2 +- src/Checkpoints.jl | 2 +- src/session.jl | 2 +- test/runtests.jl | 2 +- test/testpkg.jl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/usage.md b/docs/src/usage.md index e054648..233c9b0 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -181,7 +181,7 @@ julia> Checkpoints.config("TestPkg.bar", "./checkpoints") julia> JLSO.load("./checkpoints/date=2017-01-01/TestPkg/bar.jlso") Dict{String,Any} with 1 entry: - "data" => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469] + :data => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469] ``` ## Sessions diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 4afb7ab..3251244 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -54,7 +54,7 @@ end checkpoint(name::String, data::Pair...; tags...) = checkpoint(name, Dict(data...); tags...) -checkpoint(name::String, data; tags...) = checkpoint(name, Dict("data" => data); tags...) +checkpoint(name::String, data; tags...) = checkpoint(name, Dict(:data => data); tags...) function checkpoint(prefix::Union{Module, String}, name::String, args...; kwargs...) checkpoint("$prefix.$name", args...; kwargs...) diff --git a/src/session.jl b/src/session.jl index 7223f61..477e7bf 100644 --- a/src/session.jl +++ b/src/session.jl @@ -59,4 +59,4 @@ end checkpoint(s::Session, data::Pair...; tags...) = checkpoint(s, Dict(data...); tags...) -checkpoint(s::Session, data; tags...) = checkpoint(s, Dict("data" => data); tags...) +checkpoint(s::Session, data; tags...) = checkpoint(s, Dict(:data => data); tags...) diff --git a/test/runtests.jl b/test/runtests.jl index 3c8d817..5fc8afb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -54,7 +54,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket TestPkg.bar(a) expected_path = fp / "date=2017-01-01" / "TestPkg/bar.jlso" - @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a + @test JLSO.load(IOBuffer(read(expected_path)))[:data] == a end end end diff --git a/test/testpkg.jl b/test/testpkg.jl index 44760a2..69da271 100644 --- a/test/testpkg.jl +++ b/test/testpkg.jl @@ -14,7 +14,7 @@ function foo(x::Matrix, y::Matrix) end function bar(a::Vector) - # Save a single value for bar.jlso. The object name in that file defaults to "data". + # Save a single value for bar.jlso. The object name in that file defaults to :data. checkpoint(MODULE, "bar", a; date="2017-01-01") return a * a' end From 039994ee05c1fb269f7ee8e340b4753ab9ba9df4 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Wed, 13 Nov 2019 22:21:59 +0000 Subject: [PATCH 41/52] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a93812d..8f580e3 100644 --- a/Project.toml +++ b/Project.toml @@ -14,7 +14,7 @@ Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" AWSCore = "0.5.5, 0.6" AWSS3 = "0.6" FilePathsBase = "0.6" -JLSO = "1" +JLSO = "1.2" Memento = "0.10, 0.11, 0.12" julia = "1" From 1f1ca4da4838c1473981467639bf2f22f4998165 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 21 Jan 2020 13:29:14 -0600 Subject: [PATCH 42/52] Revert JLSO breaking `load` change in tests and deprecated using strings for object names. --- docs/src/usage.md | 12 ++++++------ src/Checkpoints.jl | 4 +++- src/handler.jl | 10 +++++----- src/session.jl | 4 ++-- test/runtests.jl | 15 ++++++++------- test/testpkg.jl | 2 +- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/src/usage.md b/docs/src/usage.md index 233c9b0..febc5f2 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -15,7 +15,7 @@ julia> module TestPkg function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables - checkpoint(MODULE, "foo", "x" => x, "y" => y) + checkpoint(MODULE, "foo", :x => x, :y => y) return x * y end @@ -63,7 +63,7 @@ As a reference, here is the sample code for `TestPkg.foo` that we'll be calling. ... function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables - checkpoint(MODULE, "foo", "x" => x, "y" => y) + checkpoint(MODULE, "foo", :x => x, :y => y) return x * y end ... @@ -181,7 +181,7 @@ julia> Checkpoints.config("TestPkg.bar", "./checkpoints") julia> JLSO.load("./checkpoints/date=2017-01-01/TestPkg/bar.jlso") Dict{String,Any} with 1 entry: - :data => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469] + "data" => [0.166881, 0.817174, 0.413097, 0.955415, 0.139473, 0.49518, 0.416731, 0.431096, 0.126912, 0.600469] ``` ## Sessions @@ -208,10 +208,10 @@ end #### Application ```julia -julia> d = Dict("x" => rand(10), "y" => rand(10)) +julia> d = Dict(:x => rand(10), :y => rand(10)) Dict{String,Array{Float64,1}} with 2 entries: - "x" => [0.517666, 0.976474, 0.961658, 0.0933946, 0.877478, 0.428836, 0.0623459, 0.548001, 0.437111, 0.0783503] - "y" => [0.0623591, 0.0441436, 0.28578, 0.289995, 0.999642, 0.26299, 0.965148, 0.899285, 0.292166, 0.595886] + :x => [0.517666, 0.976474, 0.961658, 0.0933946, 0.877478, 0.428836, 0.0623459, 0.548001, 0.437111, 0.0783503] + :y => [0.0623591, 0.0441436, 0.28578, 0.289995, 0.999642, 0.26299, 0.965148, 0.899285, 0.292166, 0.595886] julia> TestPkg.baz(d) diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 3251244..910b17e 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -48,7 +48,7 @@ Alternatively, you can also checkpoint with to a session which stages the data t commited later by `commit!(session)`. Explicitly calling checkpoint on a handler is generally not advised, but is an option. """ -function checkpoint(name::String, data::Dict; tags...) +function checkpoint(name::String, data::Dict{Symbol}; tags...) checkpoint(CHECKPOINTS[name], name, data; tags...) end @@ -112,5 +112,7 @@ function register(prefix::Union{Module, String}, labels::Vector{String}) register(map(l -> join([prefix, l], "."), labels)) end +include("deprecated.jl") + end # module diff --git a/src/handler.jl b/src/handler.jl index ec04d75..784e4d9 100644 --- a/src/handler.jl +++ b/src/handler.jl @@ -37,11 +37,11 @@ function path(handler::Handler{P}, name::String; tags...) where P end """ - stage!(handler::Handler, jlso::JLSOFIle, data::Dict) + stage!(handler::Handler, jlso::JLSOFIle, data::Dict{Symbol}) Update the JLSOFile with the new data. """ -function stage!(handler::Handler, jlso::JLSO.JLSOFile, data::Dict) +function stage!(handler::Handler, jlso::JLSO.JLSOFile, data::Dict{Symbol}) for (k, v) in data jlso[k] = v end @@ -65,9 +65,9 @@ function commit!(handler::Handler{P}, path::P, jlso::JLSO.JLSOFile) where P <: A write(path, bytes) end -function checkpoint(handler::Handler, name::String, data::Dict; tags...) +function checkpoint(handler::Handler, name::String, data::Dict{Symbol}; tags...) debug(LOGGER, "Checkpoint $name triggered, with tags: $(join(tags, ", ")).") - jlso = JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) + jlso = JLSO.JLSOFile(Dict{Symbol, Vector{UInt8}}(); handler.settings...) p = path(handler, name; tags...) stage!(handler, jlso, data) commit!(handler, p, jlso) @@ -76,7 +76,7 @@ end #= Define our no-op conditions just to be safe =# -function checkpoint(handler::Nothing, name::String, data::Dict; tags...) +function checkpoint(handler::Nothing, name::String, data::Dict{Symbol}; tags...) debug(LOGGER, "Checkpoint $name triggered, but no handler has been set.") nothing end diff --git a/src/session.jl b/src/session.jl index 477e7bf..b299e61 100644 --- a/src/session.jl +++ b/src/session.jl @@ -10,7 +10,7 @@ function Session(name::String) handler = CHECKPOINTS[name] objects = DefaultDict{AbstractPath, JLSO.JLSOFile}() do - JLSO.JLSOFile(Dict{String, Vector{UInt8}}(); handler.settings...) + JLSO.JLSOFile(Dict{Symbol, Vector{UInt8}}(); handler.settings...) end Session{typeof(handler)}(name, handler, objects) @@ -48,7 +48,7 @@ function commit!(session::Session) end end -function checkpoint(session::Session, data::Dict; tags...) +function checkpoint(session::Session, data::Dict{Symbol}; tags...) # No-ops skip when handler is nothing session.handler === nothing && return nothing diff --git a/test/runtests.jl b/test/runtests.jl index 5fc8afb..81187fa 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,8 +31,8 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @test !isfile(bar_path) data = JLSO.load(foo_path) - @test data[:x] == x - @test data[:y] == y + @test data["x"] == x + @test data["y"] == y end end @@ -54,7 +54,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket TestPkg.bar(a) expected_path = fp / "date=2017-01-01" / "TestPkg/bar.jlso" - @test JLSO.load(IOBuffer(read(expected_path)))[:data] == a + @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a end end end @@ -62,7 +62,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @testset "Sessions" begin @testset "No-op" begin mktempdir() do path - d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + d = Dict(zip(map(x -> Symbol(randstring(4)), 1:10), map(x -> rand(10), 1:10))) TestPkg.baz(d) @@ -89,7 +89,7 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket data = JLSO.load(baz_path) for (k, v) in data - @test v == d[k] + @test v == d[Symbol(k)] end end end @@ -115,12 +115,13 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket data = JLSO.load(qux_a_path) for (k, v) in data - @test v == a[k] + @test v == a[Symbol(k)] end data = JLSO.load(qux_b_path) - @test data[:data] == b + @test data["data"] == b end end end + include("deprecated.jl") end diff --git a/test/testpkg.jl b/test/testpkg.jl index 69da271..4060fa3 100644 --- a/test/testpkg.jl +++ b/test/testpkg.jl @@ -9,7 +9,7 @@ __init__() = register(MODULE, ["foo", "bar", "baz", "qux_a", "qux_b"]) function foo(x::Matrix, y::Matrix) # Save multiple variables to 1 foo.jlso file by passing in pairs of variables - checkpoint(MODULE, "foo", "x" => x, "y" => y) + checkpoint(MODULE, "foo", :x => x, :y => y) return x * y end From 908e8282aafabf297d784ba68969f8f6b7ef2201 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 21 Jan 2020 13:29:14 -0600 Subject: [PATCH 43/52] Revert JLSO breaking `load` change in tests and deprecated using strings for object names. --- test/deprecated.jl | 116 +++++++++++++++++++++++++++++++++++++++++++++ test/testpkgdep.jl | 42 ++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 test/deprecated.jl create mode 100644 test/testpkgdep.jl diff --git a/test/deprecated.jl b/test/deprecated.jl new file mode 100644 index 0000000..710003c --- /dev/null +++ b/test/deprecated.jl @@ -0,0 +1,116 @@ +@testset "Deprecated" begin + include("testpkgdep.jl") + + x = reshape(collect(1:100), 10, 10) + y = reshape(collect(101:200), 10, 10) + a = collect(1:10) + + @testset "Local handler" begin + mktempdir() do path + Checkpoints.config("TestPkgDep.foo", path) + + TestPkgDep.foo(x, y) + TestPkgDep.bar(a) + + mod_path = joinpath(path, "TestPkgDep") + @test isdir(mod_path) + + foo_path = joinpath(path, "TestPkgDep", "foo.jlso") + bar_path = joinpath(path, "TestPkgDep", "bar.jlso") + @test isfile(foo_path) + @test !isfile(bar_path) + + data = JLSO.load(foo_path) + @test data["x"] == x + @test data["y"] == y + end + end + + if get(ENV, "LIVE", "false") == "true" + @testset "S3 handler" begin + config = AWSCore.aws_config() + prefix = "Checkpoints.jl/" + bucket = get( + ENV, + "TestBucketAndPrefix", + string(aws_account_number(config), "-tests") + ) + bucket in s3_list_buckets(config) || s3_create_bucket(config, bucket) + + mkdir(Path("s3://$bucket/$prefix"); recursive=true, exist_ok=true) + + mktmpdir(Path("s3://$bucket/Checkpoints.jl/")) do fp + Checkpoints.config("TestPkgDep.bar", fp) + + TestPkgDep.bar(a) + expected_path = fp / "date=2017-01-01" / "TestPkgDep/bar.jlso" + @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a + end + end + end + + @testset "Sessions" begin + @testset "No-op" begin + mktempdir() do path + d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) + + TestPkgDep.baz(d) + + mod_path = joinpath(path, "TestPkgDep") + baz_path = joinpath(path, "TestPkgDep", "baz.jlso") + @test !isfile(baz_path) + end + end + @testset "Single" begin + mktempdir() do path + d = Dict(zip( + map(x -> randstring(4), 1:10), + map(x -> rand(10), 1:10) + )) + Checkpoints.config("TestPkgDep.baz", path) + + TestPkgDep.baz(d) + + mod_path = joinpath(path, "TestPkgDep") + @test isdir(mod_path) + + baz_path = joinpath(path, "TestPkgDep", "baz.jlso") + @test isfile(baz_path) + + data = JLSO.load(baz_path) + for (k, v) in data + @test v == d[k] + end + end + end + @testset "Multi" begin + mktempdir() do path + a = Dict(zip( + map(x -> randstring(4), 1:10), + map(x -> rand(10), 1:10) + )) + b = rand(10) + Checkpoints.config("TestPkgDep.qux" , path) + + TestPkgDep.qux(a, b) + + mod_path = joinpath(path, "TestPkgDep") + @test isdir(mod_path) + + qux_a_path = joinpath(path, "TestPkgDep", "qux_a.jlso") + @test isfile(qux_a_path) + + qux_b_path = joinpath(path, "TestPkgDep", "qux_b.jlso") + @test isfile(qux_b_path) + + data = JLSO.load(qux_a_path) + for (k, v) in data + @test v == a[k] + end + + data = JLSO.load(qux_b_path) + @test data["data"] == b + end + end + end +end diff --git a/test/testpkgdep.jl b/test/testpkgdep.jl new file mode 100644 index 0000000..f71abc1 --- /dev/null +++ b/test/testpkgdep.jl @@ -0,0 +1,42 @@ +module TestPkgDep + +using Checkpoints: register, checkpoint, Session + +# We aren't using `@__MODULE__` because that would return TestPkg on 0.6 and Main.TestPkg on 0.7 +const MODULE = "TestPkgDep" + +__init__() = register(MODULE, ["foo", "bar", "baz", "qux_a", "qux_b"]) + +function foo(x::Matrix, y::Matrix) + # Save multiple variables to 1 foo.jlso file by passing in pairs of variables + checkpoint(MODULE, "foo", "x" => x, "y" => y) + return x * y +end + +function bar(a::Vector) + # Save a single value for bar.jlso. The object name in that file defaults to :data. + checkpoint(MODULE, "bar", a; date="2017-01-01") + return a * a' +end + +function baz(data::Dict) + # Check that saving multiple values to a Session works. + Session(MODULE, "baz") do s + for (k, v) in data + checkpoint(s, k => v) + end + end +end + +function qux(a::Dict, b::Vector) + # Check that saving multiple values to multiple Sessions also works. + Session(MODULE, ["qux_a", "qux_b"]) do sa, sb + for (k, v) in a + checkpoint(sa, k => v) + end + + checkpoint(sb, b) + end +end + +end From 69c44eec3cf91c7122e500734fcb7cd6f2e2fade Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 21 Jan 2020 19:38:42 +0000 Subject: [PATCH 44/52] Apply suggestion to Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8f580e3..4795d23 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = ["Invenia Technical Computing Corporation"] -version = "0.2.3" +version = "0.2.4" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" From f734c9df357937f27a161ddf9fbf7584cc8e61f1 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 21 Jan 2020 13:29:14 -0600 Subject: [PATCH 45/52] Revert JLSO breaking `load` change in tests and deprecated using strings for object names. --- src/deprecated.jl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/deprecated.jl diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..0896e80 --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,24 @@ +# We need to provide very specific deprecation methods to avoid ambiguous to the +# Dict(:data => data) fallback. + +# 2-arg form +@deprecate( + checkpoint(name::String, data::Dict{String}; tags...), + checkpoint(name, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) +) + +@deprecate( + checkpoint(handler::Handler, data::Dict{String}; tags...), + checkpoint(handler, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) +) + +@deprecate( + checkpoint(session::Session, data::Dict{String}; tags...), + checkpoint(session, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) +) + +# 3-arg form +@deprecate( + checkpoint(handler::Handler, name::String, data::Dict{String}; tags...), + checkpoint(handler, name, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) +) From 961554529d53c5c493f3dee5d1a3a33f05247658 Mon Sep 17 00:00:00 2001 From: Nicholas Robinson Date: Fri, 14 Feb 2020 14:17:25 +0000 Subject: [PATCH 46/52] Update to JLSO v2 --- Project.toml | 8 ++-- src/Checkpoints.jl | 3 -- src/deprecated.jl | 24 ---------- test/deprecated.jl | 116 --------------------------------------------- test/runtests.jl | 11 +++-- 5 files changed, 10 insertions(+), 152 deletions(-) delete mode 100644 src/deprecated.jl delete mode 100644 test/deprecated.jl diff --git a/Project.toml b/Project.toml index 4795d23..e504d8d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" -authors = ["Invenia Technical Computing Corporation"] -version = "0.2.4" +authors = "Invenia Technical Computing Corporation" +version = "0.3.0" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -13,8 +13,8 @@ Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" [compat] AWSCore = "0.5.5, 0.6" AWSS3 = "0.6" -FilePathsBase = "0.6" -JLSO = "1.2" +FilePathsBase = "0.6, 0.7" +JLSO = "2" Memento = "0.10, 0.11, 0.12" julia = "1" diff --git a/src/Checkpoints.jl b/src/Checkpoints.jl index 910b17e..45bf46e 100644 --- a/src/Checkpoints.jl +++ b/src/Checkpoints.jl @@ -112,7 +112,4 @@ function register(prefix::Union{Module, String}, labels::Vector{String}) register(map(l -> join([prefix, l], "."), labels)) end -include("deprecated.jl") - - end # module diff --git a/src/deprecated.jl b/src/deprecated.jl deleted file mode 100644 index 0896e80..0000000 --- a/src/deprecated.jl +++ /dev/null @@ -1,24 +0,0 @@ -# We need to provide very specific deprecation methods to avoid ambiguous to the -# Dict(:data => data) fallback. - -# 2-arg form -@deprecate( - checkpoint(name::String, data::Dict{String}; tags...), - checkpoint(name, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) -) - -@deprecate( - checkpoint(handler::Handler, data::Dict{String}; tags...), - checkpoint(handler, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) -) - -@deprecate( - checkpoint(session::Session, data::Dict{String}; tags...), - checkpoint(session, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) -) - -# 3-arg form -@deprecate( - checkpoint(handler::Handler, name::String, data::Dict{String}; tags...), - checkpoint(handler, name, Dict(Symbol(k)=>v for (k,v) in pairs(data)); tags...) -) diff --git a/test/deprecated.jl b/test/deprecated.jl deleted file mode 100644 index 710003c..0000000 --- a/test/deprecated.jl +++ /dev/null @@ -1,116 +0,0 @@ -@testset "Deprecated" begin - include("testpkgdep.jl") - - x = reshape(collect(1:100), 10, 10) - y = reshape(collect(101:200), 10, 10) - a = collect(1:10) - - @testset "Local handler" begin - mktempdir() do path - Checkpoints.config("TestPkgDep.foo", path) - - TestPkgDep.foo(x, y) - TestPkgDep.bar(a) - - mod_path = joinpath(path, "TestPkgDep") - @test isdir(mod_path) - - foo_path = joinpath(path, "TestPkgDep", "foo.jlso") - bar_path = joinpath(path, "TestPkgDep", "bar.jlso") - @test isfile(foo_path) - @test !isfile(bar_path) - - data = JLSO.load(foo_path) - @test data["x"] == x - @test data["y"] == y - end - end - - if get(ENV, "LIVE", "false") == "true" - @testset "S3 handler" begin - config = AWSCore.aws_config() - prefix = "Checkpoints.jl/" - bucket = get( - ENV, - "TestBucketAndPrefix", - string(aws_account_number(config), "-tests") - ) - bucket in s3_list_buckets(config) || s3_create_bucket(config, bucket) - - mkdir(Path("s3://$bucket/$prefix"); recursive=true, exist_ok=true) - - mktmpdir(Path("s3://$bucket/Checkpoints.jl/")) do fp - Checkpoints.config("TestPkgDep.bar", fp) - - TestPkgDep.bar(a) - expected_path = fp / "date=2017-01-01" / "TestPkgDep/bar.jlso" - @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a - end - end - end - - @testset "Sessions" begin - @testset "No-op" begin - mktempdir() do path - d = Dict(zip(map(x -> randstring(4), 1:10), map(x -> rand(10), 1:10))) - - TestPkgDep.baz(d) - - mod_path = joinpath(path, "TestPkgDep") - baz_path = joinpath(path, "TestPkgDep", "baz.jlso") - @test !isfile(baz_path) - end - end - @testset "Single" begin - mktempdir() do path - d = Dict(zip( - map(x -> randstring(4), 1:10), - map(x -> rand(10), 1:10) - )) - Checkpoints.config("TestPkgDep.baz", path) - - TestPkgDep.baz(d) - - mod_path = joinpath(path, "TestPkgDep") - @test isdir(mod_path) - - baz_path = joinpath(path, "TestPkgDep", "baz.jlso") - @test isfile(baz_path) - - data = JLSO.load(baz_path) - for (k, v) in data - @test v == d[k] - end - end - end - @testset "Multi" begin - mktempdir() do path - a = Dict(zip( - map(x -> randstring(4), 1:10), - map(x -> rand(10), 1:10) - )) - b = rand(10) - Checkpoints.config("TestPkgDep.qux" , path) - - TestPkgDep.qux(a, b) - - mod_path = joinpath(path, "TestPkgDep") - @test isdir(mod_path) - - qux_a_path = joinpath(path, "TestPkgDep", "qux_a.jlso") - @test isfile(qux_a_path) - - qux_b_path = joinpath(path, "TestPkgDep", "qux_b.jlso") - @test isfile(qux_b_path) - - data = JLSO.load(qux_a_path) - for (k, v) in data - @test v == a[k] - end - - data = JLSO.load(qux_b_path) - @test data["data"] == b - end - end - end -end diff --git a/test/runtests.jl b/test/runtests.jl index 81187fa..8465551 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,8 +31,8 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket @test !isfile(bar_path) data = JLSO.load(foo_path) - @test data["x"] == x - @test data["y"] == y + @test data[:x] == x + @test data[:y] == y end end @@ -54,9 +54,11 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket TestPkg.bar(a) expected_path = fp / "date=2017-01-01" / "TestPkg/bar.jlso" - @test JLSO.load(IOBuffer(read(expected_path)))["data"] == a + @test JLSO.load(IOBuffer(read(expected_path)))[:data] == a end end + else + @warn("Skipping AWS S3 tests. Set `ENV[\"LIVE\"] = true` to run.") end @testset "Sessions" begin @@ -119,9 +121,8 @@ using AWSS3: S3Path, s3_put, s3_list_buckets, s3_create_bucket end data = JLSO.load(qux_b_path) - @test data["data"] == b + @test data[:data] == b end end end - include("deprecated.jl") end From 7f61582267ef4704672403f0d9e33c2bbf588e7a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Mar 2020 09:17:50 -0500 Subject: [PATCH 47/52] Use Julia 1.3 for online tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c86f715..f103447 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,4 +27,4 @@ include: - eval $(aws-stack-outputs gitlab-ci-runners) # Exports the test bucket to work in - ./julia-ci test - ./julia-ci coverage - extends: .test_shell_1_0 + extends: .test_shell_1_3 From 96a7931c309acae9ef51a7e64e1569a1514747ba Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 11 Mar 2020 12:06:42 -0500 Subject: [PATCH 48/52] GitLab CI before_script available for extension We no longer need to worry about overwritting behaviour from jobs we extend from. --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f103447..a8edb64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,13 +18,5 @@ include: LIVE: "true" # Runs the online S3 tests against AWS before_script: - *setup - - curl -sS -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/$CI_HELPER_BRANCH/julia-ci - - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION - script: - - source julia-ci export - - export PATH="$PATH:/usr/local/bin" - eval $(aws-stack-outputs gitlab-ci-runners) # Exports the test bucket to work in - - ./julia-ci test - - ./julia-ci coverage extends: .test_shell_1_3 From 56aef7603ea410f7e4a3e415326072b1eefabbf4 Mon Sep 17 00:00:00 2001 From: Mary Jo Ramos Date: Fri, 3 Apr 2020 11:45:17 -0500 Subject: [PATCH 49/52] Add Memento v1 --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index e504d8d..e9985bc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = "Invenia Technical Computing Corporation" -version = "0.3.0" +version = "0.3.1" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -15,7 +15,7 @@ AWSCore = "0.5.5, 0.6" AWSS3 = "0.6" FilePathsBase = "0.6, 0.7" JLSO = "2" -Memento = "0.10, 0.11, 0.12" +Memento = "0.10, 0.11, 0.12, 0.13, 1" julia = "1" [extras] From 4aa587c8b2f33b65a9f25295a9de29ee735bade3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 Jul 2020 15:59:12 -0500 Subject: [PATCH 50/52] Moved gitlab-ci-helper --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8edb64..30395b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ --- include: - - project: infrastructure/gitlab-ci-helper + - project: invenia/gitlab-ci-helper file: /templates/julia.yml .setup: &setup From 9cb20c4ae023deae996febc83dc9ba4a12863eaa Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 24 Jul 2020 08:50:47 -0500 Subject: [PATCH 51/52] Update to use ci-init --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30395b1..ad10943 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ include: .setup: &setup | - echo "$common_functions" > common && source common + echo "$ci_init" > ci_init && source ci_init && rm ci_init install_awscli install_cloudspy From 0015fb2e40f8fd1ffeff88536ec8fcc9687b3694 Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Tue, 10 Nov 2020 19:13:53 +0000 Subject: [PATCH 52/52] Update to support later releases of FilePathsBase --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index e9985bc..ca29418 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Checkpoints" uuid = "b4a3413d-e481-5afc-88ff-bdfbd6a50dce" authors = "Invenia Technical Computing Corporation" -version = "0.3.1" +version = "0.3.2" [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -13,7 +13,7 @@ Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" [compat] AWSCore = "0.5.5, 0.6" AWSS3 = "0.6" -FilePathsBase = "0.6, 0.7" +FilePathsBase = "0.6, 0.7, 0.8, 0.9" JLSO = "2" Memento = "0.10, 0.11, 0.12, 0.13, 1" julia = "1"