More windows CI debugging

.github/workflows/test.yml

@@ -7,6 +7,9 @@ on:
  branches:
  - master
  tags: '*'
+defaults:
+ run:
+ shell: bash
 jobs:
  test:
  runs-on: ${{ matrix.os }}
@@ -15,32 +18,64 @@ jobs:
  matrix:
  os:
  - ubuntu-latest
- - macOS-latest
+ # - macOS-latest
  - windows-latest
  julia-arch:
  - 'x64'
- - 'x86'
+ # - 'x86'
  pkg-server:
  - ""
  - "pkg.julialang.org"
  julia-version:
  # - '1.6'
  - 'nightly'
- exclude:
- - os: macOS-latest
- julia-arch: x86
+ # exclude:
+ #  - os: macOS-latest
+ #  julia-arch: x86
  steps:
- - uses: actions/checkout@v1.0.0
- - uses: julia-actions/setup-julia@latest
+ - name: Set git to use LF
+ if: matrix.os == 'windows-latest'
+ run: |
+ git config --global core.autocrlf false
+ git config --global core.eol lf
+ - name: Determine filesystem of drives on windows
+ if: matrix.os == 'windows-latest'
+ run: get-volume
+ shell: powershell
+ - uses: actions/checkout@v2
+
+ # - uses: julia-actions/setup-julia@latest
+ # with:
+ # version: ${{ matrix.julia-version }}
+
+ - name: Install Julia from URL (windows)
+ if: matrix.os == 'windows-latest'
+ uses: julia-actions/install-julia-from-url@main
+ with:
+ # url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/winnt/x64/1.7/julia-a6df655fea-win64.tar.gz
+ url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/winnt/x64/1.7/julia-850d42a7ec-win64.tar.gz #statstruct
+ - name: Install Julia from URL (ubuntu)
+ if: matrix.os == 'ubuntu-latest'
+ uses: julia-actions/install-julia-from-url@main
  with:
- version: ${{ matrix.julia-version }}
+ # url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/linux/x64/1.7/julia-a6df655fea-linux64.tar.gz
+ url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/linux/x64/1.7/julia-850d42a7ec-linux64.tar.gz #statstruct
+ # - name: Install Julia from URL (macos)
+ # if: matrix.os == 'macOS-latest'
+ # uses: julia-actions/install-julia-from-url@main
+ # with:
+ # url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/mac/x64/1.7/julia-a6df655fea-mac64.tar.gz
+ # url: https://s3.amazonaws.com/julialangnightlies/assert_pretesting/linux/x64/1.7/julia-850d42a7ec-linux64.tar.gz #statstruct
+ - name: Show versioninfo()
+ run: julia -e "using InteractiveUtils; versioninfo()"
+
  - name: Fix TEMP on windows
  if: matrix.os == 'windows-latest'
  run: |
  # See https://github.com/actions/virtual-environments/issues/712
- echo "TMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV
- echo "TEMP=$env:USERPROFILE\AppData\Local\Temp" >> $env:GITHUB_ENV
- - name: Run tests
+ echo "TMP=${USERPROFILE}\AppData\Local\Temp" >> ${GITHUB_ENV}
+ echo "TEMP=${USERPROFILE}\AppData\Local\Temp" >> ${GITHUB_ENV}
+ - name: Prepare local Pkg for tests
  run: |
  julia test/pkg-uuid.jl
  julia --project --color=yes -e 'write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"54cfe95a-1eb2-52ea-b672-e2afdf69b78f\"\n"));'

appveyor.yml

@@ -29,6 +29,7 @@ notifications:
  on_build_status_changed: false
 
 install:
+ - ps: get-volume
  - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1"))
 
 build_script:

src/GitTools.jl

@@ -257,31 +257,43 @@ end
 
 Calculate the git tree hash of a given path.
 """
-function tree_hash(::Type{HashType}, root::AbstractString) where HashType
+function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO,Nothing} = nothing, indent::Int=0) where HashType
  entries = Tuple{String, Vector{UInt8}, GitMode}[]
- for f in readdir(root)
+ for f in sort(readdir(root; join=true); by = f -> isdir(f) ? f*"/" : f)
  # Skip `.git` directories
  if f == ".git"
  continue
  end
 
- filepath = abspath(root, f)
+ filepath = abspath(f)
  mode = gitmode(filepath)
  if mode == mode_dir
  # If this directory contains no files, then skip it
  contains_files(filepath) || continue
 
  # Otherwise, hash it up!
- hash = tree_hash(HashType, filepath)
+ child_stream = nothing
+ if debug_out !== nothing
+ child_stream = IOBuffer()
+ end
+ hash = tree_hash(HashType, filepath; debug_out=child_stream, indent=indent+1)
+ if debug_out !== nothing
+ indent_str = "| "^indent
+ println(debug_out, "$(indent_str)+ [D] $(basename(filepath)) - $(bytes2hex(hash))")
+ print(debug_out, String(take!(child_stream)))
+ println(debug_out, indent_str)
+ end
  else
  hash = blob_hash(HashType, filepath)
+ if debug_out !== nothing
+ indent_str = "| "^indent
+ mode_str = mode == mode_normal ? "F" : "X"
+ println(debug_out, "$(indent_str)[$(mode_str)] $(basename(filepath)) - $(bytes2hex(hash))")
+ end
  end
- push!(entries, (f, hash, mode))
+ push!(entries, (basename(filepath), hash, mode))
  end
 
- # Sort entries by name (with trailing slashes for directories)
- sort!(entries, by = ((name, hash, mode),) -> mode == mode_dir ? name*"/" : name)
-
  content_size = 0
  for (n, h, m) in entries
  content_size += ndigits(UInt32(m); base=8) + 1 + sizeof(n) + 1 + sizeof(h)
@@ -296,7 +308,7 @@ function tree_hash(::Type{HashType}, root::AbstractString) where HashType
  end
  return SHA.digest!(ctx)
 end
-tree_hash(root::AbstractString) = tree_hash(SHA.SHA1_CTX, root)
+tree_hash(root::AbstractString; debug_out::Union{IO,Nothing} = nothing) = tree_hash(SHA.SHA1_CTX, root; debug_out)
 
 function check_valid_HEAD(repo)
  try LibGit2.head(repo)

test/artifacts.jl

@@ -138,6 +138,7 @@ end
  @test iszero(filemode(joinpath(artifact_dir, file_link)) & 0o222)
  @test !iszero(filemode(joinpath(artifact_dir, subdir)) & 0o222)
  @test !iszero(filemode(joinpath(artifact_dir, dir_link)) & 0o222)
+
  # Make sure we can delete the artifact directory without having
  # to manually change permissions
  rm(artifact_dir; recursive=true)

test/new.jl

@@ -16,6 +16,16 @@ unicode_uuid = UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5")
 unregistered_uuid = UUID("dcb67f36-efa0-11e8-0cef-2fc465ed98ae")
 simple_package_uuid = UUID("fc6b7c0f-8a2f-4256-bbf4-8c72c30df5be")
 
+
+tree_hash(root::AbstractString; kwargs...) = bytes2hex(@inferred Pkg.GitTools.tree_hash(root; kwargs...))
+
+#
+# # Sanity checks
+#
+pkg_path = joinpath(dirname(@__DIR__))
+@info "Pkg.jl path: $pkg_path"
+@info "Tree hash of Pkg.jl repo: $(tree_hash(pkg_path))"
+
 #
 # # Depot Changes
 #
@@ -1781,7 +1791,40 @@ end
 
  # Build log location
  isolate(loaded_depot=true) do; mktempdir() do tmp
+ @info("Dumping permission inheritance chain for test_packages")
+ r = joinpath(@__DIR__, "test_packages")
+ while dirname(r) != r
+ run(`icacls $r`)
+ r = dirname(r)
+ end
+
+ # debugging
+ r = joinpath(@__DIR__, "test_packages", "FailBuild")
+ @info "File tree for source package $r"
+ println(r, " ", "0o$(string(filemode(stat(r)), base = 8, pad = 6))")
+ @info "Checking tree hash of $r"
+ @show tree_hash(r; debug_out=stdout)
+
+ for (root, dirs, files) in walkdir(r)
+ for dir in dirs
+ println(joinpath(root, dir), " ", "0o$(string(filemode(stat(joinpath(root, dir))), base = 8, pad = 6))")
+ @show stat(joinpath(root, dir))
+ end
+ for file in files
+ fp = joinpath(root, file)
+ println(fp, " ", filesize(fp), " 0o$(string(filemode(stat(fp)), base = 8, pad = 6))")
+ @show stat(fp)
+ @show Sys.isexecutable(fp)
+ @show run(`icacls $fp`)
+ @show read(fp, String)
+ @show sha256(read(fp, String))
+ end
+ end
+
  path = git_init_package(tmp, joinpath(@__DIR__, "test_packages", "FailBuild"))
+ @show tmp, joinpath(@__DIR__, "test_packages", "FailBuild"), path
+ @show filemode(joinpath(path, "src", "FailBuild.jl")), filesize(joinpath(path, "src", "FailBuild.jl"))
+ @show stat(joinpath(path, "deps", "build.jl")), filesize(joinpath(path, "deps", "build.jl"))
  # Log file in the directory when it is deved
  Pkg.develop(path=path; io=devnull)
  log_file_dev = joinpath(path, "deps", "build.log")
@@ -1794,8 +1837,33 @@ end
  log_file_add = joinpath(path, "deps", "build.log")
  @test_throws PkgError Pkg.add(path=path; io=devnull)
  @test !isfile(joinpath(Base.find_package("FailBuild"), "..", "..", "deps", "build.log"))
+
+ # Should be "f99d57aad0e5eb2434491b47bac92bb88d463001", but differs on windows
+ treehash = tree_hash(dirname(dirname(Base.find_package("FailBuild"))))
+ @info "Tree hash for $(dirname(dirname(Base.find_package("FailBuild")))) should be f99d57aad0e5eb2434491b47bac92bb88d463001. Calculated: $treehash"
+
  log_file_add = joinpath(DEPOT_PATH[1], "scratchspaces",
  "44cfe95a-1eb2-52ea-b672-e2afdf69b78f", "f99d57aad0e5eb2434491b47bac92bb88d463001", "build.log")
+
+ # debugging
+ r = dirname(dirname(Base.find_package("FailBuild")))
+ @info "File tree for package add-ed into depot $r"
+ println(r, " ", "0o$(string(filemode(stat(r)), base = 8, pad = 6))")
+ @info "Checking tree hash of $r"
+ @show tree_hash(r; debug_out=stdout)
+ for (root, dirs, files) in walkdir(r)
+ for dir in dirs
+ println(joinpath(root, dir), " ", "0o$(string(filemode(stat(joinpath(root, dir))), base = 8, pad = 6))")
+ @show stat(joinpath(root, dir))
+ end
+ for file in files
+ fp = joinpath(root, file)
+ println(fp, " ", filesize(fp), " 0o$(string(filemode(stat(fp)), base = 8, pad = 6))")
+ @show stat(fp)
+ @show read(fp, String)
+ end
+ end
+
  @test isfile(log_file_add)
  @test occursin("oops", read(log_file_add, String))
  end end
@@ -2306,8 +2374,6 @@ end
  end
 end
 
-tree_hash(root::AbstractString) = bytes2hex(@inferred Pkg.GitTools.tree_hash(root))
-
 @testset "git tree hash computation" begin
  mktempdir() do dir
  # test "well known" empty tree hash

test/utils.jl

@@ -150,7 +150,7 @@ function write_build(path, content)
 end
 
 function with_current_env(f)
- prev_active = Base.ACTIVE_PROJECT[] 
+ prev_active = Base.ACTIVE_PROJECT[]
  Pkg.activate(".")
  try
  f()
@@ -160,7 +160,7 @@ function with_current_env(f)
 end
 
 function with_temp_env(f, env_name::AbstractString="Dummy"; rm=true)
- prev_active = Base.ACTIVE_PROJECT[] 
+ prev_active = Base.ACTIVE_PROJECT[]
  env_path = joinpath(mktempdir(), env_name)
  Pkg.generate(env_path)
  Pkg.activate(env_path)
@@ -178,7 +178,7 @@ function with_temp_env(f, env_name::AbstractString="Dummy"; rm=true)
 end
 
 function with_pkg_env(fn::Function, path::AbstractString="."; change_dir=false)
- prev_active = Base.ACTIVE_PROJECT[] 
+ prev_active = Base.ACTIVE_PROJECT[]
  Pkg.activate(path)
  try
  if change_dir