Sync buildog work with main buildbot

.travis.yml

@@ -1,4 +1,4 @@
-language: python
+language: python3
 sudo: required
 notifications:
   email: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2
+FROM python:3
 
 # Install necessary packages
 RUN pip install buildbot requests

master/auto_reload.py

@@ -1,4 +1,9 @@
-reload_scheduler = schedulers.AnyBranchScheduler(name="Buildbot Auto reload", change_filter=util.ChangeFilter(project='JuliaCI/julia-buildbot', branch=['master']), builderNames=["auto_reload"], treeStableTimer=1)
+reload_scheduler = schedulers.AnyBranchScheduler(
+    name="Buildbot Auto reload",
+    change_filter=util.ChangeFilter(project='JuliaCI/julia-buildbot', branch=[BUILDBOT_BRANCH]),
+    builderNames=["auto_reload"],
+    treeStableTimer=1
+)
 c['schedulers'].append(reload_scheduler)
 
 

master/builder_utils.py

@@ -65,6 +65,16 @@ def gen_local_filename(props_obj):
                 return "julia-{shortcommit}-Linux-arm.{os_pkg_ext}".format(**props)
             return "julia-{shortcommit}-Linux-{tar_arch}.{os_pkg_ext}".format(**props)
 
+# Map from property to upload OS name on the JuliaLang S3 bucket
+def get_upload_os_name(props):
+    if is_windows(props):
+        return "winnt"
+    elif is_mac(props):
+        return "mac"
+    elif is_freebsd(props):
+        return "freebsd"
+    else:
+        return "linux"
 
 def gen_upload_filename(props_obj):
     props = props_obj_to_dict(props_obj)
@@ -74,38 +84,53 @@ def gen_upload_filename(props_obj):
         props["os_name_file"] = "win"
     return "julia-{shortcommit}-{os_name_file}{bits}.{os_pkg_ext}".format(**props)
 
-def gen_upload_path(props_obj, namespace=None):
+def gen_upload_path(props_obj, namespace=None, latest=False):
+    # First, pull information out of props_obj
     up_arch = props_obj.getProperty("up_arch")
     majmin = props_obj.getProperty("majmin")
     upload_filename = props_obj.getProperty("upload_filename")
-    os = get_os_name(props_obj)
-    if namespace is None:
-        return "julialangnightlies/bin/%s/%s/%s/%s"%(os, up_arch, majmin, upload_filename)
-    else:
-        return "julialangnightlies/%s/bin/%s/%s/%s/%s"%(namespace, os, up_arch, majmin, upload_filename)
+    assert_build = props_obj.getProperty("assert_build")
 
-def gen_latest_upload_path(props_obj, namespace=None):
-    up_arch = props_obj.getProperty("up_arch")
-    upload_filename = props_obj.getProperty("upload_filename")
-    if upload_filename[:6] == "julia-":
+    # If we're asking for the latest information, 
+    if latest and upload_filename[:6] == "julia-":
         split_name = upload_filename.split("-")
         upload_filename = "julia-latest-%s"%(split_name[2])
-    os = get_os_name(props_obj)
-    if namespace is None:
-        return "julialangnightlies/bin/%s/%s/%s"%(os, up_arch, upload_filename)
-    else:
-        return "julialangnightlies/%s/bin/%s/%s/%s"%(namespace, os, up_arch, upload_filename)
+    os = get_upload_os_name(props_obj)
 
+    # Helper function to prepend things to a value that might be `None`.
+    def none_prepend(p, x):
+        if x is None:
+            return p
+        else:
+            return x + "_" + p
 
 
-def gen_download_url(props_obj, namespace=None):
-    base = 'https://s3.amazonaws.com'
-    return '%s/%s'%(base, gen_upload_path(props_obj, namespace=namespace))
+    # If we're running on the buildog or some other branch, prepend all our namespaces:
+    if BUILDBOT_BRANCH != "master":
+        namespace = none_prepend(BUILDBOT_BRANCH, namespace)
 
-def gen_latest_download_url(props_obj):
-    base = 'https://s3.amazonaws.com'
-    return '%s/%s'%(base, gen_latest_upload_path(props_obj))
+    # If we're running an assert build, put it into an "assert" bucket:
+    if assert_build:
+        namespace = none_prepend("assert", namespace)
 
+    # If we have a namespace, add that on to our URL first
+    url = "julialangnightlies/"
+    if namespace is not None:
+        url += namespace + "/"
+
+    # Next up, OS and Arch.
+    url += os + "/" + up_arch + "/"
+
+    # If we're asking for latest, don't go into majmin
+    if not latest:
+        url += majmin + "/"
+    url += upload_filename
+
+    return url
+
+def gen_download_url(props_obj, namespace=None, latest=False):
+    base = 'https://s3.amazonaws.com'
+    return '%s/%s'%(base, gen_upload_path(props_obj, namespace=namespace, latest=latest))
 
 
 # This is a weird buildbot hack where we really want to parse the output of our
@@ -141,7 +166,7 @@ def render_promotion_command(props_obj):
 @util.renderer
 def render_latest_promotion_command(props_obj):
     src_path = gen_upload_path(props_obj, namespace="pretesting")
-    dst_path = gen_latest_upload_path(props_obj)
+    dst_path = gen_upload_path(props_obj, latest=True)
     return ["/bin/sh", "-c", "aws s3 cp --acl public-read s3://%s s3://%s"%(src_path, dst_path)]
 
 @util.renderer
@@ -161,18 +186,10 @@ def render_pretesting_download_url(props_obj):
 def render_make_app(props_obj):
     props = props_obj_to_dict(props_obj)
 
-    new_way = "make {flags} app".format(**props)
-    old_way = "make {flags} -C contrib/mac/app && mv contrib/mac/app/*.dmg {local_filename}".format(**props)
-
-    # We emit a shell command that attempts to run `make app` (which is the nice
-    # `sf/consistent_distnames` shortcut), and if that fails, it runs the steps
-    # manually, which boil down to `make -C contrib/mac/app` and moving the
-    # result to the top-level, where we can find it.  We can remove this once
-    # 0.6 is no longer being built.
     return [
         "/bin/sh",
         "-c",
-        "~/unlock_keychain.sh && (%s || (%s))"%(new_way, old_way)
+        "~/unlock_keychain.sh && make {flags} app".format(**props)
     ]
 
 def build_download_julia_cmd(props_obj):
@@ -220,14 +237,19 @@ def download_latest_julia(props_obj):
     # Fake `gen_upload_filename()` into giving us something like
     # `julia-latest-linux64.tar.gz` instead of a true shortcommit
     props_obj.setProperty("shortcommit", "latest", "download_latest_julia")
-    upload_filename = gen_upload_filename(props_obj)
-    props_obj.setProperty("upload_filename", upload_filename, "download_latest_julia")
-
-    download_url = gen_latest_download_url(props_obj)
-    props_obj.setProperty("download_url", download_url, "download_latest_julia")
+    props_obj.setProperty(
+        "upload_filename",
+        gen_upload_filename(props_obj),
+        "download_latest_julia",
+    )
+    props_obj.setProperty(
+        "download_url",
+        gen_download_url(props_obj, latest=True),
+        "download_latest_julia",
+    )
     return build_download_julia_cmd(props_obj)
 
 @util.renderer
 def render_tester_name(props_obj):
     props = props_obj_to_dict(props_obj)
-    return "Julia %s Testing"%(props['buildername'].replace('package_', ''))
+    return "Julia CI (%s testing)"%(props['buildername'].replace('package_', ''))

master/coverage.py

@@ -3,21 +3,68 @@
 ###############################################################################
 
 run_coverage_cmd = """
-using CoverageBase, Compat, Compat.Test
+using Pkg
+Pkg.activate("CoverageBase")
+using CoverageBase
 CoverageBase.runtests(CoverageBase.testnames())
 """
 
 analyse_and_submit_cov_cmd = """
-using Coverage, CoverageBase, Compat
-
-cd(joinpath(CoverageBase.julia_top()))
-results = Coverage.process_folder("base")
-if isdefined(CoverageBase.BaseTestRunner, :STDLIBS)
-    for stdlib in CoverageBase.BaseTestRunner.STDLIBS
-        append!(results, Coverage.process_folder("site/v$(VERSION.major).$(VERSION.minor)/$stdlib/src"))
+using Pkg
+Pkg.activate("CoverageBase")
+using Coverage, CoverageBase
+# Process code-coverage files
+results = Coverage.LCOV.readfolder(raw"%(prop:juliadir)s/LCOV")
+  # remove test/ files
+filter!(results) do c
+    !occursin("test/", c.filename)
+end
+  # turn absolute paths into relative, and add base/ to relative paths
+CoverageBase.fixpath!(results)
+results = Coverage.merge_coverage_counts(results)
+  # pretty-print what we have got
+sort!(results, by=c->c.filename)
+for r in results
+    cov, tot = get_summary(r)
+    @info "Got coverage data for $(r.filename): $cov/$tot"
+end
+  # keep only files in stdlib/ and base/
+let prefixes = (joinpath("base", ""),
+                joinpath("stdlib", ""))
+    filter!(results) do c
+        any(p -> startswith(c.filename, p), prefixes)
     end
 end
-
+  # try to find these files, remove those that are not present
+CoverageBase.readsource!(results)
+filter!(results) do c
+    if isempty(c.source)
+        @info "File $(c.filename) not found"
+        false
+    else
+        true
+    end
+end
+  # add in any other files we discover
+  # todo: extend Glob.jl to support these patterns (base/**/*.jl and stdlib/*/src/**/*.jl (except test/))
+  # todo: consider also or instead looking at the Base._included_files list
+allfiles_base = sort!(split(readchomp(Cmd(`find base -name '*.jl'`, dir=CoverageBase.fixabspath(""))), '\n'))
+allfiles_stdlib = sort!(map(x -> "stdlib/" * x[3:end],
+    split(readchomp(Cmd(`find . -name '*.jl' ! -path '*/test/*' ! -path '*/docs/*'`, dir=CoverageBase.fixabspath("stdlib/"))), '\n')))
+allfiles = map(fn -> Coverage.FileCoverage(fn, read(CoverageBase.fixabspath(fn), String), Coverage.FileCoverage[]),
+    [allfiles_base; allfiles_stdlib])
+results = Coverage.merge_coverage_counts(results, allfiles)
+length(results) == length(allfiles) || @warn "Got coverage for an unexpected file:" symdiff=symdiff(map(x -> x.filename, allfiles), map(x -> x.filename, results))
+  # attempt to improve accuracy of the results
+foreach(Coverage.amend_coverage_from_src!, results)
+# Create git_info for codecov
+git_info = Any[
+    :branch => Base.GIT_VERSION_INFO.branch,
+    :commit => Base.GIT_VERSION_INFO.commit,
+    :token => ENV["CODECOV_REPO_TOKEN"],
+    ]
+# Submit to codecov
+Codecov.submit_generic(results; git_info...)
 # Create git_info for Coveralls
 git_info = Dict(
     "branch" => Base.GIT_VERSION_INFO.branch,
@@ -29,28 +76,15 @@
     ],
     "head" => Dict(
         "id" => Base.GIT_VERSION_INFO.commit,
-        "message" => "%(prop:commitmessage)s",
-        "committer_name" => "%(prop:commitname)s",
-        "committer_email" => "%(prop:commitemail)s",
-        "author_name" => "%(prop:authorname)s",
-        "author_email" => "%(prop:authoremail)s",
+        "message" => raw"%(prop:commitmessage)s",
+        "committer_name" => raw"%(prop:commitname)s",
+        "committer_email" => raw"%(prop:commitemail)s",
+        "author_name" => raw"%(prop:authorname)s",
+        "author_email" => raw"%(prop:authoremail)s",
     )
 )
-
 # Submit to Coveralls
-ENV["REPO_TOKEN"] = ENV["COVERALLS_REPO_TOKEN"]
-Coveralls.submit_token(results, git_info)
-delete!(ENV, "REPO_TOKEN")
-
-# Create git_info for codecov
-git_info = Any[
-    :branch => Base.GIT_VERSION_INFO.branch,
-    :commit => Base.GIT_VERSION_INFO.commit,
-    :token => ENV["CODECOV_REPO_TOKEN"],
-    ]
-
-# Submit to codecov
-Codecov.submit_generic(results; git_info...)
+Coveralls.submit_local(results, git_info)
 """
 
 # Steps to download a linux tarball, extract it, run coverage on it, and upload coverage stats
@@ -100,28 +134,26 @@
     ),
 
     # Run Julia, gathering coverage statistics
+    steps.MakeDirectory(dir=util.Interpolate("build/%(prop:juliadir)s/LCOV")),
     steps.ShellCommand(
-        name="Run inlined tests",
-        command=[util.Interpolate("%(prop:juliadir)s/bin/julia"), "--sysimage-native-code=no", "--code-coverage=all", "-e", run_coverage_cmd],
+        name="Run tests",
+        command=[util.Interpolate("%(prop:juliadir)s/bin/julia"),
+                 "--sysimage-native-code=no", util.Interpolate("--code-coverage=%(prop:juliadir)s/LCOV/cov-%%p.info"),
+                 "-e", run_coverage_cmd],
         timeout=3600,
     ),
-    steps.ShellCommand(
-        name="Run non-inlined tests",
-        command=[util.Interpolate("%(prop:juliadir)s/bin/julia"), "--sysimage-native-code=no", "--code-coverage=all", "--inline=no", "-e", run_coverage_cmd],
-        timeout=7200,
-    ),
     #submit the results!
     steps.ShellCommand(
         name="Gather test results and Submit",
         command=[util.Interpolate("%(prop:juliadir)s/bin/julia"), "-e", util.Interpolate(analyse_and_submit_cov_cmd)],
-        env={'COVERALLS_REPO_TOKEN':COVERALLS_REPO_TOKEN, 'CODECOV_REPO_TOKEN':CODECOV_REPO_TOKEN},
+        env={'COVERALLS_TOKEN':COVERALLS_REPO_TOKEN, 'CODECOV_REPO_TOKEN':CODECOV_REPO_TOKEN},
         logEnviron=False,
     ),
 ])
 
 
 # Add a dependent scheduler for running coverage after we build tarballs
-julia_coverage_builders = ["coverage_ubuntu16_04-x64"]
+julia_coverage_builders = ["coverage_linux64"]
 julia_coverage_scheduler = schedulers.Triggerable(name="Julia Coverage Testing", builderNames=julia_coverage_builders)
 c['schedulers'].append(julia_coverage_scheduler)
 
@@ -151,8 +183,8 @@
 
 # Add coverage builders
 c['builders'].append(util.BuilderConfig(
-    name="coverage_ubuntu16_04-x64",
-    workernames=["tabularasa_ubuntu16_04-x64"],
+    name="coverage_linux64",
+    workernames=["tabularasa_" + x for x in builder_mapping["linux64"]],
     tags=["Coverage"],
     factory=julia_coverage_factory
 ))

master/freebsd_ci.py

@@ -4,9 +4,6 @@
 
 freebsdci_factory = util.BuildFactory()
 freebsdci_factory.addSteps([
-    steps.BSDSysInfo(),
-    steps.BSDSetMakeVar(['make_jobs'], ['MAKE_JOBS_NUMBER']),
-
     steps.ShellCommand(
         name='cleanup stdlib',
         command=['rm', '-rvf', 'stdlib']