From 94e002cadd4bb9c21c2b8245b8c101cbb0ea5bbb Mon Sep 17 00:00:00 2001 From: David Widmann Date: Tue, 10 Nov 2020 19:42:40 +0100 Subject: [PATCH 1/4] Use libjulia for Libtask --- L/Libtask/build_dylib.sh | 62 ------------------------ L/Libtask/build_tarballs.jl | 66 ++++++++++++++------------ L/Libtask/bundled/LICENSE.md | 21 +++++++++ L/Libtask/bundled/task.c | 91 ++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 92 deletions(-) delete mode 100644 L/Libtask/build_dylib.sh create mode 100644 L/Libtask/bundled/LICENSE.md create mode 100644 L/Libtask/bundled/task.c diff --git a/L/Libtask/build_dylib.sh b/L/Libtask/build_dylib.sh deleted file mode 100644 index 3fd0089ddd0..00000000000 --- a/L/Libtask/build_dylib.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# target=x86_64-linux-gnu -cd $WORKSPACE/srcdir - -RELEASES=( - arm-linux-gnueabihf@v1_0@https://julialang-s3.julialang.org/bin/linux/armv7l/1.0/julia-1.0.0-linux-armv7l.tar.gz - arm-linux-gnueabihf@v1_2@https://julialang-s3.julialang.org/bin/linux/armv7l/1.2/julia-1.2.0-linux-armv7l.tar.gz - arm-linux-gnueabihf@v1_3@https://julialang-s3.julialang.org/bin/linux/armv7l/1.3/julia-1.3.0-linux-armv7l.tar.gz - aarch64-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/aarch64/1.0/julia-1.0.4-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/aarch64/1.1/julia-1.1.1-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/aarch64/1.2/julia-1.2.0-linux-aarch64.tar.gz - aarch64-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/aarch64/1.3/julia-1.3.0-linux-aarch64.tar.gz - i686-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/x86/1.0/julia-1.0.0-linux-i686.tar.gz - i686-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/x86/1.1/julia-1.1.0-linux-i686.tar.gz - i686-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/x86/1.2/julia-1.2.0-linux-i686.tar.gz - i686-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/x86/1.3/julia-1.3.0-linux-i686.tar.gz - x86_64-linux-gnu@v1_0@https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_1@https://julialang-s3.julialang.org/bin/linux/x64/1.1/julia-1.1.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_2@https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz - x86_64-linux-gnu@v1_3@https://julialang-s3.julialang.org/bin/linux/x64/1.3/julia-1.3.0-linux-x86_64.tar.gz - powerpc64le-linux-gnu@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-linux-ppc64le.tar.gz - powerpc64le-linux-gnu@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-linux-ppc64le.tar.gz - x86_64-w64-mingw32@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-win64.tar.gz - x86_64-w64-mingw32@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-win64.tar.gz - x86_64-w64-mingw32@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-win64.tar.gz - x86_64-w64-mingw32@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-win64.tar.gz - i686-w64-mingw32@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-win32.tar.gz - i686-w64-mingw32@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-win32.tar.gz - i686-w64-mingw32@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-win32.tar.gz - i686-w64-mingw32@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-win32.tar.gz - x86_64-apple-darwin14@v1_0@http://cxan.kdr2.com/julia/releases/julia-1.0.0-mac64.tar.gz - x86_64-apple-darwin14@v1_1@http://cxan.kdr2.com/julia/releases/julia-1.1.0-mac64.tar.gz - x86_64-apple-darwin14@v1_2@http://cxan.kdr2.com/julia/releases/julia-1.2.0-mac64.tar.gz - x86_64-apple-darwin14@v1_3@http://cxan.kdr2.com/julia/releases/julia-1.3.0-mac64.tar.gz -) - -for RELEASE in ${RELEASES[@]}; do - REL_TARGET=$(echo $RELEASE | cut -d@ -f1) - if [ $target != $REL_TARGET ]; then - continue - fi - rm -f *.tar.gz - rm -fr julia - wget $(echo $RELEASE | cut -d@ -f3) - tar xzvf *tar.gz - rm -f *.tar.gz - mv julia* julia - export JULIA_VERSION=$(echo $RELEASE | cut -d@ -f2) - if [ $target = x86_64-apple-darwin14 ]; then - export JULIA_HOME=$(pwd)/julia/Contents/Resources/julia - else - export JULIA_HOME=$(pwd)/julia - # export JULIA_HOME=/home/kdr2/programs/julia-1.1.0 - fi - make -C Libtask.jl/deps -done - -# https://juliapackaging.github.io/BinaryBuilder.jl/dev/build_tips/#Installing-the-license-file-1 -install_license Libtask.jl/LICENSE diff --git a/L/Libtask/build_tarballs.jl b/L/Libtask/build_tarballs.jl index 9a6d881b18e..372f7710729 100644 --- a/L/Libtask/build_tarballs.jl +++ b/L/Libtask/build_tarballs.jl @@ -1,52 +1,58 @@ # Note that this script can accept some limited command-line arguments, run # `julia build_tarballs.jl --help` to see a usage message. -using Pkg -using BinaryBuilder +using BinaryBuilder, Pkg -name = "Libtask" -version = v"0.3.2" -commit_id = "fbe338053f402d76524d0a01f3796dd7da90b781" +julia_version = v"1.3.1" -# see https://github.com/JuliaPackaging/BinaryBuilder.jl/issues/336 -# ENV["CI_COMMIT_TAG"] = ENV["TRAVIS_TAG"] = "v" * string(version) +name = "Libtask" +version = VersionNumber("0.4.0-julia.$(julia_version.major).$(julia_version.minor)") +# Collection of sources required to build Libtask sources = [ - "https://github.com/TuringLang/Libtask.jl.git" => commit_id, + DirectorySource("./bundled"), ] # Bash recipe for building across all platforms -script_file = joinpath(@__DIR__, "build_dylib.sh") -if !isfile(script_file) # when run generate_buildjl.jl - script_file = joinpath(@__DIR__, "L/Libtask/build_dylib.sh") -end -script = read(script_file, String) +script = raw""" +# create output directory +mkdir -p "${libdir}" + +# compile library +cd "${WORKSPACE}/srcdir/" +export CFLAGS="-I${includedir} -I${includedir}/julia -O2 -shared -std=gnu99 -fPIC" +if [[ "${target}" == *-mingw* ]]; then + export LDFLAGS="-ljulia -lopenlibm" +else + export LDFLAGS="-ljulia" +fi +$CC $CFLAGS $LDFLAGS task.c -o "${libdir}/libtask_julia.${dlext}" + +# install license +install_license LICENSE.md +""" # These are the platforms we will build for by default, unless further # platforms are passed in on the command line -platforms = [ - Platform("i686", "linux"; libc="glibc"), - Platform("x86_64", "linux"; libc="glibc"), - Platform("powerpc64le", "linux"; libc="glibc"), - # Platform("armv7l", "linux"; libc="glibc"), - Platform("aarch64", "linux"), - Platform("x86_64", "macos"), - Platform("i686", "windows"), - Platform("x86_64", "windows") -] +platforms = supported_platforms() + +# skip i686 musl builds (not supported by libjulia_jll) +filter!(p -> !(Sys.islinux(p) && libc(p) == "musl" && arch(p) == "i686"), platforms) + +# skip PowerPC builds in Julia 1.3 (not supported by libjulia_jll) +if julia_version < v"1.4" + filter!(p -> !(Sys.islinux(p) && arch(p) == "powerpc64le"), platforms) +end # The products that we will ensure are always built products = [ - LibraryProduct("libtask_v1_0", :libtask_v1_0) - LibraryProduct("libtask_v1_1", :libtask_v1_1) - LibraryProduct("libtask_v1_2", :libtask_v1_2) - LibraryProduct("libtask_v1_3", :libtask_v1_3) + LibraryProduct("libtask_julia", :libtask_julia), ] # Dependencies that must be installed before this package can be built dependencies = [ + BuildDependency(PackageSpec(name="libjulia_jll", version=julia_version)), ] # Build the tarballs, and possibly a `build.jl` as well. -# build_file = "products/build_$(name).v$(version).jl" -build_tarballs(ARGS, name, version, sources, - script, platforms, products, dependencies) +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; + julia_compat = "~$(julia_version.major).$(julia_version.minor)") diff --git a/L/Libtask/bundled/LICENSE.md b/L/Libtask/bundled/LICENSE.md new file mode 100644 index 00000000000..2b0ea03ff7f --- /dev/null +++ b/L/Libtask/bundled/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 The Turing Language + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/L/Libtask/bundled/task.c b/L/Libtask/bundled/task.c new file mode 100644 index 00000000000..7eb16eb95f7 --- /dev/null +++ b/L/Libtask/bundled/task.c @@ -0,0 +1,91 @@ +/* + task.c + task copying (aka continuation) for lightweight processes (symmetric coroutines) + */ + +#include "julia.h" + +// Workaround for JuliaLang/julia#32812 +jl_task_t *vanilla_get_current_task(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return (jl_task_t*)ptls->current_task; +} + + +jl_task_t *jl_enable_stack_copying(jl_task_t *t) +{ +#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 + if (!t->copy_stack) { + jl_ptls_t ptls = jl_get_ptls_states(); + t->copy_stack = 1; + t->bufsz = 0; + memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); + } +#endif + return t; +} + +jl_task_t *jl_clone_task(jl_task_t *t) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + jl_task_t *newt = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); // More efficient + //jl_task_t *newt = (jl_task_t*)jl_new_task(t->start, t->ssize); // Less efficient + memset(newt, 0, sizeof(jl_task_t)); + jl_set_typeof(newt, jl_task_type); + newt->stkbuf = NULL; + newt->gcstack = NULL; + JL_GC_PUSH1(&newt); + + newt->state = t->state; + newt->start = t->start; + newt->tls = jl_nothing; + newt->logstate = ptls->current_task->logstate; + newt->result = jl_nothing; + newt->donenotify = jl_nothing; + newt->exception = jl_nothing; + newt->backtrace = jl_nothing; + newt->eh = t->eh; + newt->gcstack = t->gcstack; + newt->tid = t->tid; // TODO: need testing + newt->started = t->started; // TODO: need testing + + +#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 + newt->copy_stack = t->copy_stack; + memcpy((void*)newt->ctx.uc_mcontext, (void*)t->ctx.uc_mcontext, sizeof(jl_jmp_buf)); +#if JULIA_VERSION_MINOR >= 2 + newt->queue = t->queue; + newt->sticky = t->sticky; +#endif +#else + newt->parent = ptls->current_task; + newt->current_module = t->current_module; + newt->ssize = t->ssize; // size of saved piece + memcpy((void*)newt->ctx, (void*)t->ctx, sizeof(jl_jmp_buf)); +#endif + + if (t->stkbuf) { +#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) + newt->ssize = t->ssize; // size of saved piece +#endif + // newt->stkbuf = allocb(t->bufsz); + // newt->bufsz = t->bufsz; + // memcpy(newt->stkbuf, t->stkbuf, t->bufsz); + // workaround, newt and t will get new stkbuf when savestack is called. + t->bufsz = 0; + newt->bufsz = 0; + newt->stkbuf = t->stkbuf; + } else { +#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) + newt->ssize = 0; +#endif + newt->bufsz = 0; + newt->stkbuf = NULL; + } + + JL_GC_POP(); + jl_gc_wb_back(newt); + + return newt; +} From d37e7afb50e0e68b156214eb41989a56dff32908 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Thu, 12 Nov 2020 17:43:17 +0100 Subject: [PATCH 2/4] Apply all flags from Libtask.jl --- L/Libtask/build_tarballs.jl | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/L/Libtask/build_tarballs.jl b/L/Libtask/build_tarballs.jl index 372f7710729..ea38d3eb3a5 100644 --- a/L/Libtask/build_tarballs.jl +++ b/L/Libtask/build_tarballs.jl @@ -5,7 +5,7 @@ using BinaryBuilder, Pkg julia_version = v"1.3.1" name = "Libtask" -version = VersionNumber("0.4.0-julia.$(julia_version.major).$(julia_version.minor)") +version = VersionNumber("1.$(julia_version.minor).0") # Collection of sources required to build Libtask sources = [ @@ -19,12 +19,22 @@ mkdir -p "${libdir}" # compile library cd "${WORKSPACE}/srcdir/" -export CFLAGS="-I${includedir} -I${includedir}/julia -O2 -shared -std=gnu99 -fPIC" + +CFLAGS="-I${includedir} -I${includedir}/julia -O2 -shared -std=gnu99 -fPIC" +if [[ "${target}" == i686-* ]]; then + CFLAGS="${CFLAGS} -march=pentium4" +fi +if [[ "${target}" == *-mingw* ]]; then + CFLAGS="${CFLAGS} -Wl,--export-all-symbols" +elif [[ "${target}" == *-linux-* ]]; then + CFLAGS="${CFLAGS} -Wl,--export-dynamic" +fi + +LDFLAGS="-L${libdir} -ljulia" if [[ "${target}" == *-mingw* ]]; then - export LDFLAGS="-ljulia -lopenlibm" -else - export LDFLAGS="-ljulia" + LDFLAGS="${LDFLAGS} -lopenlibm" fi + $CC $CFLAGS $LDFLAGS task.c -o "${libdir}/libtask_julia.${dlext}" # install license @@ -55,4 +65,5 @@ dependencies = [ # Build the tarballs, and possibly a `build.jl` as well. build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; - julia_compat = "~$(julia_version.major).$(julia_version.minor)") + julia_compat = "~$(julia_version.major).$(julia_version.minor)", + lock_microarchitecture = false) From fd1f770cc4b153f7eb980e7fef70c4e7a9d259fb Mon Sep 17 00:00:00 2001 From: David Widmann Date: Sat, 14 Nov 2020 21:39:43 +0100 Subject: [PATCH 3/4] Remove conditionals --- L/Libtask/bundled/task.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/L/Libtask/bundled/task.c b/L/Libtask/bundled/task.c index 7eb16eb95f7..d5e9160647d 100644 --- a/L/Libtask/bundled/task.c +++ b/L/Libtask/bundled/task.c @@ -15,14 +15,12 @@ jl_task_t *vanilla_get_current_task(void) jl_task_t *jl_enable_stack_copying(jl_task_t *t) { -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 if (!t->copy_stack) { jl_ptls_t ptls = jl_get_ptls_states(); t->copy_stack = 1; t->bufsz = 0; memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); } -#endif return t; } @@ -51,24 +49,12 @@ jl_task_t *jl_clone_task(jl_task_t *t) newt->started = t->started; // TODO: need testing -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1 newt->copy_stack = t->copy_stack; memcpy((void*)newt->ctx.uc_mcontext, (void*)t->ctx.uc_mcontext, sizeof(jl_jmp_buf)); -#if JULIA_VERSION_MINOR >= 2 newt->queue = t->queue; newt->sticky = t->sticky; -#endif -#else - newt->parent = ptls->current_task; - newt->current_module = t->current_module; - newt->ssize = t->ssize; // size of saved piece - memcpy((void*)newt->ctx, (void*)t->ctx, sizeof(jl_jmp_buf)); -#endif if (t->stkbuf) { -#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) - newt->ssize = t->ssize; // size of saved piece -#endif // newt->stkbuf = allocb(t->bufsz); // newt->bufsz = t->bufsz; // memcpy(newt->stkbuf, t->stkbuf, t->bufsz); @@ -77,9 +63,6 @@ jl_task_t *jl_clone_task(jl_task_t *t) newt->bufsz = 0; newt->stkbuf = t->stkbuf; } else { -#if !(JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR >= 1) - newt->ssize = 0; -#endif newt->bufsz = 0; newt->stkbuf = NULL; } From 4579fc79bb3d332f06d44a6276c4e679c22a1ac2 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Sat, 14 Nov 2020 21:52:57 +0100 Subject: [PATCH 4/4] Set version number manually --- L/Libtask/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/L/Libtask/build_tarballs.jl b/L/Libtask/build_tarballs.jl index ea38d3eb3a5..bb8a7a5f2f7 100644 --- a/L/Libtask/build_tarballs.jl +++ b/L/Libtask/build_tarballs.jl @@ -5,7 +5,7 @@ using BinaryBuilder, Pkg julia_version = v"1.3.1" name = "Libtask" -version = VersionNumber("1.$(julia_version.minor).0") +version = v"0.4.0" # Collection of sources required to build Libtask sources = [