From 116b1859c16a611c8ca1153c7feb41898510d817 Mon Sep 17 00:00:00 2001 From: Michiel Borkent Date: Sat, 23 May 2020 14:10:02 +0200 Subject: [PATCH] [#332] support GraalVM java11 --- .circleci/config.yml | 28 +- README.md | 17 +- project.clj | 4 +- reflector/.gitignore | 1 + .../{project.clj => project.clj.template} | 4 +- reflector/script/deploy_java11 | 5 + reflector/script/deploy_java8 | 5 + reflector/script/update-project.clj | 26 + reflector/src-java11/sci/impl/Reflector.java | 637 ++++++++++++++++++ .../sci/impl/Reflector.java | 0 script/compile | 12 +- 11 files changed, 720 insertions(+), 19 deletions(-) create mode 100644 reflector/.gitignore rename reflector/{project.clj => project.clj.template} (75%) create mode 100755 reflector/script/deploy_java11 create mode 100755 reflector/script/deploy_java8 create mode 100755 reflector/script/update-project.clj create mode 100644 reflector/src-java11/sci/impl/Reflector.java rename reflector/{src-java => src-java8}/sci/impl/Reflector.java (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 649a3117..5a20f7be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: jvm: docker: # specify the version you desire here - - image: circleci/clojure:lein-2.8.1 + - image: circleci/clojure:openjdk-11-lein-2.8.1 working_directory: ~/repo environment: LEIN_ROOT: "true" @@ -35,7 +35,7 @@ jobs: node: docker: # specify the version you desire here - - image: circleci/clojure:lein-2.8.1-node + - image: circleci/clojure:openjdk-11-lein-2.8.1-node working_directory: ~/repo environment: LEIN_ROOT: "true" @@ -62,11 +62,11 @@ jobs: key: v1-dependencies-{{ checksum "project.clj" }} linux: docker: - - image: circleci/clojure:lein-2.8.1 + - image: circleci/clojure:openjdk-11-lein-2.8.1 working_directory: ~/repo environment: LEIN_ROOT: "true" - GRAALVM_HOME: /home/circleci/graalvm-ce-19.1.1 + GRAALVM_HOME: /home/circleci/graalvm-ce-java11-20.1.0 # SCI_TEST_ENV: native steps: - checkout @@ -82,9 +82,9 @@ jobs: name: Download GraalVM command: | cd ~ - if ! [ -d graalvm-ce-19.1.1 ]; then - curl -O -sL https://github.com/oracle/graal/releases/download/vm-19.1.1/graalvm-ce-linux-amd64-19.1.1.tar.gz - tar xzf graalvm-ce-linux-amd64-19.1.1.tar.gz + if ! [ -d graalvm-ce-java11-20.1.0 ]; then + curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.1.0/graalvm-ce-java11-linux-amd64-20.1.0.tar.gz + tar xzf graalvm-ce-java11-linux-amd64-20.1.0.tar.gz fi - run: name: Build binary @@ -98,7 +98,7 @@ jobs: - save_cache: paths: - ~/.m2 - - ~/graalvm-ce-19.1.1 + - ~/graalvm-ce-java11-20.1.0 key: linux-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: path: /tmp/release @@ -107,7 +107,8 @@ jobs: macos: xcode: "9.0" environment: - GRAALVM_HOME: /Users/distiller/graalvm-ce-19.1.1/Contents/Home + GRAALVM_HOME: /Users/distiller/graalvm-ce-java11-20.1.0/Contents/Home + JAVA_HOME: /Users/distiller/graalvm-ce-java11-20.1.0/Contents/Home SCI_TEST_ENV: native steps: - checkout @@ -127,9 +128,9 @@ jobs: command: | cd ~ ls -la - if ! [ -d graalvm-ce-19.1.1 ]; then - curl -O -sL https://github.com/oracle/graal/releases/download/vm-19.1.1/graalvm-ce-darwin-amd64-19.1.1.tar.gz - tar xzf graalvm-ce-darwin-amd64-19.1.1.tar.gz + if ! [ -d graalvm-ce-java11-20.1.0 ]; then + curl -O -sL curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.1.0/graalvm-ce-java11-darwin-amd64-20.1.0.tar.gz + tar xzf graalvm-ce-java11-darwin-amd64-20.1.0.tar.gz fi - run: name: Build binary @@ -139,11 +140,12 @@ jobs: - run: name: Run tests command: | + export PATH=$GRAALVM_HOME/bin:$PATH script/test/native - save_cache: paths: - ~/.m2 - - ~/graalvm-ce-19.1.1 + - ~/graalvm-ce-java11-20.1.0 key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: path: /tmp/release diff --git a/README.md b/README.md index c4e43600..c426bc97 100644 --- a/README.md +++ b/README.md @@ -406,7 +406,9 @@ More examples of what is currently possible can be found at If you miss something, feel free to post an issue. -## Caveats +## GraalVM + +### Random numbers To make the `rand-*` functions behave well when compiling to a GraalVM native binary, use this setting: @@ -414,6 +416,19 @@ To make the `rand-*` functions behave well when compiling to a GraalVM native bi --initialize-at-run-time=java.lang.Math\$RandomNumberGeneratorHolder ``` +### Java 11 + +To use sci with GraalVM java11 override the dependency +`[borkdude/sci.impl.reflector "0.0.1"]` to `[borkdude/sci.impl.reflector +"0.0.1-java11]` in your `project.clj` or `deps.edn`. + +Also you'll likely need a fix for `clojure.lang.Reflector`: + +See +[clj-graal-docs](https://github.com/lread/clj-graal-docs#jdk11-and-clojurelangreflector) +and +[clj-reflector-graal-java11-fix](https://github.com/borkdude/clj-reflector-graal-java11-fix). + ## Use from JavaScript Sci is available on NPM: diff --git a/project.clj b/project.clj index a8fd14ce..cf2285bd 100644 --- a/project.clj +++ b/project.clj @@ -10,8 +10,8 @@ :url "http://opensource.org/licenses/eclipse-1.0.php"} :source-paths ["src"] :dependencies [[org.clojure/clojure "1.9.0"] + [borkdude/sci.impl.reflector "0.0.1"] ;; use 0.0.1-jdk11 with JDK 11 [borkdude/edamame "0.0.11-alpha.12"] - [borkdude/sci.impl.reflector "0.0.1"] [org.clojure/tools.reader "1.3.2"]] :plugins [[lein-codox "0.10.7"]] :profiles {:clojure-1.9.0 {:dependencies [[org.clojure/clojure "1.9.0"]]} @@ -26,6 +26,8 @@ "-Dclojure.spec.skip-macros=true"] :aot :all :main sci.impl.main} + :native-image {:dependencies [[borkdude/sci.impl.reflector "0.0.1-jdk11"] + [borkdude/clj-reflector-graal-java11-fix "0.0.1-graalvm-20.1.0"]]} :libsci {:dependencies [[cheshire "5.10.0"]] :source-paths ["src" "libsci/src"]}} ;; for testing only diff --git a/reflector/.gitignore b/reflector/.gitignore new file mode 100644 index 00000000..b50652f6 --- /dev/null +++ b/reflector/.gitignore @@ -0,0 +1 @@ +project.clj diff --git a/reflector/project.clj b/reflector/project.clj.template similarity index 75% rename from reflector/project.clj rename to reflector/project.clj.template index 98f6318f..ae53c4bd 100644 --- a/reflector/project.clj +++ b/reflector/project.clj.template @@ -1,6 +1,6 @@ -(defproject borkdude/sci.impl.reflector "0.0.1" +(defproject borkdude/sci.impl.reflector "{{version}}" :dependencies [[org.clojure/clojure "1.9.0"]] - :java-source-paths ["src-java"] + :java-source-paths ["{{java-source-path}}"] :deploy-repositories [["clojars" {:url "https://clojars.org/repo" :username :env/clojars_user :password :env/clojars_pass diff --git a/reflector/script/deploy_java11 b/reflector/script/deploy_java11 new file mode 100755 index 00000000..c0cd4b8f --- /dev/null +++ b/reflector/script/deploy_java11 @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +export SCI_REFLECTOR_JAVA11=true +script/update-project.clj +lein deploy clojars diff --git a/reflector/script/deploy_java8 b/reflector/script/deploy_java8 new file mode 100755 index 00000000..ebecb128 --- /dev/null +++ b/reflector/script/deploy_java8 @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +export SCI_REFLECTOR_JAVA11=false +script/update-project.clj +lein deploy clojars diff --git a/reflector/script/update-project.clj b/reflector/script/update-project.clj new file mode 100755 index 00000000..58b70a30 --- /dev/null +++ b/reflector/script/update-project.clj @@ -0,0 +1,26 @@ +#!/usr/bin/env clojure -Sdeps {:deps,{selmer,{:mvn/version,"1.12.17"}}} + +(require '[selmer.parser :as p]) +(require '[clojure.java.io :as io]) + +(def version (str "0.0.1" + (if (= "true" (System/getenv "SCI_REFLECTOR_JAVA11")) + "-java11" + ""))) + +(def java-source-path (str "src-java" + (if (= "true" (System/getenv "SCI_REFLECTOR_JAVA11")) + "11" + "8"))) + +(spit "project.clj" (str ";; GENERATED by script/update-project.clj, DO NOT EDIT\n" + "\n" + (p/render (slurp (io/file "project.clj.template")) + {:version version + :java-source-path java-source-path}))) + + +;; [[org.clojure/clojurescript "1.10.520"] ;; for extraction tests +;; [clj-commons/conch "0.9.2"] +;; [jonase/eastwood "0.3.6"] +;; [borkdude/missing.test.assertions "0.0.1"]] diff --git a/reflector/src-java11/sci/impl/Reflector.java b/reflector/src-java11/sci/impl/Reflector.java new file mode 100644 index 00000000..e137757c --- /dev/null +++ b/reflector/src-java11/sci/impl/Reflector.java @@ -0,0 +1,637 @@ +/** clojure.lang.Reflector adapted for sci **/ + +/** + * Copyright (c) Rich Hickey. All rights reserved. + * The use and distribution terms for this software are covered by the + * Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) + * which can be found in the file epl-v10.html at the root of this distribution. + * By using this software in any fashion, you are agreeing to be bound by + * the terms of this license. + * You must not remove this notice, or any other, from this software. + **/ + +/* rich Apr 19, 2006 */ + +package sci.impl; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.*; +import java.util.stream.Collectors; +import clojure.lang.Util; +import clojure.lang.RT; +import clojure.lang.Compiler; + +public class Reflector{ + + private static boolean canAccess(Method m, Object target) { + // JDK9+ use j.l.r.AccessibleObject::canAccess, which respects module rules + try { + return (boolean) m.canAccess(target); + } catch (Throwable t) { + throw Util.sneakyThrow(t); + } + + } + + private static Collection interfaces(Class c) { + Set interfaces = new HashSet(); + Deque toWalk = new ArrayDeque(); + toWalk.addAll(Arrays.asList(c.getInterfaces())); + Class iface = toWalk.poll(); + while (iface != null) { + interfaces.add(iface); + toWalk.addAll(Arrays.asList(iface.getInterfaces())); + iface = toWalk.poll(); + } + return interfaces; + } + + private static Method tryFindMethod(Class c, Method m) { + if(c == null) return null; + try { + return c.getMethod(m.getName(), m.getParameterTypes()); + } catch(NoSuchMethodException e) { + return null; + } + } + + private static Method toAccessibleSuperMethod(Method m, Object target) { + Method selected = m; + while(selected != null) { + if(canAccess(selected, target)) return selected; + selected = tryFindMethod(selected.getDeclaringClass().getSuperclass(), m); + } + + Collection interfaces = interfaces(m.getDeclaringClass()); + for(Class c : interfaces) { + selected = tryFindMethod(c, m); + if(selected != null) return selected; + } + return null; + } + + public static Object invokeInstanceMethod(Object target, String methodName, Object[] args) { + Class c = target.getClass(); + List methods = getMethods(c, args.length, methodName, false).stream() + .map(method -> toAccessibleSuperMethod(method, target)) + .filter(method -> (method != null)) + .collect(Collectors.toList()); + return invokeMatchingMethod(methodName, methods, target, args); + } + + private static Throwable getCauseOrElse(Exception e) { + if (e.getCause() != null) + return e.getCause(); + return e; + } + + private static RuntimeException throwCauseOrElseException(Exception e) { + if (e.getCause() != null) + throw Util.sneakyThrow(e.getCause()); + throw Util.sneakyThrow(e); + } + + private static String noMethodReport(String methodName, Object target, Object[] args){ + return "No matching method " + methodName + " found taking " + args.length + " args" + + (target==null?"":" for " + target.getClass()); + } + public static Object invokeMatchingMethod(String methodName, List methods, Object target, Object[] args) + { + Method m = null; + Object[] boxedArgs = null; + if(methods.isEmpty()) + { + throw new IllegalArgumentException(noMethodReport(methodName,target,args)); + } + else if(methods.size() == 1) + { + m = (Method) methods.get(0); + boxedArgs = boxArgs(m.getParameterTypes(), args); + } + else //overloaded w/same arity + { + Method foundm = null; + for(Iterator i = methods.iterator(); i.hasNext();) + { + m = (Method) i.next(); + + Class[] params = m.getParameterTypes(); + if(isCongruent(params, args)) + { + if(foundm == null || Compiler.subsumes(params, foundm.getParameterTypes())) + { + foundm = m; + boxedArgs = boxArgs(params, args); + } + } + } + m = foundm; + } + if(m == null) + throw new IllegalArgumentException(noMethodReport(methodName,target,args)); + + if(!Modifier.isPublic(m.getDeclaringClass().getModifiers()) || !canAccess(m, target)) + { + //public method of non-public class, try to find it in hierarchy + Method oldm = m; + m = getAsMethodOfAccessibleBase(target.getClass(), m, target); + if(m == null) + throw new IllegalArgumentException("Can't call public method of non-public class: " + + oldm.toString()); + } + try + { + return prepRet(m.getReturnType(), m.invoke(target, boxedArgs)); + } + catch(Exception e) + { + throw Util.sneakyThrow(getCauseOrElse(e)); + } + + } + + // DEPRECATED - replaced by getAsMethodOfAccessibleBase() + public static Method getAsMethodOfPublicBase(Class c, Method m){ + for(Class iface : c.getInterfaces()) + { + for(Method im : iface.getMethods()) + { + if(isMatch(im, m)) + { + return im; + } + } + } + Class sc = c.getSuperclass(); + if(sc == null) + return null; + for(Method scm : sc.getMethods()) + { + if(isMatch(scm, m)) + { + return scm; + } + } + return getAsMethodOfPublicBase(sc, m); + } + + // DEPRECATED - replaced by isAccessibleMatch() + public static boolean isMatch(Method lhs, Method rhs) { + if(!lhs.getName().equals(rhs.getName()) + || !Modifier.isPublic(lhs.getDeclaringClass().getModifiers())) + { + return false; + } + + Class[] types1 = lhs.getParameterTypes(); + Class[] types2 = rhs.getParameterTypes(); + if(types1.length != types2.length) + return false; + + boolean match = true; + for (int i=0; i 0) + return invokeMatchingMethod(name, meths, target, RT.EMPTY_ARRAY); + else + return getInstanceField(target, name); + } + } + + public static Object invokeInstanceMember(Object target, String name) { + //check for field first + Class c = target.getClass(); + Field f = getField(c, name, false); + if(f != null) //field get + { + try + { + return prepRet(f.getType(), f.get(target)); + } + catch(IllegalAccessException e) + { + throw Util.sneakyThrow(e); + } + } + return invokeInstanceMethod(target, name, RT.EMPTY_ARRAY); + } + + public static Object invokeInstanceMember(String name, Object target, Object arg1) { + //check for field first + Class c = target.getClass(); + Field f = getField(c, name, false); + if(f != null) //field set + { + try + { + f.set(target, boxArg(f.getType(), arg1)); + } + catch(IllegalAccessException e) + { + throw Util.sneakyThrow(e); + } + return arg1; + } + return invokeInstanceMethod(target, name, new Object[]{arg1}); + } + + public static Object invokeInstanceMember(String name, Object target, Object... args) { + return invokeInstanceMethod(target, name, args); + } + + + static public Field getField(Class c, String name, boolean getStatics){ + Field[] allfields = c.getFields(); + for(int i = 0; i < allfields.length; i++) + { + if(name.equals(allfields[i].getName()) + && Modifier.isStatic(allfields[i].getModifiers()) == getStatics) + return allfields[i]; + } + return null; + } + + static public List getMethods(Class c, int arity, String name, boolean getStatics){ + Method[] allmethods = c.getMethods(); + ArrayList methods = new ArrayList(); + ArrayList bridgeMethods = new ArrayList(); + for(int i = 0; i < allmethods.length; i++) + { + Method method = allmethods[i]; + if(name.equals(method.getName()) + && Modifier.isStatic(method.getModifiers()) == getStatics + && method.getParameterTypes().length == arity) + { + try + { + if(method.isBridge() + && c.getMethod(method.getName(), method.getParameterTypes()) + .equals(method)) + bridgeMethods.add(method); + else + methods.add(method); + } + catch(NoSuchMethodException e) + { + } + } + // && (!method.isBridge() + // || (c == StringBuilder.class && + // c.getMethod(method.getName(), method.getParameterTypes()) + // .equals(method)))) + // { + // methods.add(allmethods[i]); + // } + } + + if(methods.isEmpty()) + methods.addAll(bridgeMethods); + + if(!getStatics && c.isInterface()) + { + allmethods = Object.class.getMethods(); + for(int i = 0; i < allmethods.length; i++) + { + if(name.equals(allmethods[i].getName()) + && Modifier.isStatic(allmethods[i].getModifiers()) == getStatics + && allmethods[i].getParameterTypes().length == arity) + { + methods.add(allmethods[i]); + } + } + } + return methods; + } + + + static Object boxArg(Class paramType, Object arg){ + if(!paramType.isPrimitive()) + return paramType.cast(arg); + else if(paramType == boolean.class) + return Boolean.class.cast(arg); + else if(paramType == char.class) + return Character.class.cast(arg); + else if(arg instanceof Number) + { + Number n = (Number) arg; + if(paramType == int.class) + return n.intValue(); + else if(paramType == float.class) + return n.floatValue(); + else if(paramType == double.class) + return n.doubleValue(); + else if(paramType == long.class) + return n.longValue(); + else if(paramType == short.class) + return n.shortValue(); + else if(paramType == byte.class) + return n.byteValue(); + } + throw new IllegalArgumentException("Unexpected param type, expected: " + paramType + + ", given: " + arg.getClass().getName()); + } + + static Object[] boxArgs(Class[] params, Object[] args){ + if(params.length == 0) + return null; + Object[] ret = new Object[params.length]; + for(int i = 0; i < params.length; i++) + { + Object arg = args[i]; + Class paramType = params[i]; + ret[i] = boxArg(paramType, arg); + } + return ret; + } + + static public boolean paramArgTypeMatch(Class paramType, Class argType){ + if(argType == null) + return !paramType.isPrimitive(); + if(paramType == argType || paramType.isAssignableFrom(argType)) + return true; + if(paramType == int.class) + return argType == Integer.class + || argType == long.class + || argType == Long.class + || argType == short.class + || argType == byte.class;// || argType == FixNum.class; + else if(paramType == float.class) + return argType == Float.class + || argType == double.class; + else if(paramType == double.class) + return argType == Double.class + || argType == float.class;// || argType == DoubleNum.class; + else if(paramType == long.class) + return argType == Long.class + || argType == int.class + || argType == short.class + || argType == byte.class;// || argType == BigNum.class; + else if(paramType == char.class) + return argType == Character.class; + else if(paramType == short.class) + return argType == Short.class; + else if(paramType == byte.class) + return argType == Byte.class; + else if(paramType == boolean.class) + return argType == Boolean.class; + return false; + } + + static boolean isCongruent(Class[] params, Object[] args){ + boolean ret = false; + if(args == null) + return params.length == 0; + if(params.length == args.length) + { + ret = true; + for(int i = 0; ret && i < params.length; i++) + { + Object arg = args[i]; + Class argType = (arg == null) ? null : arg.getClass(); + Class paramType = params[i]; + ret = paramArgTypeMatch(paramType, argType); + } + } + return ret; + } + + public static Object prepRet(Class c, Object x){ + if (!(c.isPrimitive() || c == Boolean.class)) + return x; + if(x instanceof Boolean) + return ((Boolean) x)?Boolean.TRUE:Boolean.FALSE; + // else if(x instanceof Integer) + // { + // return ((Integer)x).longValue(); + // } + // else if(x instanceof Float) + // return Double.valueOf(((Float) x).doubleValue()); + return x; + } +} diff --git a/reflector/src-java/sci/impl/Reflector.java b/reflector/src-java8/sci/impl/Reflector.java similarity index 100% rename from reflector/src-java/sci/impl/Reflector.java rename to reflector/src-java8/sci/impl/Reflector.java diff --git a/script/compile b/script/compile index 891c41c8..2939ae58 100755 --- a/script/compile +++ b/script/compile @@ -17,10 +17,18 @@ fi SCI_VERSION=$(cat resources/SCI_VERSION) -lein with-profiles +clojure-1.10.2-alpha1 do clean, uberjar +export JAVA_HOME=$GRAALVM_HOME +export PATH=$GRAALVM_HOME/bin:$PATH + +lein with-profiles +clojure-1.10.2-alpha1,+native-image do clean, uberjar + +SCI_JAR=target/sci-$SCI_VERSION-standalone.jar +# SVM_JAR=$(find $GRAALVM_HOME | grep svm.jar) +# $GRAALVM_HOME/bin/javac -cp $SCI_JAR:$SVM_JAR src-java-graalvm/borkdude/Clojure_lang_Reflector_Substitutions.java $NATIVE_IMAGE \ - -jar target/sci-$SCI_VERSION-standalone.jar \ + -jar $SCI_JAR \ + -cp src-java-graalvm \ -H:Name=sci \ -H:+ReportExceptionStackTraces \ -J-Dclojure.spec.skip-macros=true \