From 90a1fc636aab20cb1db3d68a0b9f8351bc1b68fb Mon Sep 17 00:00:00 2001 From: Jared Forsyth Date: Sat, 9 Dec 2017 19:29:00 -0700 Subject: [PATCH] more android build work --- .gitignore | 1 + builder/BuildUtils.re | 60 ++++++++++++++ builder/Builder.re | 184 ++++++++++++++++++++++++++++++++++++++++++ builder/Getdeps.re | 142 ++++++++++++++++++++++++++++++++ reasongl-android | 1 + reasongl-ios | 1 + 6 files changed, 389 insertions(+) create mode 100644 builder/BuildUtils.re create mode 100644 builder/Builder.re create mode 100644 builder/Getdeps.re create mode 120000 reasongl-android create mode 120000 reasongl-ios diff --git a/.gitignore b/.gitignore index 1d60033..ac6243d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs bundler/Gravitron.app bundler/Gravitron.zip user_data_gravitron_data +_build diff --git a/builder/BuildUtils.re b/builder/BuildUtils.re new file mode 100644 index 0000000..c63914f --- /dev/null +++ b/builder/BuildUtils.re @@ -0,0 +1,60 @@ + +let withReSuffix = path => Filename.chop_extension(path) ++ ".re"; + +let copy = (source, dest) => { + let fs = Unix.openfile(source, [Unix.O_RDONLY], 0o640); + let fd = Unix.openfile(dest, [Unix.O_WRONLY, Unix.O_CREAT, Unix.O_TRUNC], 0o640); + let buffer_size = 8192; + let buffer = Bytes.create(buffer_size); + let rec copy_loop = () => switch(Unix.read(fs, buffer, 0, buffer_size)) { + | 0 => () + | r => { + ignore(Unix.write(fd, buffer, 0, r)); copy_loop(); + } + }; + copy_loop(); + Unix.close(fs); + Unix.close(fd); +}; + +let readdir = (dir) => { + let maybeGet = (handle) => + try (Some(Unix.readdir(handle))) { + | End_of_file => None + }; + let rec loop = (handle) => + switch (maybeGet(handle)) { + | None => + Unix.closedir(handle); + [] + | Some(name) => [name, ...loop(handle)] + }; + loop(Unix.opendir(dir)) +}; + +/** + * Get the output of a command, in lines. + */ +let readCommand = (command) => { + print_endline(command); + let chan = Unix.open_process_in(command); + try { + let rec loop = () => { + switch (Pervasives.input_line(chan)) { + | exception End_of_file => [] + | line => [line, ...loop()] + } + }; + let lines = loop(); + switch (Unix.close_process_in(chan)) { + | WEXITED(0) => Some(lines) + | WEXITED(_) + | WSIGNALED(_) + | WSTOPPED(_) => + print_endline("Unable to determine dependency order of files"); + None + } + } { + | End_of_file => None + } +}; diff --git a/builder/Builder.re b/builder/Builder.re new file mode 100644 index 0000000..52015ec --- /dev/null +++ b/builder/Builder.re @@ -0,0 +1,184 @@ + +let isNewer = (src, dest) => { + let stat = try (Some(Unix.stat(dest))) { + | Unix.Unix_error(Unix.ENOENT, _, _) => None + }; + switch stat { + | None => true + | Some(stat) => { + let ss = Unix.stat(src); + ss.Unix.st_mtime > stat.Unix.st_mtime + } + } +}; + +let copyIfNewer = (src, dest) => { + if (isNewer(src, dest)) { + BuildUtils.copy(src, dest) + } +}; + +let copyDirContents = (source, dest) => { + let contents = BuildUtils.readdir(source) + |> List.filter(x => x != "." && x != "..") + |> List.filter(name => { + let src = Filename.concat(source, name); + let stat = Unix.stat(src); + stat.Unix.st_kind === Unix.S_REG + }); + List.map( + name => { + /* print_endline(name); */ + copyIfNewer( + Filename.concat(source, name), + Filename.concat(dest, name) + ); + Filename.concat(dest, name) + }, + contents + ) +}; + +let unwrap = (message, opt) => switch opt { | None => failwith(message) | Some(x) => x}; + +type config = { + name: string, + env: string, + mainFile: string, + cOpts: string, + mlOpts: string, + dependencyDirs: list(string), + shared: bool, + outDir: string, + buildDir: string, + ocamlDir: string, + refmt: string, + ppx: list(string), +}; + +let isSourceFile = x => Filename.check_suffix(x, ".re") || Filename.check_suffix(x, ".ml"); + +let copyAndSort = ({mainFile, dependencyDirs, buildDir, ocamlDir, refmt}) => { + try (Unix.stat(buildDir) |> ignore) { + | Unix.Unix_error(Unix.ENOENT, _, _) => Unix.mkdir(buildDir, 0o740); + }; + let allNames = List.map(dirname => copyDirContents(dirname, buildDir), [Filename.dirname(mainFile), ...dependencyDirs]) |> List.concat; + let mainFileName = Filename.concat(buildDir, Filename.basename(mainFile)); + let reasonOrOcamlFiles = List.filter(isSourceFile, allNames); + let filesInOrder = unwrap("Failed to run ocamldep", Getdeps.sortSourceFilesInDependencyOrder(~ocamlDir, ~refmt, reasonOrOcamlFiles, mainFileName)); + /* print_endline(String.concat(" -- ", filesInOrder)); */ + (allNames, filesInOrder) +}; + +let ocamlopt = config => { + let ppxFlags = String.concat(" ", List.map(name => "-ppx " ++ name, config.ppx)); + Printf.sprintf( + "%s %s %s %s %s -I %s -w -40 -pp '%s --print binary'", + config.env, + Filename.concat(config.ocamlDir, "bin/ocamlopt"), + ppxFlags, + Str.split(Str.regexp(" "), config.cOpts) |> List.map(x => "-ccopt " ++ x) |> String.concat(" "), + config.mlOpts, + Filename.concat(config.ocamlDir, "lib/ocaml"), + config.refmt + ) +}; + +let compileMl = (config, force, sourcePath) => { + let cmx = Filename.chop_extension(sourcePath) ++ ".cmx"; + if (force || isNewer(sourcePath, cmx)) { + BuildUtils.readCommand(Printf.sprintf( + "%s -c -I %s -o %s -impl %s", + ocamlopt(config), + Filename.dirname(sourcePath), + cmx, + sourcePath + )) |> unwrap( + "Failed to build " ++ sourcePath + ) |> ignore; + (true, cmx) + } else { + (false, cmx) + } +}; + +let compileC = (config, force, sourcePath) => { + let dest = Filename.chop_extension(sourcePath) ++ ".o"; + if (force || isNewer(sourcePath, dest)) { + let out = Filename.basename(dest); + BuildUtils.readCommand(Printf.sprintf( + "%s -I %s -c -ccopt -std=c11 %s", + ocamlopt(config), + Filename.dirname(sourcePath), + sourcePath + )) |> unwrap( + "Failed to build " ++ sourcePath + ) |> ignore; + BuildUtils.copy(out, dest); + Unix.unlink(out); + (true, dest) + } else { + (false, dest) + }; +}; + +let compileShared = (config, cmxs, os) => { + let dest = Filename.concat(config.outDir, "lib" ++ config.name ++ ".so"); + let sourceFiles = [ + Filename.concat(config.ocamlDir, "lib/ocaml/libasmrun.a"), + "bigarray.cmx", + ...List.append(cmxs, os) + ]; + BuildUtils.readCommand(Printf.sprintf( + "%s -I %s -output-obj %s -o %s", + ocamlopt(config), + config.buildDir, + String.concat(" ", sourceFiles), + dest + )) |> unwrap( + "Failed to build " ++ dest + ) |> ignore; +}; + +let compileStatic = (config, cmxs, os) => failwith("not impl static"); + +let mapNewer = (fn, files) => { + List.fold_left( + ((force, results), name) => { + let (changed, result) = fn(force, name); + (changed || force, results @ [result]) + }, + (false, []), + files + ) |> snd +}; + +let compile = config => { + print_endline("Building"); + let (allNames, filesInOrder) = copyAndSort(config); + /** Build .cmx's */ + let cmxs = mapNewer(compileMl(config), filesInOrder); + /** Build .o's */ + let os = mapNewer(compileC(config), List.filter(name => Filename.check_suffix(name, ".c"), allNames)); + /** Build them together */ + config.shared ? compileShared(config, cmxs, os) : compileStatic(config, cmxs, os); + print_endline("Built!"); +}; + +compile({ + name: "gravitron", + shared: true, + mainFile: "./src/prod.re", + cOpts: "-fno-omit-frame-pointer -O3 -fPIC -llog -landroid -lGLESv3 -lEGL", + mlOpts: "-runtime-variant _pic", + dependencyDirs: ["./reasongl-interface/src", "./reasongl-android/src", "./reprocessing/src"], + buildDir: "_build", + env: "BSB_BACKEND=native-android", + outDir: "./", + ppx: ["./reasongl-android/matchenv.ppx"], + ocamlDir: "~/.opam/4.04.0-android32/android-sysroot", + refmt: "~/.opam/4.04.2/bin/refmt", + /* ppx: ["node_modules/matchenv/lib/bs/native/index.native"], */ + /* ocamlDir: "./node_modules/bs-platform/vendor/ocaml", */ + /* refmt: "./node_modules/bs-platform/bin/refmt3.exe" */ +}); diff --git a/builder/Getdeps.re b/builder/Getdeps.re new file mode 100644 index 0000000..acf33d7 --- /dev/null +++ b/builder/Getdeps.re @@ -0,0 +1,142 @@ + +open BuildUtils; + +let getSourceNames = (mainFile) => { + let sourceDirectory = Filename.dirname(mainFile); + List.filter( + (name) => Filename.check_suffix(name, ".re"), + readdir(sourceDirectory) + ) + |> List.map((name) => sourceDirectory ++ "/" ++ name) +}; + +let parseOcamldep = (lines) => { + List.map( + line => { + switch (Str.split(Str.regexp(":"), line)) { + | [target, deps] => { + let target = withReSuffix(String.trim(target)); + let deps = + Str.split(Str.regexp(" "), deps) + |> List.map(String.trim) + |> List.filter(x => String.length(x) > 0) + |> List.map(withReSuffix) + ; + (target, deps) + } + | [target] => { + (target |> String.trim |> withReSuffix, []) + } + | _ => failwith("Invalid ocamldep output: " ++ line) + } + }, + lines + ); +}; + +let oneFilesDeps = (depsList, mainFile) => { + let touched = Hashtbl.create(List.length(depsList)); + let rec loop = (fname) => { + Hashtbl.add(touched, fname, true); + let deps = try (List.assoc(fname, depsList)) { + | Not_found => failwith("Dependency not listed by ocamldep: " ++ fname) + } + ; + List.iter(loop, deps); + }; + loop(mainFile); + List.filter(item => Hashtbl.mem(touched, fst(item)), depsList); +}; + +let resolveDependencyOrder = (depsList, mainFile) => { + let mainDeps = oneFilesDeps(depsList, mainFile); + let scores = Hashtbl.create(List.length(mainDeps)); + + /** Initialize everything to zero */ + List.iter( + ((target, deps)) => { + Hashtbl.add(scores, target, 0); + List.iter(name => Hashtbl.add(scores, name, 0), deps); + }, + mainDeps + ); + + let loop = () => { + List.fold_left( + (updated, (target, deps)) => { + let highestDep = List.fold_left( + (highest, dep) => max(highest, Hashtbl.find(scores, dep)), + 0, + deps + ); + let current = Hashtbl.find(scores, target); + if (current < highestDep + 1) { + Hashtbl.add(scores, target, highestDep + 1); + true + } else { + updated + } + }, + false, + mainDeps + ) + }; + + /* this should settle pretty quickly */ + while (loop()) (); + + let files = List.map(fst, mainDeps); + let sorted = List.sort((a, b) => Hashtbl.find(scores, a) - Hashtbl.find(scores, b), files); + sorted +}; + +let sortSourceFilesInDependencyOrder = (sourceFiles, mainFile, ~ocamlDir, ~refmt) => { + let cmd = + Printf.sprintf( + "%s %s -pp '%s --print binary' -ml-synonym .re -I %s -one-line -native %s", + Filename.concat(ocamlDir, "bin/ocamlrun"), + Filename.concat(ocamlDir, "bin/ocamldep"), + refmt, + Filename.dirname(mainFile), + String.concat(" ", sourceFiles) + ); + switch (readCommand(cmd)) { + | None => None + | Some(raw) => + let depsList = parseOcamldep(raw); + let filesInOrder = resolveDependencyOrder(depsList, mainFile); + Some(filesInOrder) + } +}; + +let lastModifiedTimes = Hashtbl.create(10); + +let needsRebuild = (fileNames) => + List.fold_left( + ((needsRebuild, notReady), name) => { + /* If a file hasn't been compiled that we expect to be there, we set `notReady` to true + * As soon as bucklescript has built it, it will be ready. */ + let mtime = + try (Some(Unix.stat(name).Unix.st_mtime)) { + | Unix.Unix_error(Unix.ENOENT, _, _) => None + }; + switch mtime { + | None => (needsRebuild, true) + | Some(st_mtime) => + if (Hashtbl.mem(lastModifiedTimes, name)) { + if (st_mtime > Hashtbl.find(lastModifiedTimes, name)) { + Hashtbl.add(lastModifiedTimes, name, st_mtime); + (true, notReady) + } else { + (needsRebuild, notReady) + } + } else { + Hashtbl.add(lastModifiedTimes, name, st_mtime); + (true, notReady) + } + } + }, + (false, false), + fileNames + ); + diff --git a/reasongl-android b/reasongl-android new file mode 120000 index 0000000..d20b1a6 --- /dev/null +++ b/reasongl-android @@ -0,0 +1 @@ +../../fork/reasongl-android \ No newline at end of file diff --git a/reasongl-ios b/reasongl-ios new file mode 120000 index 0000000..3c1f750 --- /dev/null +++ b/reasongl-ios @@ -0,0 +1 @@ +../../fork/reasongl-ios \ No newline at end of file