From 05ce79ccdb82131cfcb07ba81d9a095e84f0051a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 13 Jun 2015 11:45:55 -0700 Subject: [PATCH 1/5] Make winrpm.sh crawl package dependencies and no longer always cd to dist-extras and explicitly check for its own dependencies (cherry picked from commit acb3e683bdbf50e06e69e61dab6170a4ab542fa9) ref PR #11705 --- contrib/windows/winrpm.sh | 72 ++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/contrib/windows/winrpm.sh b/contrib/windows/winrpm.sh index 152d486dae63a..53011a018b1b5 100755 --- a/contrib/windows/winrpm.sh +++ b/contrib/windows/winrpm.sh @@ -6,9 +6,13 @@ set -e url=$1 toinstall=$2 -# run in dist-extras -mkdir -p $(dirname "$0")/../../dist-extras -cd $(dirname "$0")/../../dist-extras + +for i in curl xmllint gunzip sort sha256sum 7z; do + if [ -z "$(which $i 2>/dev/null)" ]; then + echo "error: this script requires having $i installed" >&2 + exit 1 + fi +done # there is a curl --retry flag but it wasn't working here for some reason retry_curl() { @@ -16,7 +20,7 @@ retry_curl() { curl -fLsS $1 && return #sleep 2 done - echo "failed to download $1" >&2 + echo "error: failed to download $1" >&2 exit 1 } @@ -73,17 +77,69 @@ rpm_select() { [@ver='$maxver'][@rel='$maxrel']][1]" - } -mkdir -p noarch for i in $toinstall; do - pkgi=$(rpm_select $i) # fail if no available candidates for requested packages - if [ -z "$pkgi" ]; then + if [ -z "$(rpm_select $i)" ]; then exit 1 fi +done + +# outputs package and dll names, e.g. mingw64(zlib1.dll) +rpm_requires() { + for i in $(rpm_select $1 | \ + $xp "/package/format/requires/entry/@name" - 2>/dev/null); do + eval $i + echo $name + done +} + +# outputs package name, warns if multiple providers with different names +rpm_provides() { + providers=$(echo $primary | $xp "//*[$loc'package'][./*[$loc'format'] \ + /*[$loc'provides']/*[$loc'entry'][@name='$1']]/*[$loc'name']" - | \ + sed -e 's|||g' -e 's||\n|g' | sort -u) + if [ $(echo $providers | wc -w) -gt 1 ]; then + echo "warning: found multiple providers $providers for $1, adding all" >&2 + fi + echo $providers +} + +newpkgs=$toinstall +allrequires="" +while [ -n "$newpkgs" ]; do + newrequires="" + for i in $newpkgs; do + for j in $(rpm_requires $i); do + # leading and trailing spaces to ensure word match + case " $allrequires $newrequires " in + *" $j "*) # already on list + ;; + *) + newrequires="$newrequires $j";; + esac + done + done + allrequires="$allrequires $newrequires" + newpkgs="" + for i in $newrequires; do + provides="$(rpm_provides $i)" + case " $toinstall $newpkgs " in + *" $provides "*) # already on list + ;; + *) + newpkgs="$newpkgs $provides";; + esac + done + toinstall="$toinstall $newpkgs" +done + +mkdir -p noarch +for i in $toinstall; do + pkgi=$(rpm_select $i) checksum=$(echo $pkgi | $xp "/package/checksum/text()" -) eval $(echo $pkgi | $xp "/package/location/@href" -) echo "downloading $href" - ../deps/jldownload $href $url/$href + $(dirname "$0")/../../deps/jldownload $href $url/$href echo "$checksum *$href" | sha256sum -c 7z x -y $href cpiofile=$(basename $href | sed 's/.rpm$/.cpio/') From 8de9241bf683b815d29ab55a7a2ddd29dc12ba62 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 13 Jun 2015 14:59:59 -0700 Subject: [PATCH 2/5] Add a script to download mingw-w64 toolchain from opensuse Partially fix using the opensuse toolchain from cygwin Only create junction to includes if it does not exist yet (cherry picked from commit 690a37606e4b808e56fb5a76514b8ebe0b399a7c) Conflicts: deps/Makefile --- contrib/windows/get_toolchain.sh | 59 ++++++++++++++++++++++++++++++++ deps/Makefile | 4 +++ 2 files changed, 63 insertions(+) create mode 100755 contrib/windows/get_toolchain.sh diff --git a/contrib/windows/get_toolchain.sh b/contrib/windows/get_toolchain.sh new file mode 100755 index 0000000000000..3d7448e61a85a --- /dev/null +++ b/contrib/windows/get_toolchain.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# This file is a part of Julia. License is MIT: http://julialang.org/license + +# download mingw-w64 compilers from opensuse build service, usage: +# ./get_toolchain.sh 64 +# (or ./get_toolchain.sh 32) +# depends on curl, xmllint, gunzip, sort -V, sha256sum, and p7zip + +# Run in top-level Julia directory +cd `dirname "$0"`/../.. +# Stop on error +set -e +bits=$1 + +case $bits in + 32) + host=i686-w64-mingw32 + exc=sjlj + ;; + 64) + host=x86_64-w64-mingw32 + exc=seh + ;; + *) + echo 'error: run script either as `./get_toolchain.sh 32` or `./get_toolchain.sh 64`' >&2 + exit 1 + ;; +esac +echo "Downloading $host toolchain, check $PWD/get_toolchain.log for full output" +contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win$bits/openSUSE_13.1 \ + "mingw$bits-gcc mingw$bits-gcc-c++ mingw$bits-gcc-fortran \ + mingw$bits-libssp0 mingw$bits-libstdc++6 mingw$bits-libgfortran3" > get_toolchain.log + +mingwdir=usr/$host/sys-root/mingw +chmod +x $mingwdir/bin/* $mingwdir/$host/bin/* $mingwdir/libexec/gcc/$host/*/* +mkdir -p usr/bin +for i in gcc_s_$exc-1 ssp-0 stdc++-6 gfortran-3 quadmath-0; do + cp $mingwdir/bin/lib$i.dll usr/bin +done +$mingwdir/bin/g++ --version +# copy around binutils and make a junction for includes +cp $mingwdir/$host/bin/* $mingwdir/bin +case $(uname) in + CYGWIN*) + mklink="cmd /C mklink /J" + # treat these like cross-compilers if we're running from cygwin + for i in gcc g++ gfortran; do + mv $mingwdir/bin/$i.exe $mingwdir/bin/$host-$i.exe + done + ;; + *) + mklink="cmd //C mklink //J" + ;; +esac +if ! [ -e $mingwdir/$host/include ]; then + $mklink $(cygpath -w $mingwdir/$host/include) $(cygpath -w $mingwdir/include) +fi +echo "Toolchain successfully downloaded to $PWD/$mingwdir" +echo "Add toolchain to your path by running \`export PATH=$PWD/$mingwdir/bin:\$PATH\`" diff --git a/deps/Makefile b/deps/Makefile index fed17e9386e43..969c3dd7125d5 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -451,6 +451,7 @@ install-llvm: $(LLVM_OBJ_TARGET) UV_SRC_TARGET = libuv/.libs/libuv.a UV_OBJ_TARGET = $(build_libdir)/libuv.a + ifneq ($(USEMSVC), 1) UV_OPTS = else @@ -592,6 +593,9 @@ install-double-conversion: $(GRISU_OBJ_TARGET) ## openlibm ## OPENLIBM_FLAGS = ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) +ifeq (CYGWIN,$(findstring CYGWIN,$(BUILD_OS))) +OPENLIBM_FLAGS += OPENLIBM_HOME="$(call cygpath_w,$(abspath $(OPENLIBM_SRC_DIR)))" +endif OPENLIBM_OBJ_TARGET = $(build_shlibdir)/libopenlibm.$(SHLIB_EXT) $(build_libdir)/libopenlibm.a OPENLIBM_OBJ_SOURCE = openlibm/libopenlibm.$(SHLIB_EXT) From 5943c050390172c53e72c43544279ef438446537 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 13 Jun 2015 21:58:00 -0700 Subject: [PATCH 3/5] Rewrite the MSYS2 build instructions testers needed! Rewrite cross-compiling instructions to use docker and opensuse 13.1 and fix markdown lists Vagrant instead of Docker adjust msys2 setup steps Test with msys2's python2 Remove path recommendation, not really needed any more [av skip] (cherry picked from commit 709b8f4dd374d50c1f523d325fd9276b43d4c23b) Conflicts: README.windows.md --- README.windows.md | 203 ++++++++++++++-------------------------------- deps/Makefile | 2 +- 2 files changed, 64 insertions(+), 141 deletions(-) diff --git a/README.windows.md b/README.windows.md index d2000ce6d90dd..dc7fee6ef950a 100644 --- a/README.windows.md +++ b/README.windows.md @@ -25,11 +25,9 @@ Both the 32-bit and 64-bit versions are supported. The 32-bit (i686) binary will run on either a 32-bit and 64-bit operating system. The 64-bit (x86_64) binary will only run on 64-bit Windows and will otherwise refuse to launch. -1. Download and install [7-Zip](http://www.7-zip.org/download.html). Install the full program, not just the command line version. +1. [Download](http://julialang.org/downloads) the latest version of Julia. Extract the binary to a reasonable destination folder, e.g. `C:\julia`. -2. [Download](http://julialang.org/downloads) the latest version of Julia. Extract the binary to a reasonable destination folder, e.g. `C:\julia`. - -3. Double-click the file `julia.bat` to launch Julia. +2. Double-click the `julia` shortcut to launch Julia. # Line endings @@ -56,104 +54,58 @@ or edit `%USERPROFILE%\.gitconfig` and add/edit the lines: ## Compiling with MinGW/MSYS2 ### MSYS2 provides a robust MSYS experience. -### The instructions in this section were tested with the latest versions of all packages specified as of 2014-02-28. - -1. Install [7-Zip](http://www.7-zip.org/download.html). - -2. Install [Python 2.x](http://www.python.org/download/releases). Do **not** install Python 3. - -3. Install [MinGW-builds](http://sourceforge.net/projects/mingwbuilds/), a Windows port of GCC, as follows. Do **not** use the regular MinGW distribution. - 1. Download the [MinGW-builds installer](http://downloads.sourceforge.net/project/mingwbuilds/mingw-builds-install/mingw-builds-install.exe). - 2. Run the installer. When prompted, choose: - - Version: the most recent version (these instructions were tested with 4.8.1) - - Architecture: `x32` or `x64` as appropriate and desired. - - Threads: `win32` (not posix) - - Exception: `sjlj` (for x32) or `seh` (for x64). Do not choose dwarf2. - - Build revision: most recent available (tested with 5) - 3. Do **not** install to a directory with spaces in the name. You will have to change the default installation path, for example, - - `C:\mingw-builds\x64-4.8.1-win32-seh-rev5` for 64 bits - - `C:\mingw-builds\x32-4.8.1-win32-sjlj-rev5` for 32 bits -4. Install and configure [MSYS2](http://sourceforge.net/projects/msys2), a minimal POSIX-like environment for Windows. +1. Install and configure [MSYS2](https://msys2.github.io), a minimal POSIX-like environment for Windows. - 1. Download the latest base [32-bit](http://sourceforge.net/projects/msys2/files/Base/i686/) or [64-bit](http://sourceforge.net/projects/msys2/files/Base/x86_64/) distribution, consistent with the architecture you chose for MinGW-builds. The archive will have a name like `msys2-base-x86_64-yyyymmdd.tar.xz` and these instructions were tested with `msys2-base-x86_64-20140216.tar.xz`. + 1. Download and run the latest installer for the [32-bit](http://sourceforge.net/projects/msys2/files/Base/i686/) or [64-bit](http://sourceforge.net/projects/msys2/files/Base/x86_64/) distribution. The installer will have a name like `msys2-i686-yyyymmdd.exe` or `msys2-x86_64-yyyymmdd.exe`. - 2. Using [7-Zip](http://www.7-zip.org/download.html), extract the archive to any convenient directory. - - *N.B.* Some versions of this archive contain zero-byte files that clash with existing files. If prompted, choose **not** to overwrite existing files. - - You may need to extract the tarball in a separate step. This will create an `msys32` or `msys64` directory, according to the architecture you chose. - - Move the `msys32` or `msys64` directory into your MinGW-builds directory, which is `C:\mingw-builds` if you followed the suggestions in step 3. We will omit the "32" or "64" in the steps below and refer to this as "the msys directory". - - 3. Double-click `msys2_shell.bat` in the msys directory. This will initialize MSYS2. The shell will tell you to `exit` and restart the shell. For now, ignore it. - - 4. Update MSYS2 and install packages required to build julia, using the `pacman` package manager included in MSYS2: + 2. Double-click `msys2_shell.bat` in the installed msys directory. Initialize the MSYS2 base system using the `pacman` package manager included in MSYS2: ``` - pacman-key --init #Download keys - pacman -Syu #Update package database and full system upgrade + pacman --needed -Sy bash pacman pacman-mirrors msys2-runtime ``` - Now `exit` the MSYS2 shell and restart it, *even if you already restarted it above*. This is necessary in case the system upgrade updated the main MSYS2 libs. Reopen the MSYS2 shell and continue with: - ``` - pacman -S diffutils git m4 make patch tar msys/openssh -``` - - 5. Configure your MSYS2 shell for convenience: + 3. Exit and restart MSYS2, then install packages required to build julia: ``` - echo "mount C:/Python27 /python" >> ~/.bashrc - # uncomment ONE of the following two lines - #echo "mount C:/mingw-builds/x64-4.8.1-win32-seh-rev5/mingw64 /mingw" >> ~/.bashrc - #echo "mount C:/mingw-builds/x32-4.8.1-win32-sjlj-rev5/mingw32 /mingw" >> ~/.bashrc - echo "export PATH=/usr/local/bin:/usr/bin:/opt/bin:/mingw/bin:/python" >> ~/.bashrc + pacman -Syu #Update package database and full system upgrade + pacman -S diffutils git m4 make patch tar python2 p7zip msys/openssh ``` - *N.B.* The `export` clobbers whatever `$PATH` is already defined. This is suggested to avoid path-masking. If you use MSYS2 for purposes other than building Julia, you may prefer to append rather than clobber. - - *N.B.* All of the path-separators in the mount commands are unix-style. - - - 6. Configuration of the toolchain is complete. Now `exit` the MSYS2 shell. - -5. Build Julia and its dependencies from source. - 1. Relaunch the MSYS2 shell and type + 4. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. +2. Build Julia and its dependencies from source. + 1. Open a new MSYS2 shell and clone the Julia sources ``` - . ~/.bashrc # Some versions of MSYS2 do not run this automatically -``` - - Ignore any warnings you see from `mount` about `/mingw` and `/python` not existing. - - 2. Get the Julia sources and start the build: - ``` git clone https://github.com/JuliaLang/julia.git cd julia - make -j 4 # Adjust the number of cores (4) to match your build environment. ``` - 3. The Julia build can (as of 2014-02-28) fail after building OpenBLAS. - This appears (?) to be a result of the OpenBLAS build trying to run the Microsoft Visual C++ `lib.exe` tool -- which we don't need to do -- without checking for its existence. - This uncaught error kills the Julia build. If this happens, follow the instructions in the helpful error message and continue the build, *viz.* - + 2. Run the following script to download the correct versions of the MinGW-w64 compilers ``` - cd deps/openblas-v0.2.9.rc1 # This path will depend on the version of OpenBLAS. - make install - cd ../.. - make -j 4 # Adjust the number of cores (4) to match your build environment. + contrib/windows/get_toolchain.sh 32 # for 32 bit Julia + # or + contrib/windows/get_toolchain.sh 64 # for 64 bit Julia ``` + Then follow the printed instructions by running either + ``` + export PATH=$PWD/usr/i686-w64-mingw32/sys-root/mingw/bin:$PATH # for 32 bit Julia + # or + export PATH=$PWD/usr/x86_64-w64-mingw32/sys-root/mingw/bin:$PATH # for 64 bit Julia +``` + to add the downloaded MinGW-w64 compilers to your path (temporarily, only needed during the shell session when you build Julia). - 4. Some versions of PCRE (*e.g.* 8.31) will compile correctly but fail a test. - This will cause the Julia build to fail. To circumvent testing for PCRE and allow the rest - of the build to continue, - ``` - touch deps/pcre-8.31/checked # This path will depend on the version of PCRE. + 3. Start the build + ``` make -j 4 # Adjust the number of cores (4) to match your build environment. ``` -6. Setup Package Development Environment + +3. Setup Package Development Environment 1. The `Pkg` module in Base provides many convenient tools for [developing and publishing packages](http://docs.julialang.org/en/latest/manual/packages/). One of the packages added through pacman above was `openssh`, which will allow secure access to GitHub APIs. Follow GitHub's [guide](https://help.github.com/articles/generating-ssh-keys) to setting up SSH keys to ensure your local machine can communicate with GitHub effectively. - 5. In case of the issues with building packages (i.e. ICU fails to build with the following error message ```error compiling xp_parse: error compiling xp_make_parser: could not load module libexpat-1: %```) run ```make win-extras``` and then copy everything from the ```dist-extras``` folder into ```usr/bin```. + 2. In case of the issues with building packages (i.e. ICU fails to build with the following error message ```error compiling xp_parse: error compiling xp_make_parser: could not load module libexpat-1: %```) run ```make win-extras``` and then copy everything from the ```dist-extras``` folder into ```usr/bin```. ## Cygwin-to-MinGW cross compiling @@ -221,69 +173,40 @@ Julia can be also compiled from source in [Cygwin](http://www.cygwin.com), using If you prefer to cross-compile, the following steps should get you started. -### Ubuntu and Mac Dependencies (these steps will work for almost any linux platform) - -First, you will need to ensure your system has the required dependencies. We need wine (>=1.7.5), -a system compiler, and some downloaders. - -On Ubuntu: - - apt-add-repository ppa:ubuntu-wine/ppa - apt-get upate - apt-get install wine1.7 subversion cvs gcc wget p7zip-full - - -On Mac: Install XCode, XCode command line tools, X11 (now [XQuartz](http://xquartz.macosforge.org/)), -and [MacPorts](http://www.macports.org/install.php) or [Homebrew](http://mxcl.github.io/homebrew/). -Then run ```port install wine wget``` or ```brew install wine wget```, as appropriate. - -On Both: - -Unfortunately, the version of gcc installed by Ubuntu is currently 4.6, which does not compile OpenBLAS correctly. -On Mac, the situation is the same: the version in MacPorts is very old and Homebrew does not have it. -So first we need to get a cross-compile version of gcc. -Most binary packages appear to not include gfortran, so we will need to compile it from source (or ask @vtjnash to send you a tgz of his build). -This is typically quite a bit of work, so we will use [this script](https://code.google.com/p/mingw-w64-dgn/) to make it easy. - -1. `svn checkout http://mingw-w64-dgn.googlecode.com/svn/trunk/ mingw-w64-dgn` -2. `cd mingw-w64-dgn` -3. edit `rebuild_cross.sh` and make the following two changes: - a. uncomment `export MAKE_OPT="-j 2"`, if appropriate for your machine - b. add `fortran` to the end of `--enable-languages=c,c++,objc,obj-c++` -5. `bash update_source.sh` -4. `bash rebuild_cross.sh` -5. `mv cross ~/cross-w64` -6. `export PATH=$HOME/cross-w64/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps!, you can put this line in your .profile to make it easy - -Then we can essentially just repeat these steps for the 32-bit compiler, reusing some of the work: - -7. `cd ..` -8. `cp -a mingw-w64-dgn mingw-w32-dgn` -9. `cd mingw-w32-dgn` -10. `rm -r cross build` -11. `bash rebuild_cross.sh 32r` -12. `mv cross ~/cross-w32` -13. `export PATH=$HOME/cross-w32/bin:$PATH` # NOTE: it is important that you remember to always do this before using make in the following steps!, you can put this line in your .profile to make it easy - -Note: for systems that support rpm-based package managers, the OpenSUSE build service appears to contain a fully up-to-date versions of the necessary dependencies. - -### Arch Linux Dependencies - -1. Install the following packages from the official Arch repository: -`sudo pacman -S cloog gcc-ada libmpc p7zip ppl subversion zlib` -2. The rest of the prerequisites consist of the mingw-w64 packages, which are available in the AUR Arch repository. They must be installed exactly in the order they are given or else their installation will fail. The `yaourt` package manager is used for illustration purposes; you may instead follow the [Arch instructions for installing packages from AUR](https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_packages) or may use your preferred package manager. To start with, install `mingw-w64-binutils` via the command -`yaourt -S mingw-w64-binutils` -3. `yaourt -S mingw-w64-headers-svn` -4. `yaourt -S mingw-w64-headers-bootstrap` -5. `yaourt -S mingw-w64-gcc-base` -6. `yaourt -S mingw-w64-crt-svn` -7. Remove `mingw-w64-headers-bootstrap` without removing its dependent mingw-w64 installed packages by using the command -`yaourt -Rdd mingw-w64-headers-bootstrap` -8. `yaourt -S mingw-w64-winpthreads` -9. Remove `mingw-w64-gcc-base` without removing its installed mingw-w64 dependencies: -`yaourt -Rdd mingw-w64-gcc-base` -10. Complete the installation of the required `mingw-w64` packages: -`yaourt -S mingw-w64-gcc` +For maximum compatibility with packages that use [WinRPM.jl](https://github.com/JuliaLang/WinRPM.jl) for binary dependencies on Windows, it is recommended that you use OpenSUSE 13.1 for cross-compiling a Windows build of Julia. If you use a different Linux distribution or OS X, install [Vagrant](http://www.vagrantup.com/downloads) and use the following `Vagrantfile`: +``` +# Vagrantfile for MinGW-w64 cross-compilation of Julia + +$script = <