From 4ded299a740612b87b8cc576efdb998f5a733192 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 5 Mar 2013 19:20:00 -0500 Subject: [PATCH 01/10] Using mini_portile to compile libxml2 and libxslt on any (non-windows) platform. --- ext/nokogiri/extconf.rb | 141 ++++++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 40 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 1f7301d9ee..c74f517636 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -49,46 +49,107 @@ else lib_prefix = '' - HEADER_DIRS = [ - # First search /opt/local for macports - '/opt/local/include', - - # Then search /usr/local for people that installed from source - '/usr/local/include', - - # Check the ruby install locations - INCLUDEDIR, - - # Finally fall back to /usr - '/usr/include', - '/usr/include/libxml2', - ] - - LIB_DIRS = [ - # First search /opt/local for macports - '/opt/local/lib', - - # Then search /usr/local for people that installed from source - '/usr/local/lib', - - # Check the ruby install locations - LIBDIR, - - # Finally fall back to /usr - '/usr/lib', - ] - - XML2_HEADER_DIRS = [ - '/opt/local/include/libxml2', - '/usr/local/include/libxml2', - File.join(INCLUDEDIR, "libxml2") - ] + HEADER_DIRS - - # If the user has homebrew installed, use the libxml2 inside homebrew - brew_prefix = `brew --prefix libxml2 2> /dev/null`.chomp - unless brew_prefix.empty? - LIB_DIRS.unshift File.join(brew_prefix, 'lib') - XML2_HEADER_DIRS.unshift File.join(brew_prefix, 'include/libxml2') + if ENV['BUILD_LIBXML2'] + require 'mini_portile' + require 'yaml' + + common_recipe = lambda do |recipe| + recipe.target = File.join(ROOT, "ports") + recipe.files = ["ftp://ftp.xmlsoft.org/libxml2/#{recipe.name}-#{recipe.version}.tar.gz"] + + class << recipe + def download + # this should *never* get run at gem installation time. + puts "#{__FILE__}:#{__LINE__}: WARNING: downloading files should never happen at gem installation time." + FileUtils.mkdir_p archives_path unless File.directory? archives_path + Dir.chdir archives_path do + @files.each do |url| + system "wget #{url} || curl -O #{url}" + end + end + end + end + + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + unless File.exist?(checkpoint) + recipe.cook + FileUtils.touch checkpoint + end + recipe.activate + end + + libxml2_recipe = MiniPortile.new("libxml2", "2.7.8").tap do |recipe| + recipe.configure_options = [ + "--enable-shared", + "--disable-static", + "--without-python", + "--without-readline", + "--with-c14n", + "--with-debug" + ] + common_recipe.call recipe + end + + libxslt_recipe = MiniPortile.new("libxslt", "1.1.26").tap do |recipe| + recipe.configure_options = [ + "--enable-shared", + "--disable-static", + "--without-python", + "--without-crypto", + "--with-debug", + "--with-libxml-prefix=#{libxml2_recipe.path}" + ] + common_recipe.call recipe + end + + $LDFLAGS << " -Wl,-rpath,#{libxml2_recipe.path}/lib" + $LDFLAGS << " -Wl,-rpath,#{libxslt_recipe.path}/lib" + + HEADER_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "include") } + LIB_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "lib") } + XML2_HEADER_DIRS = HEADER_DIRS + [File.join(libxml2_recipe.path, "include", "libxml2")] + else + HEADER_DIRS = [ + # First search /opt/local for macports + '/opt/local/include', + + # Then search /usr/local for people that installed from source + '/usr/local/include', + + # Check the ruby install locations + INCLUDEDIR, + + # Finally fall back to /usr + '/usr/include', + '/usr/include/libxml2', + ] + + LIB_DIRS = [ + # First search /opt/local for macports + '/opt/local/lib', + + # Then search /usr/local for people that installed from source + '/usr/local/lib', + + # Check the ruby install locations + LIBDIR, + + # Finally fall back to /usr + '/usr/lib', + ] + + XML2_HEADER_DIRS = [ + '/opt/local/include/libxml2', + '/usr/local/include/libxml2', + File.join(INCLUDEDIR, "libxml2") + ] + HEADER_DIRS + + # If the user has homebrew installed, use the libxml2 inside homebrew + brew_prefix = `brew --prefix libxml2 2> /dev/null`.chomp + unless brew_prefix.empty? + LIB_DIRS.unshift File.join(brew_prefix, 'lib') + XML2_HEADER_DIRS.unshift File.join(brew_prefix, 'include/libxml2') + end end end From efac85f638b81aa4dbdec95b36e7d60ab6e09392 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 5 Mar 2013 22:35:20 -0500 Subject: [PATCH 02/10] dependencies.yml is now the central location for library dependencies. Note that this is for both windows cross-compile and packaged dependencies. --- dependencies.yml | 4 ++++ ext/nokogiri/extconf.rb | 6 ++++-- tasks/cross_compile.rb | 8 ++++---- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 dependencies.yml diff --git a/dependencies.yml b/dependencies.yml new file mode 100644 index 0000000000..e1f8d73f1a --- /dev/null +++ b/dependencies.yml @@ -0,0 +1,4 @@ +libxml2: "2.7.7" +libxslt: "1.1.26" +zlib: "1.2.7" +libiconv: "1.13.1" diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index c74f517636..f8e37b6b59 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -78,7 +78,9 @@ def download recipe.activate end - libxml2_recipe = MiniPortile.new("libxml2", "2.7.8").tap do |recipe| + dependencies = YAML.load_file(File.join(ROOT, "dependencies.yml")) + + libxml2_recipe = MiniPortile.new("libxml2", dependencies["libxml2"]).tap do |recipe| recipe.configure_options = [ "--enable-shared", "--disable-static", @@ -90,7 +92,7 @@ def download common_recipe.call recipe end - libxslt_recipe = MiniPortile.new("libxslt", "1.1.26").tap do |recipe| + libxslt_recipe = MiniPortile.new("libxslt", dependencies["libxslt"]).tap do |recipe| recipe.configure_options = [ "--enable-shared", "--disable-static", diff --git a/tasks/cross_compile.rb b/tasks/cross_compile.rb index 70cf94bcdc..e6ea8ce3a5 100644 --- a/tasks/cross_compile.rb +++ b/tasks/cross_compile.rb @@ -3,11 +3,11 @@ HOST = Rake::ExtensionCompiler.mingw_host require 'mini_portile' +dependencies = YAML.load_file("dependencies.yml") $recipes = {} -$recipes["zlib"] = MiniPortile.new "zlib", "1.2.7" -$recipes["libiconv"] = MiniPortile.new "libiconv", "1.13.1" -$recipes["libxml2"] = MiniPortile.new "libxml2", "2.7.7" -$recipes["libxslt"] = MiniPortile.new "libxslt", "1.1.26" +%w[zlib libiconv libxml2 libxslt].each do |lib| + $recipes[lib] = MiniPortile.new lib, dependencies[lib] +end $recipes.each { |_, recipe| recipe.host = HOST } file "lib/nokogiri/nokogiri.rb" do From 3c54e6ec886b14b385ed7069702d6db663d40891 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 5 Mar 2013 22:46:39 -0500 Subject: [PATCH 03/10] Upgrading libxml2 dependency from 2.7.7 to 2.7.8. Affects only windows users at this time. --- dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yml b/dependencies.yml index e1f8d73f1a..fcaa020d17 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -1,4 +1,4 @@ -libxml2: "2.7.7" +libxml2: "2.7.8" libxslt: "1.1.26" zlib: "1.2.7" libiconv: "1.13.1" From bf3e58d69c7bacd94a90cb817ae58ec056d8f536 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 6 Mar 2013 08:51:54 -0500 Subject: [PATCH 04/10] mini_portile is a gem dependency if we're building libxml2. --- Rakefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 7ddb674cb9..bfaef4f84e 100644 --- a/Rakefile +++ b/Rakefile @@ -45,7 +45,6 @@ HOE = Hoe.spec 'nokogiri' do ["hoe-debugging", ">= 1.0.3"], ["hoe-gemspec", ">= 1.0"], ["hoe-git", ">= 1.4"], - ["mini_portile", ">= 0.2.2"], ["minitest", "~> 2.2.2"], ["rake", ">= 0.9"], ["rake-compiler", "~> 0.8.0"], @@ -53,6 +52,16 @@ HOE = Hoe.spec 'nokogiri' do ["rexical", ">= 1.0.5"] ] + if ENV['BUILD_LIBXML2'] + self.extra_deps += [ + ["mini_portile", "~> 0.3.0"], + ] + else + self.extra_dev_deps += [ + ["mini_portile", "~> 0.3.0"], + ] + end + if java? self.spec_extras = { :platform => 'java' } else From 31aafdde1b90b2dd10b0f0aa2a4e754fba0e32a5 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 6 Mar 2013 08:53:01 -0500 Subject: [PATCH 05/10] If we're building libxml2, include the tarballs in the gem manifest. --- Rakefile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Rakefile b/Rakefile index bfaef4f84e..6f4937aa97 100644 --- a/Rakefile +++ b/Rakefile @@ -115,6 +115,19 @@ else HOE.spec.files.reject! { |f| f =~ %r{\.(java|jar)$} } + if ENV['BUILD_LIBXML2'] + task gem_build_path do + add_file_to_gem "dependencies.yml" + + dependencies = YAML.load_file("dependencies.yml") + %w[libxml2 libxslt].each do |lib| + version = dependencies[lib] + archive = File.join("ports", "archives", "#{lib}-#{version}.tar.gz") + add_file_to_gem archive + end + end + end + Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext| ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact) ext.config_options << ENV['EXTOPTS'] From 609044bf97e53cc8c6ab538c4f5d7ea0211e589f Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 8 Mar 2013 12:25:05 -0500 Subject: [PATCH 06/10] Upgrade mini_portile to support FTP, and removing my FTP-support hacks. --- Rakefile | 4 ++-- ext/nokogiri/extconf.rb | 13 ------------- tasks/cross_compile.rb | 18 ------------------ 3 files changed, 2 insertions(+), 33 deletions(-) diff --git a/Rakefile b/Rakefile index 6f4937aa97..0e502f8b31 100644 --- a/Rakefile +++ b/Rakefile @@ -54,11 +54,11 @@ HOE = Hoe.spec 'nokogiri' do if ENV['BUILD_LIBXML2'] self.extra_deps += [ - ["mini_portile", "~> 0.3.0"], + ["mini_portile", "~> 0.5.0"], ] else self.extra_dev_deps += [ - ["mini_portile", "~> 0.3.0"], + ["mini_portile", "~> 0.5.0"], ] end diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index f8e37b6b59..f6ef1e2eb4 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -57,19 +57,6 @@ recipe.target = File.join(ROOT, "ports") recipe.files = ["ftp://ftp.xmlsoft.org/libxml2/#{recipe.name}-#{recipe.version}.tar.gz"] - class << recipe - def download - # this should *never* get run at gem installation time. - puts "#{__FILE__}:#{__LINE__}: WARNING: downloading files should never happen at gem installation time." - FileUtils.mkdir_p archives_path unless File.directory? archives_path - Dir.chdir archives_path do - @files.each do |url| - system "wget #{url} || curl -O #{url}" - end - end - end - end - checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" unless File.exist?(checkpoint) recipe.cook diff --git a/tasks/cross_compile.rb b/tasks/cross_compile.rb index e6ea8ce3a5..9cb9dbdcfe 100644 --- a/tasks/cross_compile.rb +++ b/tasks/cross_compile.rb @@ -90,15 +90,6 @@ def install "--without-readline", "CFLAGS='-DIN_LIBXML'" ] - class << recipe - def download - Dir.chdir archives_path do - @files.each do |url| - sh "wget #{url} || curl -O #{url}" - end - end - end - end checkpoint = "#{CROSS_DIR}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" unless File.exist?(checkpoint) @@ -120,15 +111,6 @@ def download "--without-crypto", "CFLAGS='-DIN_LIBXML'" ] - class << recipe - def download - Dir.chdir archives_path do - @files.each do |url| - sh "wget #{url} || curl -O #{url}" - end - end - end - end checkpoint = "#{CROSS_DIR}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" unless File.exist?(checkpoint) From 84be636f5b646350e9dab50d34c4170c64c594cb Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 20 Mar 2013 01:00:07 -0400 Subject: [PATCH 07/10] Compile libxml2 with the --with-threads option. --- ext/nokogiri/extconf.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index f6ef1e2eb4..568c94a8cb 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -74,7 +74,8 @@ "--without-python", "--without-readline", "--with-c14n", - "--with-debug" + "--with-debug", + "--with-threads" ] common_recipe.call recipe end From ff096233ba48304f1a5d74c9772fe771eb1d33e4 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 14 Apr 2013 11:53:05 -0400 Subject: [PATCH 08/10] Setting fat-source dependency on libxml 2.8.0. --- dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yml b/dependencies.yml index fcaa020d17..8c51ffefb0 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -1,4 +1,4 @@ -libxml2: "2.7.8" +libxml2: "2.8.0" libxslt: "1.1.26" zlib: "1.2.7" libiconv: "1.13.1" From b79a4fe71948b4326e1aa92ffe63e6dfde7b6d3b Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 14 Apr 2013 12:05:15 -0400 Subject: [PATCH 09/10] Making fat-source libraries the default, unless NOKOGIRI_USE_SYSTEM_LIBRARIES is set. --- ext/nokogiri/extconf.rb | 88 +++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 568c94a8cb..0d1d90e6c0 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -49,7 +49,50 @@ else lib_prefix = '' - if ENV['BUILD_LIBXML2'] + if ENV['NOKOGIRI_USE_SYSTEM_LIBRARIES'] + HEADER_DIRS = [ + # First search /opt/local for macports + '/opt/local/include', + + # Then search /usr/local for people that installed from source + '/usr/local/include', + + # Check the ruby install locations + INCLUDEDIR, + + # Finally fall back to /usr + '/usr/include', + '/usr/include/libxml2', + ] + + LIB_DIRS = [ + # First search /opt/local for macports + '/opt/local/lib', + + # Then search /usr/local for people that installed from source + '/usr/local/lib', + + # Check the ruby install locations + LIBDIR, + + # Finally fall back to /usr + '/usr/lib', + ] + + XML2_HEADER_DIRS = [ + '/opt/local/include/libxml2', + '/usr/local/include/libxml2', + File.join(INCLUDEDIR, "libxml2") + ] + HEADER_DIRS + + # If the user has homebrew installed, use the libxml2 inside homebrew + brew_prefix = `brew --prefix libxml2 2> /dev/null`.chomp + unless brew_prefix.empty? + LIB_DIRS.unshift File.join(brew_prefix, 'lib') + XML2_HEADER_DIRS.unshift File.join(brew_prefix, 'include/libxml2') + end + + else require 'mini_portile' require 'yaml' @@ -98,48 +141,6 @@ HEADER_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "include") } LIB_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "lib") } XML2_HEADER_DIRS = HEADER_DIRS + [File.join(libxml2_recipe.path, "include", "libxml2")] - else - HEADER_DIRS = [ - # First search /opt/local for macports - '/opt/local/include', - - # Then search /usr/local for people that installed from source - '/usr/local/include', - - # Check the ruby install locations - INCLUDEDIR, - - # Finally fall back to /usr - '/usr/include', - '/usr/include/libxml2', - ] - - LIB_DIRS = [ - # First search /opt/local for macports - '/opt/local/lib', - - # Then search /usr/local for people that installed from source - '/usr/local/lib', - - # Check the ruby install locations - LIBDIR, - - # Finally fall back to /usr - '/usr/lib', - ] - - XML2_HEADER_DIRS = [ - '/opt/local/include/libxml2', - '/usr/local/include/libxml2', - File.join(INCLUDEDIR, "libxml2") - ] + HEADER_DIRS - - # If the user has homebrew installed, use the libxml2 inside homebrew - brew_prefix = `brew --prefix libxml2 2> /dev/null`.chomp - unless brew_prefix.empty? - LIB_DIRS.unshift File.join(brew_prefix, 'lib') - XML2_HEADER_DIRS.unshift File.join(brew_prefix, 'include/libxml2') - end end end @@ -192,4 +193,5 @@ def have_iconv? end create_makefile('nokogiri/nokogiri') + # :startdoc: From e811c083d14eb93ee3a9437a195ccae0b55e57b2 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 14 Apr 2013 12:13:09 -0400 Subject: [PATCH 10/10] VERSION_INFO["libxml"]["source"] now indicates whether libxml2 comes from "system" or "packaged" libraries. --- ext/nokogiri/extconf.rb | 2 ++ ext/nokogiri/nokogiri.c | 6 ++++++ lib/nokogiri/version.rb | 9 +++++++++ 3 files changed, 17 insertions(+) diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 0d1d90e6c0..7a0090813e 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -138,6 +138,8 @@ $LDFLAGS << " -Wl,-rpath,#{libxml2_recipe.path}/lib" $LDFLAGS << " -Wl,-rpath,#{libxslt_recipe.path}/lib" + $CFLAGS << " -DNOKOGIRI_USE_PACKAGED_LIBRARIES" + HEADER_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "include") } LIB_DIRS = [libxml2_recipe, libxslt_recipe].map { |_| File.join(_.path, "lib") } XML2_HEADER_DIRS = HEADER_DIRS + [File.join(libxml2_recipe.path, "include", "libxml2")] diff --git a/ext/nokogiri/nokogiri.c b/ext/nokogiri/nokogiri.c index 9bae313032..5d12e89457 100644 --- a/ext/nokogiri/nokogiri.c +++ b/ext/nokogiri/nokogiri.c @@ -90,6 +90,12 @@ void Init_nokogiri() NOKOGIRI_STR_NEW2(xmlParserVersion) ); +#ifdef NOKOGIRI_USE_PACKAGED_LIBRARIES + rb_const_set(mNokogiri, rb_intern("NOKOGIRI_USE_PACKAGED_LIBRARIES"), Qtrue); +#else + rb_const_set(mNokogiri, rb_intern("NOKOGIRI_USE_PACKAGED_LIBRARIES"), Qfalse); +#endif + #ifdef LIBXML_ICONV_ENABLED rb_const_set(mNokogiri, rb_intern("LIBXML_ICONV_ENABLED"), Qtrue); #else diff --git a/lib/nokogiri/version.rb b/lib/nokogiri/version.rb index 6450f0ffe9..6ae12799ee 100644 --- a/lib/nokogiri/version.rb +++ b/lib/nokogiri/version.rb @@ -25,6 +25,14 @@ def libxml2? defined?(LIBXML_VERSION) end + def libxml2_using_system? + ! libxml2_using_packaged? + end + + def libxml2_using_packaged? + NOKOGIRI_USE_PACKAGED_LIBRARIES + end + def warnings return [] unless libxml2? @@ -49,6 +57,7 @@ def to_hash if libxml2? hash_info['libxml'] = {} hash_info['libxml']['binding'] = 'extension' + hash_info['libxml']['source'] = libxml2_using_packaged? ? "packaged" : "system" hash_info['libxml']['compiled'] = compiled_parser_version hash_info['libxml']['loaded'] = loaded_parser_version hash_info['warnings'] = warnings