From 7b49c56a6a916bc6586ac558a54a2c7616430bbb Mon Sep 17 00:00:00 2001 From: Anthony Fok Date: Sat, 13 Aug 2022 11:29:01 -0600 Subject: [PATCH] snap: Make external dependencies actually work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git: - Set GIT_EXEC_PATH and include usr/lib/git-core so that git can find the git-remote-https helper (needed by e.g. "go mod download"). Go: - Put Go in its own snap part, and use stage-snaps instead of build-snaps so that it is included in the final snap. - Set GOCACHE to a writable directory. (In a previous commit, HOME is set to $SNAP_REAL_HOME which is unwritable.) Hugo: - Patch config/security/securityConfig.go "[security.exec] osEnv" whitelist (during snap build) so that external dependencies can use the required environment variables to run properly from within the snap. Asciidoctor: - Replace shebang line in asciidoctor so it can find the ruby executable. - Set RUBYLIB so that Ruby can find its libraries. Caveat/TODO: The Ruby version is hardcoded in our custom RUBYLIB. Embedded Dart Sass: - Download from GitHub and install it, for amd64 and arm64 only. Node.js: - Remove my incomplete include list so that npx is actually installed. - Set npm_config_{cache,init_module,userconfig} to writable locations. Pandoc: - Set pandoc_datadir so that Pandoc can find its data files. rst2html: - Install python3-docutils package for rst2html, rst2html5, etc. - Set PYTHONHOME so that Python can find its libraries. Note that asciidoctor, pandoc and rst2html are not in Hugo’s default "[security.exec] allow" whitelist, and the snap package does not change that default, so they still needed to be whitelisted manually in config.toml if necessary by the end user. Special thanks to Joe Mooring (@jmooring) for meticulously diagnosing the issue and providing a comprehensive test repository at https://github.com/jmooring/hugo-snap-test without which I would not have been able to understand and resolve the issue. Fixes #9078 --- snap/snapcraft.yaml | 104 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2e547980b1f..8bf0126c51f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,8 @@ name: hugo version: git +issues: https://github.com/gohugoio/hugo/issues +source-code: https://github.com/gohugoio/hugo.git +website: https://gohugo.io/ summary: Fast and Flexible Static Site Generator description: | Hugo is a static HTML and CSS website generator written in Go. It is @@ -21,17 +24,26 @@ plugs: etc-gitconfig: interface: system-files read: - - /etc/gitconfig + - /etc/gitconfig gitconfig: interface: personal-files read: - - $HOME/.gitconfig - - $HOME/.config/git/config + - $HOME/.gitconfig + - $HOME/.config/git/config + +environment: + HOME: $SNAP_REAL_HOME + GIT_EXEC_PATH: $SNAP/usr/lib/git-core + GOCACHE: $SNAP_USER_DATA/.cache/go-build + npm_config_cache: $SNAP_USER_DATA/.npm + npm_config_init_module: $SNAP_USER_DATA/.npm-init.js + npm_config_userconfig: $SNAP_USER_DATA/.npmrc + pandoc_datadir: $SNAP/usr/share/pandoc + PYTHONHOME: /usr:$SNAP/usr + RUBYLIB: $SNAP/usr/lib/ruby/vendor_ruby/2.7.0:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/ruby/vendor_ruby/2.7.0:$SNAP/usr/lib/ruby/vendor_ruby:$SNAP/usr/lib/ruby/2.7.0:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/ruby/2.7.0 apps: hugo: - environment: - HOME: $SNAP_REAL_HOME command: bin/hugo completer: hugo-completion plugs: @@ -46,28 +58,48 @@ parts: plugin: nil stage-packages: - git - organize: - usr/bin/: bin/ prime: - - bin/git + - usr/bin/git + - usr/lib + + go: + plugin: nil + stage-snaps: + - go/1.19/stable + prime: + - bin/go + - pkg/tool + - -pkg/tool/* hugo: plugin: nil - build-snaps: [go/1.19/stable] source: . + after: + - git + - go override-build: | + echo "\nStarting override-build:" set -ex - echo "\nStarting override-build:" export GOPATH=$(realpath ../go) export PATH=$GOPATH/bin:$PATH + echo " * Patch securityConfig.go to allow" + echo " - GIT_EXEC_PATH and LD_LIBRARY_PATH to be passed to git" + echo " - npm_config_{cache,init_module,userconfig} to be passed to npx" + echo " - pandoc_datadir to be passed to pandoc" + echo " - PYTHONHOME and SNAP to be passed to rst2html" + echo " - RUBYLIB to be passed to asciidoctor" + sed -i '/OsEnv: NewWhitelist/s/)\$/|GIT_EXEC_PATH|LD_LIBRARY_PATH|npm_config_(cache|init_module|userconfig)|pandoc_datadir|PYTHONHOME|RUBYLIB|SNAP&/' config/security/securityConfig.go + git diff config/security/securityConfig.go + echo " * SNAPCRAFT_IMAGE_INFO=${SNAPCRAFT_IMAGE_INFO=}" # Example: SNAPCRAFT_IMAGE_INFO='{"build_url": "https://launchpad.net/~gohugoio/+snap/hugo-extended-dev/+build/344022"}' export HUGO_BUILD_TAGS="" if echo $SNAPCRAFT_IMAGE_INFO | grep -q '/+snap/hugo-extended'; then HUGO_BUILD_TAGS="extended" fi + echo " * Building hugo (HUGO_BUILD_TAGS=\"$HUGO_BUILD_TAGS\")..." go build -v -ldflags '-X github.com/gohugoio/hugo/common/hugo.vendorInfo=snap' -tags "$HUGO_BUILD_TAGS" ./hugo version @@ -86,25 +118,53 @@ parts: strip --remove-section=.comment --remove-section=.note $SNAPCRAFT_PART_INSTALL/bin/hugo ls -l $SNAPCRAFT_PART_INSTALL/bin/hugo + asciidoctor: + plugin: nil + stage-packages: + - asciidoctor + override-build: | + set -ex + snapcraftctl build + sed -i '1s|#!/usr/bin/ruby|#!/usr/bin/env ruby|' $SNAPCRAFT_PART_INSTALL/usr/bin/asciidoctor + + dart-sass-embedded: + plugin: nil + build-packages: + - curl + override-build: | + set -ex + snapcraftctl build + case "$SNAPCRAFT_TARGET_ARCH" in + amd64) arch=x64 ;; + arm64) arch=arm64 ;; + i386) arch=ia32 ;; + *) arch="" ;; + esac + if [[ -n $arch ]]; then + url=$(curl -s https://api.github.com/repos/sass/dart-sass-embedded/releases/latest | awk -F\" "/browser_download_url.*-linux-${arch}.tar.gz/{print \$(NF-1)}") + curl -LO --retry-connrefused --retry 10 "$url" + tar xf sass_embedded-*-linux-$arch.tar.gz sass_embedded/dart-sass-embedded + install -d $SNAPCRAFT_PART_INSTALL/bin + cp -av sass_embedded/dart-sass-embedded $SNAPCRAFT_PART_INSTALL/bin/ + fi + node: plugin: nil stage-packages: - nodejs - organize: - usr/bin/: bin/ - usr/lib/: lib/ - prime: - - bin/node - - lib/*/lib*.so* pandoc: plugin: nil stage-packages: - - libatomic1 - pandoc - - pandoc-data + + rst2html: + plugin: nil + stage-packages: + - python3-docutils + override-build: | + set -ex + snapcraftctl build + sed -i "s|'/usr/share/docutils/'|os.path.expandvars('\$SNAP/usr/share/docutils/')|" $SNAPCRAFT_PART_INSTALL/usr/lib/python3/dist-packages/docutils/__init__.py organize: - usr/bin/: bin/ - usr/lib/: lib/ - prime: - - bin/pandoc + usr/share/docutils/scripts/python3: usr/bin