From 67b8ac020833a14d98e8a147b8aae5caf2288a41 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 20 Nov 2022 17:24:52 -0500 Subject: [PATCH] build: separate stdlib manifests by version (#47596) We install these by version, so it helps a lot if they reflect that in the manifest name. Also some other mild cleanup. --- Make.inc | 17 ---------------- Makefile | 38 ++++++++++++++++++++++++----------- cli/Makefile | 3 +-- deps/tools/common.mk | 15 +++++++++----- deps/tools/git-external.mk | 3 ++- deps/tools/stdlib-external.mk | 9 +++++++-- deps/tools/uninstallers.mk | 1 + stdlib/Makefile | 16 +++++++++------ sysimage.mk | 4 ++-- 9 files changed, 59 insertions(+), 47 deletions(-) diff --git a/Make.inc b/Make.inc index 40caeffb9d313..24f5964e9dcc0 100644 --- a/Make.inc +++ b/Make.inc @@ -135,23 +135,6 @@ endif export BUILDROOT unexport O -# Make sure the user didn't try to specify a path that will confuse the shell / make -METACHARACTERS := ][?*{}() $$%:;&|!\#,\\`\": -ifneq (,$(findstring ',$(value BUILDROOT))) -$(error cowardly refusing to build into directory with a single-quote in the path) -endif -ifneq (,$(findstring ',$(value JULIAHOME))) -$(error cowardly refusing to build from source directory with a single-quote in the path) -endif -ifneq (,$(shell echo '$(value BUILDROOT)' | grep '[$(METACHARACTERS)]')) -$(error cowardly refusing to build into directory with a shell-metacharacter in the path\ - (got: $(value BUILDROOT))) -endif -ifneq (,$(shell echo '$(value JULIAHOME)' | grep '[$(METACHARACTERS)]')) -$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\ - (got: $(value JULIAHOME))) -endif - # we include twice to pickup user definitions better # include from JULIAHOME first so that BUILDROOT can override MAYBE_HOST := diff --git a/Makefile b/Makefile index 15a8cd1c855f9..3f81263b26ab2 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,23 @@ include $(JULIAHOME)/Make.inc # import LLVM_SHARED_LIB_NAME include $(JULIAHOME)/deps/llvm-ver.make +# Make sure the user didn't try to build in a path that will confuse the shell or make +METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\":]\|/\./\|/\.\./ +ifneq (,$(findstring ',$(value BUILDROOT))) +$(error cowardly refusing to build into directory with a single-quote in the path) +endif +ifneq (,$(findstring ',$(value JULIAHOME))) +$(error cowardly refusing to build from source directory with a single-quote in the path) +endif +ifneq (,$(shell echo '$(value BUILDROOT)/' | grep '$(METACHARACTERS)')) +$(error cowardly refusing to build into directory with a shell-metacharacter in the path\ + (got: $(value BUILDROOT))) +endif +ifneq (,$(shell echo '$(value JULIAHOME)/' | grep '$(METACHARACTERS)')) +$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\ + (got: $(value JULIAHOME))) +endif + VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" @@ -13,7 +30,7 @@ DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_priva ifneq ($(BUILDROOT),$(JULIAHOME)) BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/llvmpasses) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk -DIRS := $(DIRS) $(BUILDDIRS) +DIRS += $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @# add Makefiles to the build directories for convenience (pointing back to the source location of each) @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ @@ -44,10 +61,6 @@ $(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(bu julia_flisp.boot.inc.phony: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony -# Build the HTML docs (skipped if already exists, notably in tarballs) -$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print) - @$(MAKE) docs - julia-symlink: julia-cli-$(JULIA_BUILD_MODE) ifeq ($(OS),WINNT) echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat @@ -236,7 +249,7 @@ define stringreplace endef -install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html +install: $(build_depsbindir)/stringreplace docs @$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE) @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \ mkdir -p $(DESTDIR)$$subdir; \ @@ -484,7 +497,7 @@ app: darwinframework: $(MAKE) -C $(JULIAHOME)/contrib/mac/framework -light-source-dist.tmp: $(BUILDROOT)/doc/_build/html/en/index.html +light-source-dist.tmp: docs ifneq ($(BUILDROOT),$(JULIAHOME)) $(error make light-source-dist does not work in out-of-tree builds) endif @@ -561,12 +574,13 @@ distcleanall: cleanall @-$(MAKE) -C $(BUILDROOT)/deps distcleanall @-$(MAKE) -C $(BUILDROOT)/doc cleanall -.PHONY: default debug release check-whitespace release-candidate \ +.FORCE: +.PHONY: .FORCE default debug release check-whitespace release-candidate \ julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \ julia-cli-release julia-cli-debug julia-src-release julia-src-debug \ julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \ - test testall testall1 test test-* test-revise-* \ - clean distcleanall cleanall clean-* \ + test testall testall1 test \ + clean distcleanall cleanall $(CLEAN_TARGETS) \ run-julia run-julia-debug run-julia-release run \ install binary-dist light-source-dist.tmp light-source-dist \ dist full-source-dist source-dist @@ -583,12 +597,12 @@ testall: check-whitespace $(JULIA_BUILD_MODE) testall1: check-whitespace $(JULIA_BUILD_MODE) @env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) -test-%: check-whitespace $(JULIA_BUILD_MODE) +test-%: check-whitespace $(JULIA_BUILD_MODE) .FORCE @([ $$(( $$(date +%s) - $$(date -r $(build_private_libdir)/sys.$(SHLIB_EXT) +%s) )) -le 100 ] && \ printf '\033[93m HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. \033[0m\n') || true @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) -test-revise-%: +test-revise-%: .FORCE @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test revise-$* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) # download target for some hardcoded windows dependencies diff --git a/cli/Makefile b/cli/Makefile index 58c1f82f48662..dfe7b594ee46e 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -70,8 +70,7 @@ dump-trampolines: $(SRCDIR)/trampolines/trampolines_$(ARCH).S $(CC) $(SHIPFLAGS) $(LOADER_CFLAGS) $< -S | sed -E 's/ ((%%)|;) /\n/g' | sed -E 's/.global/\n.global/g' DIRS = $(build_bindir) $(build_libdir) -$(DIRS): - @mkdir -p $@ +$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) ifeq ($(OS),WINNT) $(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION diff --git a/deps/tools/common.mk b/deps/tools/common.mk index b09786682b941..c19886114c14e 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -107,8 +107,8 @@ endif DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging) $(build_prefix)/manifest) $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) - $(build_prefix): | $(DIRS) + $(eval $(call dir_target,$(SRCCACHE))) @@ -174,6 +174,7 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tar | $(build_prefix $(UNTAR) $$< -C $$(build_prefix) $6 echo '$$(UNINSTALL_$(strip $1))' > $$@ +.PHONY: $(addsuffix -$(strip $1),stage install distclean uninstall reinstall) endef define staged-uninstaller @@ -192,14 +193,18 @@ endef define symlink_install # (target-name, rel-from, abs-to) clean-$1: uninstall-$1 install-$1: $$(build_prefix)/manifest/$1 -reinstall-$1: install-$1 +reinstall-$1: + +$$(MAKE) uninstall-$1 + +$$(MAKE) stage-$1 + +$$(MAKE) install-$1 +.PHONY: $(addsuffix -$1,clean install reinstall) UNINSTALL_$(strip $1) := $2 symlink-uninstaller $3 -$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest +$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $$(abspath $$(dir $3/$1)) $$(abspath $$(dir $$(build_prefix)/manifest/$1)) -+[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1 ifeq ($$(BUILD_OS), WINNT) - cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,) + cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,) else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS))) cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2) else ifdef JULIA_VAGRANT_BUILD @@ -213,7 +218,7 @@ endef define symlink-uninstaller uninstall-$1: ifeq ($$(BUILD_OS), WINNT) - -cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&) + -cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&) else rm -rf $3/$1 endif diff --git a/deps/tools/git-external.mk b/deps/tools/git-external.mk index 65b40b87ee937..cf1610ac1bf5d 100644 --- a/deps/tools/git-external.mk +++ b/deps/tools/git-external.mk @@ -68,11 +68,12 @@ $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE) $(TAR) -C $$(dir $$@) --strip-components 1 -xf $$< echo 1 > $$@ -checksum-$(1): $$($2_SRC_FILE) +checksum-$1: $$($2_SRC_FILE) $$(JLCHECKSUM) $$< endif # DEPS_GIT $$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets) distclean-$1: rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR) +.PHONY: $(addsuffix -$1,checksum distclean) endef diff --git a/deps/tools/stdlib-external.mk b/deps/tools/stdlib-external.mk index 60f50b56ee2e0..0a99111605a45 100644 --- a/deps/tools/stdlib-external.mk +++ b/deps/tools/stdlib-external.mk @@ -16,12 +16,17 @@ $$(eval $$(call git-external,$1,$2,,,$$(BUILDDIR))) $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted @# no build steps echo 1 > $$@ -$$(eval $$(call symlink_install,$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib/$$$$(VERSDIR))) +$$(eval $$(call symlink_install,$$$$(VERSDIR)/$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib)) clean-$1: -rm -f $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled get-$1: $$($2_SRC_FILE) extract-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted configure-$1: extract-$1 compile-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled - +install-$1: install-$$(VERSDIR)/$1 +uninstall-$1: uninstall-$$(VERSDIR)/$1 +reinstall-$1: reinstall-$$(VERSDIR)/$1 +version-check-$1: version-check-$$(VERSDIR)/$1 +clean-$1: clean-$$(VERSDIR)/$1 +.PHONY: $(addsuffix -$1,get extract configure compile install uninstall reinstall clean) endef diff --git a/deps/tools/uninstallers.mk b/deps/tools/uninstallers.mk index 48387914643db..0051786ed1d0a 100644 --- a/deps/tools/uninstallers.mk +++ b/deps/tools/uninstallers.mk @@ -17,6 +17,7 @@ else uninstall-$1: @echo "skipping uninstall: $1 not installed" endif +.PHONY: uninstall-$1 endef $(foreach dep,$(DEP_LIBS_STAGED_ALL),$(eval $(call define-uninstaller,$(dep)))) diff --git a/stdlib/Makefile b/stdlib/Makefile index 7957520c31ea3..427bf7fe29ec7 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -14,9 +14,8 @@ include $(JULIAHOME)/deps/*.version VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) - -$(build_datarootdir)/julia/stdlib/$(VERSDIR): - mkdir -p $@ +DIRS := $(build_datarootdir)/julia/stdlib/$(VERSDIR) $(build_prefix)/manifest/$(VERSDIR) +$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \ BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \ @@ -43,14 +42,19 @@ $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll)))) STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \ Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \ - Markdown Mmap Printf Profile Random REPL Serialization SHA \ - SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \ + Markdown Mmap Printf Profile Random REPL Serialization \ + SharedArrays Sockets Test TOML Unicode UUIDs \ $(JLL_NAMES) STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z)))) +ifneq ($(filter $(STDLIBS),$(STDLIBS_EXT)),) +$(error ERROR duplicated STDLIBS in list) +endif + + # Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/ $(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module)))) @@ -68,5 +72,5 @@ clean: $(addprefix clean-, $(STDLIBS_EXT)) $(CLEAN_TARGETS) extstdlibclean distclean: $(addprefix distclean-, $(STDLIBS_EXT)) clean checksumall: $(addprefix checksum-, $(STDLIBS_EXT)) -DEP_LIBS_STAGED_ALL := $(STDLIBS_EXT) +DEP_LIBS_STAGED_ALL := $(addprefix $(VERSDIR)/,$(STDLIBS_EXT)) include $(JULIAHOME)/deps/tools/uninstallers.mk diff --git a/sysimage.mk b/sysimage.mk index 2d154672d8130..b426a74454b1d 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -10,7 +10,7 @@ sysimg-bc: $(build_private_libdir)/sys-bc.a sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT) sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) -VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` +VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%-o.a @$(call PRINT_LINK, $(CXX) $(LDFLAGS) -shared $(fPIC) -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ \ @@ -54,7 +54,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl) BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) \ - $(build_prefix)/manifest/Pkg + $(wildcard $(build_prefix)/manifest/$(VERSDIR)/*) RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS)