From 2e8c0db9f8d5d53dd80916cd7df5739ecb3494bf Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:17:28 +0100 Subject: [PATCH 01/12] package/lowpowd: Modernize and move to opendingux --- board/opendingux/Config.in | 4 ++++ .../opendingux/package}/lowpowd/Config.in | 0 .../opendingux/package}/lowpowd/S99lowpowd.sh | 0 .../opendingux/package}/lowpowd/lowpowd.c | 0 board/opendingux/package/lowpowd/lowpowd.mk | 13 ++++++++++++ package/Config.in | 1 - package/lowpowd/lowpowd.mk | 21 ------------------- 7 files changed, 17 insertions(+), 22 deletions(-) rename {package => board/opendingux/package}/lowpowd/Config.in (100%) rename {package => board/opendingux/package}/lowpowd/S99lowpowd.sh (100%) rename {package => board/opendingux/package}/lowpowd/lowpowd.c (100%) create mode 100644 board/opendingux/package/lowpowd/lowpowd.mk delete mode 100644 package/lowpowd/lowpowd.mk diff --git a/board/opendingux/Config.in b/board/opendingux/Config.in index fc0c3a0586..81404dc626 100644 --- a/board/opendingux/Config.in +++ b/board/opendingux/Config.in @@ -25,4 +25,8 @@ comment "System tools" source "$BR2_EXTERNAL/package/dingux-commander/Config.in" source "$BR2_EXTERNAL/package/od-network-config/Config.in" source "$BR2_EXTERNAL/package/pwswd/Config.in" + +comment "Hardware handling" + source "$BR2_EXTERNAL/package/lowpowd/Config.in" + endmenu diff --git a/package/lowpowd/Config.in b/board/opendingux/package/lowpowd/Config.in similarity index 100% rename from package/lowpowd/Config.in rename to board/opendingux/package/lowpowd/Config.in diff --git a/package/lowpowd/S99lowpowd.sh b/board/opendingux/package/lowpowd/S99lowpowd.sh similarity index 100% rename from package/lowpowd/S99lowpowd.sh rename to board/opendingux/package/lowpowd/S99lowpowd.sh diff --git a/package/lowpowd/lowpowd.c b/board/opendingux/package/lowpowd/lowpowd.c similarity index 100% rename from package/lowpowd/lowpowd.c rename to board/opendingux/package/lowpowd/lowpowd.c diff --git a/board/opendingux/package/lowpowd/lowpowd.mk b/board/opendingux/package/lowpowd/lowpowd.mk new file mode 100644 index 0000000000..7b346f1282 --- /dev/null +++ b/board/opendingux/package/lowpowd/lowpowd.mk @@ -0,0 +1,13 @@ +LOWPOWD_SITE = $(BR2_EXTERNAL)/package/lowpowd +LOWPOWD_SITE_METHOD = local + +define LOWPOWD_BUILD_CMDS + $(TARGET_CC) $(TARGET_CFLAGS) $(@D)/lowpowd.c -o $(@D)/lowpowd +endef + +define LOWPOWD_INSTALL_TARGET_CMDS + $(INSTALL) -m 0755 -D $(@D)/lowpowd $(TARGET_DIR)/usr/sbin/lowpowd + $(INSTALL) -m 0755 -D $(@D)/S99lowpowd.sh $(TARGET_DIR)/etc/init.d/S99lowpowd.sh +endef + +$(eval $(generic-package)) diff --git a/package/Config.in b/package/Config.in index 643576151f..32192fb76f 100755 --- a/package/Config.in +++ b/package/Config.in @@ -341,7 +341,6 @@ endif source "package/lcdproc/Config.in" source "package/linux-console-tools/Config.in" source "package/lm-sensors/Config.in" - source "package/lowpowd/Config.in" source "package/lshw/Config.in" source "package/lsuio/Config.in" source "package/lvm2/Config.in" diff --git a/package/lowpowd/lowpowd.mk b/package/lowpowd/lowpowd.mk deleted file mode 100644 index 9551686a44..0000000000 --- a/package/lowpowd/lowpowd.mk +++ /dev/null @@ -1,21 +0,0 @@ -LOWPOWD_DIR := $(BUILD_DIR)/lowpowd - -$(LOWPOWD_DIR)/lowpowd.c: - mkdir -p $(LOWPOWD_DIR) - cp package/lowpowd/lowpowd.c $@ - -$(LOWPOWD_DIR)/lowpowd: $(LOWPOWD_DIR)/lowpowd.c - $(TARGET_CC) $(TARGET_CFLAGS) $^ -o $@ - -$(TARGET_DIR)/usr/sbin/lowpowd: $(LOWPOWD_DIR)/lowpowd - $(INSTALL) -m 755 $^ $@ - -$(TARGET_DIR)/etc/init.d/S99lowpowd.sh: package/lowpowd/S99lowpowd.sh - $(INSTALL) -m 755 $^ $@ - -.PHONY: lowpowd -lowpowd: $(TARGET_DIR)/usr/sbin/lowpowd $(TARGET_DIR)/etc/init.d/S99lowpowd.sh - -ifeq ($(BR2_PACKAGE_LOWPOWD),y) -TARGETS += lowpowd -endif From 8686b9924e7f4e3ee50ff4f381067e283d847f1e Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:21:18 +0100 Subject: [PATCH 02/12] package/unlockvt: Modernize and move to opendingux --- board/opendingux/Config.in | 1 + .../opendingux/package}/unlockvt/Config.in | 0 .../opendingux/package}/unlockvt/unlockvt.c | 0 board/opendingux/package/unlockvt/unlockvt.mk | 12 ++++++++++++ package/Config.in | 1 - package/unlockvt/unlockvt.mk | 18 ------------------ 6 files changed, 13 insertions(+), 19 deletions(-) rename {package => board/opendingux/package}/unlockvt/Config.in (100%) rename {package => board/opendingux/package}/unlockvt/unlockvt.c (100%) create mode 100644 board/opendingux/package/unlockvt/unlockvt.mk delete mode 100644 package/unlockvt/unlockvt.mk diff --git a/board/opendingux/Config.in b/board/opendingux/Config.in index 81404dc626..b91a354d2e 100644 --- a/board/opendingux/Config.in +++ b/board/opendingux/Config.in @@ -28,5 +28,6 @@ comment "System tools" comment "Hardware handling" source "$BR2_EXTERNAL/package/lowpowd/Config.in" + source "$BR2_EXTERNAL/package/unlockvt/Config.in" endmenu diff --git a/package/unlockvt/Config.in b/board/opendingux/package/unlockvt/Config.in similarity index 100% rename from package/unlockvt/Config.in rename to board/opendingux/package/unlockvt/Config.in diff --git a/package/unlockvt/unlockvt.c b/board/opendingux/package/unlockvt/unlockvt.c similarity index 100% rename from package/unlockvt/unlockvt.c rename to board/opendingux/package/unlockvt/unlockvt.c diff --git a/board/opendingux/package/unlockvt/unlockvt.mk b/board/opendingux/package/unlockvt/unlockvt.mk new file mode 100644 index 0000000000..1a41e74d6d --- /dev/null +++ b/board/opendingux/package/unlockvt/unlockvt.mk @@ -0,0 +1,12 @@ +UNLOCKVT_SITE = $(BR2_EXTERNAL)/package/unlockvt +UNLOCKVT_SITE_METHOD = local + +define UNLOCKVT_BUILD_CMDS + $(TARGET_CC) $(TARGET_CFLAGS) $(@D)/unlockvt.c -o $(@D)/unlockvt +endef + +define UNLOCKVT_INSTALL_TARGET_CMDS + $(INSTALL) -m 0755 -D $(@D)/unlockvt $(TARGET_DIR)/usr/sbin/unlockvt +endef + +$(eval $(generic-package)) diff --git a/package/Config.in b/package/Config.in index 32192fb76f..6587779fc9 100755 --- a/package/Config.in +++ b/package/Config.in @@ -225,7 +225,6 @@ endif source "package/sdl2_net/Config.in" source "package/sdl2_ttf/Config.in" source "package/timidity-instruments/Config.in" - source "package/unlockvt/Config.in" comment "Other GUIs" source "package/efl/Config.in" diff --git a/package/unlockvt/unlockvt.mk b/package/unlockvt/unlockvt.mk deleted file mode 100644 index 03732a64c3..0000000000 --- a/package/unlockvt/unlockvt.mk +++ /dev/null @@ -1,18 +0,0 @@ -UNLOCKVT_DIR=$(BUILD_DIR)/unlockvt - -$(UNLOCKVT_DIR)/unlockvt.c: - rm -rf $(UNLOCKVT_DIR) - mkdir $(UNLOCKVT_DIR) - cp package/unlockvt/unlockvt.c $(UNLOCKVT_DIR) - -$(UNLOCKVT_DIR)/unlockvt: $(UNLOCKVT_DIR)/unlockvt.c - $(TARGET_CC) $(TARGET_CFLAGS) -s $^ -o $@ - -$(TARGET_DIR)/usr/sbin/unlockvt: $(UNLOCKVT_DIR)/unlockvt - $(INSTALL) -m 755 $^ $@ - -unlockvt: $(TARGET_DIR)/usr/sbin/unlockvt - -ifeq ($(BR2_PACKAGE_UNLOCKVT),y) -TARGETS += unlockvt -endif From 2ff04e6e429481f40ea7320630a7990ee5f6e711 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:40:11 +0100 Subject: [PATCH 03/12] Remove trailing slashes from package URLs Trailing slashes are not allowed in the new buildroot infra --- package/bar/bar.mk | 2 +- package/dialog/dialog.mk | 2 +- package/hostapd/hostapd.mk | 2 +- package/i2c-tools/i2c-tools.mk | 2 +- package/libevdev/libevdev.mk | 2 +- package/patchelf/patchelf.mk | 2 +- package/physfs/physfs.mk | 2 +- package/sdl2_mixer/sdl2_mixer.mk | 2 +- package/sdl2_ttf/sdl2_ttf.mk | 2 +- package/timidity-instruments/timidity-instruments.mk | 6 +++--- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package/bar/bar.mk b/package/bar/bar.mk index 31a4705836..b2e72405be 100644 --- a/package/bar/bar.mk +++ b/package/bar/bar.mk @@ -5,7 +5,7 @@ ############################################################# BAR_VERSION = 1.4 BAR_SOURCE = bar-$(BAR_VERSION)-src.tar.bz2 -BAR_SITE = http://www.theiling.de/downloads/ +BAR_SITE = http://www.theiling.de/downloads define BAR_INSTALL_TARGET_CMDS $(INSTALL) -m 0755 $(@D)/bar $(TARGET_DIR)/usr/bin/bar diff --git a/package/dialog/dialog.mk b/package/dialog/dialog.mk index 6346638d6e..a773ab715c 100644 --- a/package/dialog/dialog.mk +++ b/package/dialog/dialog.mk @@ -6,7 +6,7 @@ DIALOG_VERSION = 1.2-20140219 DIALOG_SOURCE = dialog-$(DIALOG_VERSION).tgz -DIALOG_SITE = http://sources.buildroot.net/ +DIALOG_SITE = http://sources.buildroot.net DIALOG_CONF_OPTS = --with-ncurses --with-curses-dir=$(STAGING_DIR)/usr \ --disable-rpath-hack DIALOG_DEPENDENCIES = host-pkgconf ncurses diff --git a/package/hostapd/hostapd.mk b/package/hostapd/hostapd.mk index 9f9a6a4d9a..7c690d19cb 100644 --- a/package/hostapd/hostapd.mk +++ b/package/hostapd/hostapd.mk @@ -5,7 +5,7 @@ ################################################################################ HOSTAPD_VERSION = 2.8 -HOSTAPD_SITE = https://w1.fi/releases/ +HOSTAPD_SITE = https://w1.fi/releases HOSTAPD_SUBDIR = hostapd HOSTAPD_CONFIG = $(HOSTAPD_DIR)/$(HOSTAPD_SUBDIR)/.config HOSTAPD_DEPENDENCIES = libnl diff --git a/package/i2c-tools/i2c-tools.mk b/package/i2c-tools/i2c-tools.mk index 392b19199a..2a0975806d 100644 --- a/package/i2c-tools/i2c-tools.mk +++ b/package/i2c-tools/i2c-tools.mk @@ -6,7 +6,7 @@ I2C_TOOLS_VERSION = 4.0 I2C_TOOLS_SOURCE = i2c-tools-$(I2C_TOOLS_VERSION).tar.xz -I2C_TOOLS_SITE = http://sources.buildroot.net/i2c-tools/ +I2C_TOOLS_SITE = http://sources.buildroot.net/i2c-tools I2C_TOOLS_LICENSE = GPLv2+, GPLv2 (py-smbus) I2C_TOOLS_LICENSE_FILES = COPYING diff --git a/package/libevdev/libevdev.mk b/package/libevdev/libevdev.mk index fde7d3ce93..2c60f73f55 100644 --- a/package/libevdev/libevdev.mk +++ b/package/libevdev/libevdev.mk @@ -5,7 +5,7 @@ ################################################################################ LIBEVDEV_VERSION = 1.2.2 -LIBEVDEV_SITE = http://www.freedesktop.org/software/libevdev/ +LIBEVDEV_SITE = http://www.freedesktop.org/software/libevdev LIBEVDEV_SOURCE = libevdev-$(LIBEVDEV_VERSION).tar.xz LIBEVDEV_LICENSE = X11 LIBEVDEV_LICENSE_FILES = COPYING diff --git a/package/patchelf/patchelf.mk b/package/patchelf/patchelf.mk index b6b9e05e2e..043224dab2 100644 --- a/package/patchelf/patchelf.mk +++ b/package/patchelf/patchelf.mk @@ -5,7 +5,7 @@ ################################################################################ PATCHELF_VERSION = 0.8 -PATCHELF_SITE = http://releases.nixos.org/patchelf/patchelf-0.8/ +PATCHELF_SITE = http://releases.nixos.org/patchelf/patchelf-0.8 PATCHELF_LICENSE = GPLv3+ PATCHELF_LICENSE_FILES = COPYING diff --git a/package/physfs/physfs.mk b/package/physfs/physfs.mk index 53ccaab682..3682c3f675 100644 --- a/package/physfs/physfs.mk +++ b/package/physfs/physfs.mk @@ -6,7 +6,7 @@ PHYSFS_VERSION = 2.0.3 PHYSFS_SOURCE = physfs-$(PHYSFS_VERSION).tar.bz2 -PHYSFS_SITE = https://icculus.org/physfs/downloads/ +PHYSFS_SITE = https://icculus.org/physfs/downloads PHYSFS_INSTALL_STAGING = YES PHYSFS_DEPENDENCIES = zlib diff --git a/package/sdl2_mixer/sdl2_mixer.mk b/package/sdl2_mixer/sdl2_mixer.mk index 2aa35e5d8f..c4f496f6b5 100644 --- a/package/sdl2_mixer/sdl2_mixer.mk +++ b/package/sdl2_mixer/sdl2_mixer.mk @@ -6,7 +6,7 @@ SDL2_MIXER_VERSION = 2.0.0 SDL2_MIXER_SOURCE = SDL2_mixer-$(SDL2_MIXER_VERSION).tar.gz -SDL2_MIXER_SITE = http://www.libsdl.org/projects/SDL_mixer/release/ +SDL2_MIXER_SITE = http://www.libsdl.org/projects/SDL_mixer/release SDL2_MIXER_INSTALL_STAGING = YES SDL2_MIXER_LICENSE = zlib SDL2_MIXER_LICENSE_FILES = COPYING.txt diff --git a/package/sdl2_ttf/sdl2_ttf.mk b/package/sdl2_ttf/sdl2_ttf.mk index 7253966c33..8639d2c3ec 100644 --- a/package/sdl2_ttf/sdl2_ttf.mk +++ b/package/sdl2_ttf/sdl2_ttf.mk @@ -6,7 +6,7 @@ SDL2_TTF_VERSION = 2.0.14 SDL2_TTF_SOURCE = SDL2_ttf-$(SDL2_TTF_VERSION).tar.gz -SDL2_TTF_SITE = http://www.libsdl.org/projects/SDL_ttf/release/ +SDL2_TTF_SITE = http://www.libsdl.org/projects/SDL_ttf/release SDL2_TTF_INSTALL_STAGING = YES SDL2_TTF_LICENSE = zlib SDL2_TTF_LICENSE_FILES = COPYING.txt diff --git a/package/timidity-instruments/timidity-instruments.mk b/package/timidity-instruments/timidity-instruments.mk index 6fbb22f7c9..372781b92a 100644 --- a/package/timidity-instruments/timidity-instruments.mk +++ b/package/timidity-instruments/timidity-instruments.mk @@ -3,9 +3,9 @@ # Timidity instruments # ############################################################# -TIMIDITY_INSTRUMENTS_VERSION:=unversioned -TIMIDITY_INSTRUMENTS_SOURCE:=timidity.tar.gz -TIMIDITY_INSTRUMENTS_SITE:=http://www.libsdl.org/projects/SDL_mixer/timidity/ +TIMIDITY_INSTRUMENTS_VERSION = unversioned +TIMIDITY_INSTRUMENTS_SOURCE = timidity.tar.gz +TIMIDITY_INSTRUMENTS_SITE = http://www.libsdl.org/projects/SDL_mixer/timidity define TIMIDITY_INSTRUMENTS_INSTALL_TARGET_CMDS install -d -m 755 $(TARGET_DIR)/usr/share/midi/instruments From e49ecf349e136e5ddb5f87dd34c82974863cae56 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:44:14 +0100 Subject: [PATCH 04/12] Rename remaining _OPT vars to _OPTS Required by upstream pkg-generic.mk --- docs/manual/adding-packages-autotools.txt | 4 ++-- docs/manual/adding-packages-kconfig.txt | 2 +- docs/manual/adding-packages-luarocks.txt | 6 +++--- docs/manual/adding-packages-perl.txt | 2 +- docs/manual/adding-packages-python.txt | 4 ++-- package/busybox/busybox.mk | 2 +- .../gstreamer/gst-plugin-x170/gst-plugin-x170.mk | 2 +- .../gst1-plugins-bad/gst1-plugins-bad.mk | 2 +- package/lightning/lightning.mk | 2 +- package/pkg-autotools.mk | 8 ++++---- package/pkg-cmake.mk | 4 ++-- package/pkg-kconfig.mk | 6 +++--- package/pkg-luarocks.mk | 4 ++-- package/pkg-perl.mk | 8 ++++---- package/pkg-python.mk | 14 +++++++------- package/python-numpy/python-numpy.mk | 2 +- package/uclibc/uclibc.mk | 2 +- package/webkit/webkit.mk | 2 +- 18 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/manual/adding-packages-autotools.txt b/docs/manual/adding-packages-autotools.txt index b48cfcb6b5..ece6c49563 100644 --- a/docs/manual/adding-packages-autotools.txt +++ b/docs/manual/adding-packages-autotools.txt @@ -126,7 +126,7 @@ cases, typical packages will therefore only use a few of them. +LIBFOO_AUTORECONF=YES+. These are passed in the environment of the 'autoreconf' command. By default, empty. -* +LIBFOO_AUTORECONF_OPT+ to specify additional options +* +LIBFOO_AUTORECONF_OPTS+ to specify additional options passed to the 'autoreconf' program if +LIBFOO_AUTORECONF=YES+. By default, empty. @@ -136,7 +136,7 @@ cases, typical packages will therefore only use a few of them. 'gettextize'.) Only valid when +LIBFOO_AUTORECONF=YES+. Valid values are +YES+ and +NO+. The default is +NO+. -* +LIBFOO_GETTEXTIZE_OPT+, to specify additional options passed to +* +LIBFOO_GETTEXTIZE_OPTS+, to specify additional options passed to the 'gettextize' program, if +LIBFOO_GETTEXTIZE=YES+. You may use that if, for example, the +.po+ files are not located in the standard place (i.e. in +po/+ at the root of the package.) By diff --git a/docs/manual/adding-packages-kconfig.txt b/docs/manual/adding-packages-kconfig.txt index 2a78316137..55f0571fa9 100644 --- a/docs/manual/adding-packages-kconfig.txt +++ b/docs/manual/adding-packages-kconfig.txt @@ -46,7 +46,7 @@ be set to suit the needs of the package under consideration: * +FOO_KCONFIG_EDITORS+: a space-separated list of kconfig editors to support, for example 'menuconfig xconfig'. By default, 'menuconfig'. -* +FOO_KCONFIG_OPT+: extra options to pass when calling the kconfig +* +FOO_KCONFIG_OPTS+: extra options to pass when calling the kconfig ediftors. This may need to include '$(FOO_MAKE_OPTS)', for example. By default, empty. diff --git a/docs/manual/adding-packages-luarocks.txt b/docs/manual/adding-packages-luarocks.txt index 10137a8ef6..c914238f0b 100644 --- a/docs/manual/adding-packages-luarocks.txt +++ b/docs/manual/adding-packages-luarocks.txt @@ -20,8 +20,8 @@ with an example : 07: LUAFOO_VERSION = 1.0.2-1 08: LUAFOO_DEPENDENCIES = foo 09: -10: LUAFOO_BUILD_OPT += FOO_INCDIR=$(STAGING_DIR)/usr/include -11: LUAFOO_BUILD_OPT += FOO_LIBDIR=$(STAGING_DIR)/usr/lib +10: LUAFOO_BUILD_OPTS += FOO_INCDIR=$(STAGING_DIR)/usr/include +11: LUAFOO_BUILD_OPTS += FOO_LIBDIR=$(STAGING_DIR)/usr/lib 12: LUAFOO_LICENSE = luaFoo license 13: LUAFOO_LICENSE_FILES = COPYING 14: @@ -86,5 +86,5 @@ also defined. They can be overridden in specific cases. * +LUAFOO_SUBDIR+, which defaults to +luafoo-$(LUAFOO_VERSION_WITHOUT_ROCKSPEC_REVISION)+ -* +LUAFOO_BUILD_OPT+ contains additional build options for the +* +LUAFOO_BUILD_OPTS+ contains additional build options for the +luarocks build+ call. diff --git a/docs/manual/adding-packages-perl.txt b/docs/manual/adding-packages-perl.txt index 6c44f98afb..63fafe6558 100644 --- a/docs/manual/adding-packages-perl.txt +++ b/docs/manual/adding-packages-perl.txt @@ -105,7 +105,7 @@ cases, typical packages will therefore only use a few of them. configure options to pass to the +perl Makefile.PL+ or +perl Build.PL+. By default, empty. -* +PERL_FOO_BUILD_OPT+/+HOST_PERL_FOO_BUILD_OPT+, to specify additional +* +PERL_FOO_BUILD_OPTS+/+HOST_PERL_FOO_BUILD_OPTS+, to specify additional options to pass to +make pure_all+ or +perl Build build+ in the build step. By default, empty. diff --git a/docs/manual/adding-packages-python.txt b/docs/manual/adding-packages-python.txt index bca0f6a62e..3cbb87d069 100644 --- a/docs/manual/adding-packages-python.txt +++ b/docs/manual/adding-packages-python.txt @@ -122,9 +122,9 @@ therefore only use a few of them, or none. setuptools target packages) and +HOST_PKG_PYTHON_SETUPTOOLS_ENV+ (for setuptools host packages). -* +PYTHON_FOO_BUILD_OPT+, to specify additional options to pass to the +* +PYTHON_FOO_BUILD_OPTS+, to specify additional options to pass to the Python +setup.py+ script during the build step. For target distutils - packages, the +PKG_PYTHON_DISTUTILS_BUILD_OPT+ options are already + packages, the +PKG_PYTHON_DISTUTILS_BUILD_OPTS+ options are already passed automatically by the infrastructure. * +PYTHON_FOO_INSTALL_TARGET_OPTS+, +PYTHON_FOO_INSTALL_STAGING_OPTS+, diff --git a/package/busybox/busybox.mk b/package/busybox/busybox.mk index 1ff2962275..3068bf72ea 100644 --- a/package/busybox/busybox.mk +++ b/package/busybox/busybox.mk @@ -47,7 +47,7 @@ endif BUSYBOX_KCONFIG_FILE = $(BUSYBOX_CONFIG_FILE) BUSYBOX_KCONFIG_EDITORS = menuconfig xconfig gconfig -BUSYBOX_KCONFIG_OPT = $(BUSYBOX_MAKE_OPTS) +BUSYBOX_KCONFIG_OPTS = $(BUSYBOX_MAKE_OPTS) define BUSYBOX_PERMISSIONS /bin/busybox f 4755 0 0 - - - - - diff --git a/package/gstreamer/gst-plugin-x170/gst-plugin-x170.mk b/package/gstreamer/gst-plugin-x170/gst-plugin-x170.mk index 29929bcd2b..378cd7a817 100644 --- a/package/gstreamer/gst-plugin-x170/gst-plugin-x170.mk +++ b/package/gstreamer/gst-plugin-x170/gst-plugin-x170.mk @@ -12,7 +12,7 @@ GST_PLUGIN_X170_LICENSE = BSD-1c # There is no generated configure script in the tarball. GST_PLUGIN_X170_AUTORECONF = YES -GST_PLUGIN_X170_AUTORECONF_OPT = -Im4/ +GST_PLUGIN_X170_AUTORECONF_OPTS = -Im4/ GST_PLUGIN_X170_DEPENDENCIES = gstreamer libglib2 on2-8170-libs $(eval $(autotools-package)) diff --git a/package/gstreamer1/gst1-plugins-bad/gst1-plugins-bad.mk b/package/gstreamer1/gst1-plugins-bad/gst1-plugins-bad.mk index 4e54b3b569..4c28085d8d 100644 --- a/package/gstreamer1/gst1-plugins-bad/gst1-plugins-bad.mk +++ b/package/gstreamer1/gst1-plugins-bad/gst1-plugins-bad.mk @@ -13,7 +13,7 @@ GST1_PLUGINS_BAD_LICENSE_FILES = COPYING COPYING.LIB GST1_PLUGINS_BAD_LICENSE = LGPLv2+ LGPLv2.1+ GST1_PLUGINS_BAD_AUTORECONF = YES -GST1_PLUGINS_BAD_AUTORECONF_OPT = -I $(@D)/common/m4 +GST1_PLUGINS_BAD_AUTORECONF_OPTS = -I $(@D)/common/m4 GST1_PLUGINS_BAD_GETTEXTIZE = YES GST1_PLUGINS_BAD_CONF_OPTS = \ diff --git a/package/lightning/lightning.mk b/package/lightning/lightning.mk index 841d2845b9..965f67adeb 100644 --- a/package/lightning/lightning.mk +++ b/package/lightning/lightning.mk @@ -5,7 +5,7 @@ ################################################################################ LIGHTNING_VERSION = 2.0.4 -LIGHTNING_SITE = http://ftp.gnu.org/gnu/lightning/ +LIGHTNING_SITE = http://ftp.gnu.org/gnu/lightning LIGHTNING_LICENSE = GPLv3 LIGHTNING_LICENSE_FILES = COPYING LIGHTNING_INSTALL_STAGING = YES diff --git a/package/pkg-autotools.mk b/package/pkg-autotools.mk index cfb2eee248..693ec3af4d 100644 --- a/package/pkg-autotools.mk +++ b/package/pkg-autotools.mk @@ -96,11 +96,11 @@ ifndef $(2)_GETTEXTIZE endif ifeq ($(4),host) - $(2)_GETTEXTIZE_OPT ?= $$($(3)_GETTEXTIZE_OPT) + $(2)_GETTEXTIZE_OPTS ?= $$($(3)_GETTEXTIZE_OPTS) endif ifeq ($(4),host) - $(2)_AUTORECONF_OPT ?= $$($(3)_AUTORECONF_OPT) + $(2)_AUTORECONF_OPTS ?= $$($(3)_AUTORECONF_OPTS) endif $(2)_CONF_ENV ?= @@ -214,7 +214,7 @@ endef # define GETTEXTIZE_HOOK @$$(call MESSAGE,"Gettextizing") - $(Q)cd $$($$(PKG)_SRCDIR) && $$(GETTEXTIZE) $$($$(PKG)_GETTEXTIZE_OPT) + $(Q)cd $$($$(PKG)_SRCDIR) && $$(GETTEXTIZE) $$($$(PKG)_GETTEXTIZE_OPTS) endef # @@ -222,7 +222,7 @@ endef # define AUTORECONF_HOOK @$$(call MESSAGE,"Autoreconfiguring") - $$(Q)cd $$($$(PKG)_SRCDIR) && $$($$(PKG)_AUTORECONF_ENV) $$(AUTORECONF) $$($$(PKG)_AUTORECONF_OPT) + $$(Q)cd $$($$(PKG)_SRCDIR) && $$($$(PKG)_AUTORECONF_ENV) $$(AUTORECONF) $$($$(PKG)_AUTORECONF_OPTS) endef # This must be repeated from inner-generic-package, otherwise we get an empty diff --git a/package/pkg-cmake.mk b/package/pkg-cmake.mk index a9a8e9ea43..a61cbe951b 100644 --- a/package/pkg-cmake.mk +++ b/package/pkg-cmake.mk @@ -56,7 +56,7 @@ $(2)_CONF_OPTS ?= $(2)_MAKE ?= $$(MAKE) $(2)_MAKE_ENV ?= $(2)_MAKE_OPTS ?= -$(2)_INSTALL_HOST_OPT ?= install +$(2)_INSTALL_HOST_OPTS ?= install $(2)_INSTALL_STAGING_OPTS ?= DESTDIR=$$(STAGING_DIR) install $(2)_INSTALL_TARGET_OPTS ?= DESTDIR=$$(TARGET_DIR) install @@ -185,7 +185,7 @@ endif # ifndef $(2)_INSTALL_CMDS define $(2)_INSTALL_CMDS - $$(HOST_MAKE_ENV) $$($$(PKG)_MAKE_ENV) $$($$(PKG)_MAKE) $$($$(PKG)_MAKE_OPTS) $$($$(PKG)_INSTALL_HOST_OPT) -C $$($$(PKG)_BUILDDIR) + $$(HOST_MAKE_ENV) $$($$(PKG)_MAKE_ENV) $$($$(PKG)_MAKE) $$($$(PKG)_MAKE_OPTS) $$($$(PKG)_INSTALL_HOST_OPTS) -C $$($$(PKG)_BUILDDIR) endef endif diff --git a/package/pkg-kconfig.mk b/package/pkg-kconfig.mk index cd08c1ab98..071153a0bf 100644 --- a/package/pkg-kconfig.mk +++ b/package/pkg-kconfig.mk @@ -33,7 +33,7 @@ $(call inner-generic-package,$(1),$(2),$(3),$(4)) # Default values $(2)_KCONFIG_EDITORS ?= menuconfig -$(2)_KCONFIG_OPT ?= +$(2)_KCONFIG_OPTS ?= $(2)_KCONFIG_FIXUP_CMDS ?= # FOO_KCONFIG_FILE is required @@ -51,7 +51,7 @@ $$($(2)_DIR)/.config: $$($(2)_KCONFIG_FILE) | $(1)-patch $$($(2)_DIR)/.stamp_kconfig_fixup_done: $$($(2)_DIR)/.config $$($(2)_KCONFIG_FIXUP_CMDS) @yes "" | $$($(2)_MAKE_ENV) $$(MAKE) -C $$($(2)_DIR) \ - $$($(2)_KCONFIG_OPT) oldconfig + $$($(2)_KCONFIG_OPTS) oldconfig $$(Q)touch $$@ # Before running configure, the configuration file should be present and fixed @@ -60,7 +60,7 @@ $$($(2)_TARGET_CONFIGURE): $$($(2)_DIR)/.stamp_kconfig_fixup_done # Configuration editors (menuconfig, ...) $$(addprefix $(1)-,$$($(2)_KCONFIG_EDITORS)): $$($(2)_DIR)/.stamp_kconfig_fixup_done $$($(2)_MAKE_ENV) $$(MAKE) -C $$($(2)_DIR) \ - $$($(2)_KCONFIG_OPT) $$(subst $(1)-,,$$@) + $$($(2)_KCONFIG_OPTS) $$(subst $(1)-,,$$@) rm -f $$($(2)_DIR)/.stamp_{kconfig_fixup_done,configured,built} rm -f $$($(2)_DIR)/.stamp_{target,staging}_installed diff --git a/package/pkg-luarocks.mk b/package/pkg-luarocks.mk index 95b397c9f9..f0fb73d0aa 100644 --- a/package/pkg-luarocks.mk +++ b/package/pkg-luarocks.mk @@ -33,7 +33,7 @@ define inner-luarocks-package -$(2)_BUILD_OPT ?= +$(2)_BUILD_OPTS ?= $(2)_SUBDIR ?= $(1)-$$(shell echo "$$($(3)_VERSION)" | sed -e "s/-[0-9]$$$$//") $(2)_ROCKSPEC ?= $(1)-$$($(3)_VERSION).rockspec $(2)_SOURCE ?= $(1)-$$($(3)_VERSION).src.rock @@ -59,7 +59,7 @@ endif ifndef $(2)_INSTALL_TARGET_CMDS define $(2)_INSTALL_TARGET_CMDS cd $$($(2)_SRCDIR) && \ - $$(LUAROCKS_RUN) make --keep $$($(2)_ROCKSPEC) $$($(2)_BUILD_OPT) + $$(LUAROCKS_RUN) make --keep $$($(2)_ROCKSPEC) $$($(2)_BUILD_OPTS) endef endif diff --git a/package/pkg-perl.mk b/package/pkg-perl.mk index 8ebfce535a..7abafc35ae 100644 --- a/package/pkg-perl.mk +++ b/package/pkg-perl.mk @@ -129,11 +129,11 @@ ifeq ($(4),target) # Build package for target define $(2)_BUILD_CMDS cd $$($$(PKG)_SRCDIR) && if [ -f Build.PL ] ; then \ - perl Build $$($(2)_BUILD_OPT) build; \ + perl Build $$($(2)_BUILD_OPTS) build; \ else \ $$(MAKE1) \ PERL_INC=$$(STAGING_DIR)/usr/lib/perl5/$$(PERL_VERSION)/$$(PERL_ARCHNAME)/CORE \ - $$($(2)_BUILD_OPT) pure_all; \ + $$($(2)_BUILD_OPTS) pure_all; \ fi endef else @@ -141,9 +141,9 @@ else # Build package for host define $(2)_BUILD_CMDS cd $$($$(PKG)_SRCDIR) && if [ -f Build.PL ] ; then \ - perl Build $$($(2)_BUILD_OPT) build; \ + perl Build $$($(2)_BUILD_OPTS) build; \ else \ - $$(MAKE1) $$($(2)_BUILD_OPT) pure_all; \ + $$(MAKE1) $$($(2)_BUILD_OPTS) pure_all; \ fi endef endif diff --git a/package/pkg-python.mk b/package/pkg-python.mk index be4011a8b8..43d1f0ff28 100644 --- a/package/pkg-python.mk +++ b/package/pkg-python.mk @@ -36,7 +36,7 @@ PKG_PYTHON_DISTUTILS_ENV = \ _python_prefix=/usr \ _python_exec_prefix=/usr -PKG_PYTHON_DISTUTILS_BUILD_OPT = \ +PKG_PYTHON_DISTUTILS_BUILD_OPTS = \ --executable=/usr/bin/python PKG_PYTHON_DISTUTILS_INSTALL_TARGET_OPTS = \ @@ -111,7 +111,7 @@ $(2)_SRCDIR = $$($(2)_DIR)/$$($(2)_SUBDIR) $(2)_BUILDDIR = $$($(2)_SRCDIR) $(2)_ENV ?= -$(2)_BUILD_OPT ?= +$(2)_BUILD_OPTS ?= $(2)_INSTALL_OPTS ?= ifndef $(2)_SETUP_TYPE @@ -127,13 +127,13 @@ ifeq ($$($(2)_SETUP_TYPE),distutils) ifeq ($(4),target) $(2)_BASE_ENV = $$(PKG_PYTHON_DISTUTILS_ENV) $(2)_BASE_BUILD_TGT = build -$(2)_BASE_BUILD_OPT = $$(PKG_PYTHON_DISTUTILS_BUILD_OPT) +$(2)_BASE_BUILD_OPTS = $$(PKG_PYTHON_DISTUTILS_BUILD_OPTS) $(2)_BASE_INSTALL_TARGET_OPTS = $$(PKG_PYTHON_DISTUTILS_INSTALL_TARGET_OPTS) $(2)_BASE_INSTALL_STAGING_OPTS = $$(PKG_PYTHON_DISTUTILS_INSTALL_STAGING_OPTS) else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_DISTUTILS_ENV) $(2)_BASE_BUILD_TGT = build -$(2)_BASE_BUILD_OPT = +$(2)_BASE_BUILD_OPTS = $(2)_BASE_INSTALL_OPTS = $$(HOST_PKG_PYTHON_DISTUTILS_INSTALL_OPTS) endif # Setuptools @@ -141,13 +141,13 @@ else ifeq ($$($(2)_SETUP_TYPE),setuptools) ifeq ($(4),target) $(2)_BASE_ENV = $$(PKG_PYTHON_SETUPTOOLS_ENV) $(2)_BASE_BUILD_TGT = build -$(2)_BASE_BUILD_OPT = +$(2)_BASE_BUILD_OPTS = $(2)_BASE_INSTALL_TARGET_OPTS = $$(PKG_PYTHON_SETUPTOOLS_INSTALL_TARGET_OPTS) $(2)_BASE_INSTALL_STAGING_OPTS = $$(PKG_PYTHON_SETUPTOOLS_INSTALL_STAGING_OPTS) else $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_SETUPTOOLS_ENV) $(2)_BASE_BUILD_TGT = build -$(2)_BASE_BUILD_OPT = +$(2)_BASE_BUILD_OPTS = $(2)_BASE_INSTALL_OPTS = $$(HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS) endif else @@ -263,7 +263,7 @@ define $(2)_BUILD_CMDS $$($$(PKG)_BASE_ENV) $$($$(PKG)_ENV) \ $$($(2)_PYTHON_INTERPRETER) setup.py \ $$($$(PKG)_BASE_BUILD_TGT) \ - $$($$(PKG)_BASE_BUILD_OPT) $$($$(PKG)_BUILD_OPT)) + $$($$(PKG)_BASE_BUILD_OPTS) $$($$(PKG)_BUILD_OPTS)) endef endif diff --git a/package/python-numpy/python-numpy.mk b/package/python-numpy/python-numpy.mk index 3d3a6c52d0..0ab58b0000 100644 --- a/package/python-numpy/python-numpy.mk +++ b/package/python-numpy/python-numpy.mk @@ -16,7 +16,7 @@ PYTHON_NUMPY_DEPENDENCIES += clapack PYTHON_NUMPY_SITE_CFG_LIBS += blas lapack endif -PYTHON_NUMPY_BUILD_OPT = --fcompiler=None +PYTHON_NUMPY_BUILD_OPTS = --fcompiler=None define PYTHON_NUMPY_CONFIGURE_CMDS -rm -f $(@D)/site.cfg diff --git a/package/uclibc/uclibc.mk b/package/uclibc/uclibc.mk index a5c7f4e10f..6272969a27 100644 --- a/package/uclibc/uclibc.mk +++ b/package/uclibc/uclibc.mk @@ -44,7 +44,7 @@ endif UCLIBC_KCONFIG_FILE = $(UCLIBC_CONFIG_FILE) UCLIBC_KCONFIG_FRAGMENT_FILES = $(call qstrip,$(BR2_UCLIBC_CONFIG_FRAGMENT_FILES)) -UCLIBC_KCONFIG_OPT = \ +UCLIBC_KCONFIG_OPTS = \ $(UCLIBC_MAKE_FLAGS) \ PREFIX=$(STAGING_DIR) \ DEVEL_PREFIX=/usr/ \ diff --git a/package/webkit/webkit.mk b/package/webkit/webkit.mk index a346793604..77b533a2b7 100644 --- a/package/webkit/webkit.mk +++ b/package/webkit/webkit.mk @@ -17,7 +17,7 @@ WEBKIT_DEPENDENCIES += \ # webkit-disable-tests.patch changes configure.ac therefore autoreconf required WEBKIT_AUTORECONF = YES -WEBKIT_AUTORECONF_OPT = -I $(@D)/Source/autotools +WEBKIT_AUTORECONF_OPTS = -I $(@D)/Source/autotools # parallel make install deadlocks with make 3.81 WEBKIT_INSTALL_STAGING_OPTS = -j1 DESTDIR=$(STAGING_DIR) install From 6d330497aa4bc94a7e1c0116e5393b6a07e6e998 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:50:31 +0100 Subject: [PATCH 05/12] Backport: Rename TARGETS to PACKAGES Backport of 8a58e0238 --- Makefile | 30 +++++++++++++++--------------- package/matchbox/matchbox.mk | 2 +- package/pkg-generic.mk | 2 +- system/system.mk | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index ffd8e46d7d..c316d1e40d 100644 --- a/Makefile +++ b/Makefile @@ -310,7 +310,7 @@ unexport TERMINFO GNU_HOST_NAME := $(shell support/gnuconfig/config.guess) -TARGETS := +PACKAGES := # silent mode requested? QUIET := $(if $(findstring s,$(MAKEFLAGS)),-q) @@ -388,7 +388,7 @@ include arch/arch.mk include support/dependencies/dependencies.mk # We also need the various per-package makefiles, which also add -# each selected package to TARGETS if that package was selected +# each selected package to PACKAGES if that package was selected # in the .config file. include toolchain/*.mk include toolchain/*/*.mk @@ -409,17 +409,17 @@ include fs/common.mk include $(BR2_EXTERNAL)/external.mk -TARGETS_SOURCE := $(patsubst %,%-source,$(TARGETS)) -TARGETS_DIRCLEAN := $(patsubst %,%-dirclean,$(TARGETS)) +PACKAGES_SOURCE := $(patsubst %,%-source,$(PACKAGES)) +PACKAGES_DIRCLEAN := $(patsubst %,%-dirclean,$(PACKAGES)) # host-* dependencies have to be handled specially, as those aren't -# visible in Kconfig and hence not added to a variable like TARGETS. +# visible in Kconfig and hence not added to a variable like PACKAGES. # instead, find all the host-* targets listed in each _DEPENDENCIES # variable for each enabled target. # Notice: this only works for newstyle gentargets/autotargets packages TARGETS_HOST_DEPS = $(sort $(filter host-%,$(foreach dep,\ $(addsuffix _DEPENDENCIES,\ - $(call UPPERCASE,$(TARGETS) $(TARGETS_ROOTFS))),\ + $(call UPPERCASE,$(PACKAGES) $(TARGETS_ROOTFS))),\ $($(dep))))) # Host packages can in turn have their own dependencies. Likewise find # all the package names listed in the HOST__DEPENDENCIES for each @@ -431,8 +431,8 @@ HOST_DEPS = $(sort $(foreach dep,\ $($(dep)))) HOST_SOURCE += $(addsuffix -source,$(sort $(TARGETS_HOST_DEPS) $(HOST_DEPS))) -TARGETS_LEGAL_INFO := $(patsubst %,%-legal-info,\ - $(TARGETS) $(TARGETS_HOST_DEPS) $(HOST_DEPS)))) +PACKAGES_LEGAL_INFO := $(patsubst %,%-legal-info,\ + $(PACKAGES) $(TARGETS_HOST_DEPS) $(HOST_DEPS)))) dirs: $(BUILD_DIR) $(STAGING_DIR) $(TARGET_DIR) \ $(HOST_DIR) $(BINARIES_DIR) @@ -447,8 +447,8 @@ world: target-post-image .PHONY: all world toolchain dirs clean distclean source outputmakefile \ legal-info legal-info-prepare legal-info-clean printvars \ target-finalize target-post-image \ - $(TARGETS) $(TARGETS_ROOTFS) \ - $(TARGETS_DIRCLEAN) $(TARGETS_SOURCE) $(TARGETS_LEGAL_INFO) \ + $(PACKAGES) $(TARGETS_ROOTFS) \ + $(PACKAGES_DIRCLEAN) $(PACKAGES_SOURCE) $(PACKAGES_LEGAL_INFO) \ $(BUILD_DIR) $(STAGING_DIR) $(TARGET_DIR) \ $(HOST_DIR) $(BINARIES_DIR) @@ -534,7 +534,7 @@ endif ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y) GENERATE_LOCALE = $(call qstrip,$(BR2_GENERATE_LOCALE)) ifneq ($(GENERATE_LOCALE),) -TARGETS += host-localedef +PACKAGES += host-localedef define GENERATE_LOCALES $(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/ @@ -578,7 +578,7 @@ endif $(TARGETS_ROOTFS): target-finalize -target-finalize: $(TARGETS) +target-finalize: $(PACKAGES) @$(call MESSAGE,"Finalizing target directory") $(foreach hook,$(TARGET_FINALIZE_HOOKS),$($(hook))$(sep)) rm -rf $(TARGET_DIR)/usr/include $(TARGET_DIR)/usr/share/aclocal \ @@ -648,7 +648,7 @@ target-post-image: $(TARGETS_ROOTFS) target-finalize $(call MESSAGE,"Executing post-image script $(s)"); \ $(EXTRA_ENV) $(s) $(BINARIES_DIR) $(call qstrip,$(BR2_ROOTFS_POST_SCRIPT_ARGS))$(sep)) -source: $(TARGETS_SOURCE) $(HOST_SOURCE) +source: $(PACKAGES_SOURCE) $(HOST_SOURCE) external-deps: @$(MAKE1) -Bs DL_MODE=SHOW_EXTERNAL_DEPS $(EXTRAMAKEARGS) source | sort -u @@ -666,7 +666,7 @@ legal-info-prepare: $(LEGAL_INFO_DIR) @$(call legal-warning,the toolchain has not been saved) @cp $(BR2_CONFIG) $(LEGAL_INFO_DIR)/buildroot.config -legal-info: dirs legal-info-clean legal-info-prepare $(TARGETS_LEGAL_INFO) \ +legal-info: dirs legal-info-clean legal-info-prepare $(PACKAGES_LEGAL_INFO) \ $(REDIST_SOURCES_DIR_TARGET) $(REDIST_SOURCES_DIR_HOST) @cat support/legal-info/README.header >>$(LEGAL_REPORT) @if [ -r $(LEGAL_WARNINGS) ]; then \ @@ -677,7 +677,7 @@ legal-info: dirs legal-info-clean legal-info-prepare $(TARGETS_LEGAL_INFO) \ @rm -f $(LEGAL_WARNINGS) show-targets: - @echo $(HOST_DEPS) $(TARGETS_HOST_DEPS) $(TARGETS) $(TARGETS_ROOTFS) + @echo $(HOST_DEPS) $(TARGETS_HOST_DEPS) $(PACKAGES) $(TARGETS_ROOTFS) graph-build: $(O)/build/build-time.log @install -d $(O)/graphs diff --git a/package/matchbox/matchbox.mk b/package/matchbox/matchbox.mk index aec57c0a3b..fe1a7db59c 100644 --- a/package/matchbox/matchbox.mk +++ b/package/matchbox/matchbox.mk @@ -1,4 +1,4 @@ ifeq ($(BR2_PACKAGE_MATCHBOX),y) include $(sort $(wildcard package/matchbox/*/*.mk)) -TARGETS += matchbox-lib matchbox-wm +PACKAGES += matchbox-lib matchbox-wm endif diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index 97cffa05ed..4973efd75a 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -743,7 +743,7 @@ $$(foreach pkg,$$($(2)_PROVIDES),\ $$(eval $$(call virt-provides-single,$$(pkg),$$(call UPPERCASE,$$(pkg)),$(1))$$(sep))) endif -TARGETS += $(1) +PACKAGES += $(1) ifneq ($$($(2)_PERMISSIONS),) PACKAGES_PERMISSIONS_TABLE += $$($(2)_PERMISSIONS)$$(sep) diff --git a/system/system.mk b/system/system.mk index b6146158da..3a73dfd732 100644 --- a/system/system.mk +++ b/system/system.mk @@ -34,7 +34,7 @@ TARGET_FINALIZE_HOOKS += SYSTEM_ISSUE endif ifneq ($(TARGET_GENERIC_ROOT_PASSWD),) -TARGETS += host-mkpasswd +PACKAGES += host-mkpasswd endif ifeq ($(BR2_ROOTFS_SKELETON_DEFAULT),y) From 0119ff19f5721b7fcb88b16f88d529e6527e4840 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sat, 4 Apr 2020 23:57:22 +0100 Subject: [PATCH 06/12] Copy support/dependencies from upstream Copied and replaced $(HOST_DIR)/bin with $(HOST_DIR)/usr/bin --- support/dependencies/check-host-bison-flex.mk | 10 ++ support/dependencies/check-host-cmake.mk | 14 ++ support/dependencies/check-host-cmake.sh | 45 ++++++ support/dependencies/check-host-coreutils.mk | 6 + support/dependencies/check-host-coreutils.sh | 12 ++ support/dependencies/check-host-gzip.mk | 3 + support/dependencies/check-host-gzip.sh | 21 +++ support/dependencies/check-host-lzip.mk | 2 +- support/dependencies/check-host-make.mk | 21 +++ support/dependencies/check-host-make.sh | 37 +++++ support/dependencies/check-host-python3.mk | 8 + support/dependencies/check-host-python3.sh | 31 ++++ support/dependencies/check-host-tar.mk | 7 +- support/dependencies/check-host-xzcat.mk | 4 +- support/dependencies/dependencies.mk | 30 ++-- support/dependencies/dependencies.sh | 143 +++++++++++------- 16 files changed, 316 insertions(+), 78 deletions(-) create mode 100644 support/dependencies/check-host-bison-flex.mk create mode 100644 support/dependencies/check-host-cmake.mk create mode 100755 support/dependencies/check-host-cmake.sh create mode 100644 support/dependencies/check-host-coreutils.mk create mode 100755 support/dependencies/check-host-coreutils.sh create mode 100644 support/dependencies/check-host-gzip.mk create mode 100755 support/dependencies/check-host-gzip.sh create mode 100644 support/dependencies/check-host-make.mk create mode 100755 support/dependencies/check-host-make.sh create mode 100644 support/dependencies/check-host-python3.mk create mode 100755 support/dependencies/check-host-python3.sh diff --git a/support/dependencies/check-host-bison-flex.mk b/support/dependencies/check-host-bison-flex.mk new file mode 100644 index 0000000000..233b6c51cc --- /dev/null +++ b/support/dependencies/check-host-bison-flex.mk @@ -0,0 +1,10 @@ +# If the system lacks bison or flex, add +# dependencies to suitable host packages + +ifeq ($(shell which bison 2>/dev/null),) +BR2_BISON_HOST_DEPENDENCY = host-bison +endif + +ifeq ($(shell which flex 2>/dev/null),) +BR2_FLEX_HOST_DEPENDENCY = host-flex +endif diff --git a/support/dependencies/check-host-cmake.mk b/support/dependencies/check-host-cmake.mk new file mode 100644 index 0000000000..e0563185b7 --- /dev/null +++ b/support/dependencies/check-host-cmake.mk @@ -0,0 +1,14 @@ +# Set this to either 3.10 or higher, depending on the highest minimum +# version required by any of the packages bundled in Buildroot. If a +# package is bumped or a new one added, and it requires a higher +# version, our cmake infra will catch it and build its own. +# +BR2_CMAKE_VERSION_MIN = 3.10 + +BR2_CMAKE_CANDIDATES ?= cmake cmake3 +BR2_CMAKE ?= $(call suitable-host-package,cmake,\ + $(BR2_CMAKE_VERSION_MIN) $(BR2_CMAKE_CANDIDATES)) +ifeq ($(BR2_CMAKE),) +BR2_CMAKE = $(HOST_DIR)/usr/bin/cmake +BR2_CMAKE_HOST_DEPENDENCY = host-cmake +endif diff --git a/support/dependencies/check-host-cmake.sh b/support/dependencies/check-host-cmake.sh new file mode 100755 index 0000000000..fadeae9f6b --- /dev/null +++ b/support/dependencies/check-host-cmake.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# prevent shift error +[ $# -lt 2 ] && exit 1 + +major_min="${1%.*}" +minor_min="${1#*.}" + +shift + +for candidate; do + + # Try to locate the candidate. Discard it if not located. + cmake=`which "${candidate}" 2>/dev/null` + [ -n "${cmake}" ] || continue + + # Extract version X.Y from versions in the form X.Y or X.Y.Z + # with X, Y and Z numbers with one or more digits each, e.g. + # 3.2 -> 3.2 + # 3.2.3 -> 3.2 + # 3.2.42 -> 3.2 + # 3.10 -> 3.10 + # 3.10.4 -> 3.10 + # 3.10.42 -> 3.10 + # Discard the candidate if no version can be obtained + version="$(${cmake} --version \ + |sed -r -e '/.* ([[:digit:]]+\.[[:digit:]]+).*$/!d;' \ + -e 's//\1/' + )" + [ -n "${version}" ] || continue + + major="${version%.*}" + minor="${version#*.}" + + if [ ${major} -gt ${major_min} ]; then + echo "${cmake}" + exit + elif [ ${major} -eq ${major_min} -a ${minor} -ge ${minor_min} ]; then + echo "${cmake}" + exit + fi +done + +# echo nothing: no suitable cmake found +exit 1 diff --git a/support/dependencies/check-host-coreutils.mk b/support/dependencies/check-host-coreutils.mk new file mode 100644 index 0000000000..87a3f446ea --- /dev/null +++ b/support/dependencies/check-host-coreutils.mk @@ -0,0 +1,6 @@ +# Check whether the host's coreutils are up to date enough +# to provide 'ln --relative' and 'realpath'. + +ifeq (,$(call suitable-host-package,coreutils)) +BR2_COREUTILS_HOST_DEPENDENCY = host-coreutils +endif diff --git a/support/dependencies/check-host-coreutils.sh b/support/dependencies/check-host-coreutils.sh new file mode 100755 index 0000000000..4d36d74933 --- /dev/null +++ b/support/dependencies/check-host-coreutils.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# Exit on the first error +set -e + +# Does ln supports the --relative/-r option? +ln --relative --help >/dev/null 2>&1 + +# Does realpath exist? +realpath --help >/dev/null 2>&1 + +echo OK diff --git a/support/dependencies/check-host-gzip.mk b/support/dependencies/check-host-gzip.mk new file mode 100644 index 0000000000..bf9a369a7d --- /dev/null +++ b/support/dependencies/check-host-gzip.mk @@ -0,0 +1,3 @@ +ifeq (,$(call suitable-host-package,gzip)) +BR2_GZIP_HOST_DEPENDENCY = host-gzip +endif diff --git a/support/dependencies/check-host-gzip.sh b/support/dependencies/check-host-gzip.sh new file mode 100755 index 0000000000..5f344c5f9b --- /dev/null +++ b/support/dependencies/check-host-gzip.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +candidate="$1" # ignored + +gzip="$(which gzip)" +if [ ! -x "${gzip}" ]; then + # echo nothing: no suitable gzip found + exit 1 +fi + +# gzip displays its version string on stdout +# pigz displays its version string on stderr +version="$("${gzip}" --version 2>&1)" +case "${version}" in + (*pigz*) + # echo nothing: no suitable gzip found + exit 1 + ;; +esac + +printf "%s" "${gzip}" diff --git a/support/dependencies/check-host-lzip.mk b/support/dependencies/check-host-lzip.mk index 98a4f42388..ac7ff1f353 100644 --- a/support/dependencies/check-host-lzip.mk +++ b/support/dependencies/check-host-lzip.mk @@ -1,4 +1,4 @@ ifeq (,$(call suitable-host-package,lzip,$(LZCAT))) BR2_LZIP_HOST_DEPENDENCY = host-lzip -LZCAT = $(HOST_DIR)/bin/lzip -d -c +LZCAT = $(HOST_DIR)/usr/bin/lzip -d -c endif diff --git a/support/dependencies/check-host-make.mk b/support/dependencies/check-host-make.mk new file mode 100644 index 0000000000..7ebdc81801 --- /dev/null +++ b/support/dependencies/check-host-make.mk @@ -0,0 +1,21 @@ +# Since version 2.28, glibc requires GNU Make >= 4.0 +# https://www.sourceware.org/ml/libc-alpha/2018-08/msg00003.html +# +# Set this to either 4.0 or higher, depending on the highest minimum +# version required by any of the packages bundled in Buildroot. If a +# package is bumped or a new one added, and it requires a higher +# version, our package infra will catch it and whine. +# +BR2_MAKE_VERSION_MIN = 4.0 + +BR2_MAKE ?= $(call suitable-host-package,make,\ + $(BR2_MAKE_VERSION_MIN) $(MAKE)) + +ifeq ($(BR2_MAKE),) +BR2_MAKE = $(HOST_DIR)/usr/bin/host-make -j$(PARALLEL_JOBS) +BR2_MAKE1 = $(HOST_DIR)/usr/bin/host-make -j1 +BR2_MAKE_HOST_DEPENDENCY = host-make +else +BR2_MAKE = $(MAKE) +BR2_MAKE1 = $(MAKE1) +endif diff --git a/support/dependencies/check-host-make.sh b/support/dependencies/check-host-make.sh new file mode 100755 index 0000000000..0de7e9f6fa --- /dev/null +++ b/support/dependencies/check-host-make.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# prevent shift error +[ $# -lt 2 ] && exit 1 + +major_min="${1%.*}" +minor_min="${1#*.}" + +shift + +# The host make program is already checked by dependencies.sh but we +# want to check the version number even if Buildroot is able to use +# GNU make >= 3.81 but some packages may require a more recent version. +make="$1" + +# Output of 'make --version' examples: +# GNU Make 4.2.1 +# GNU Make 4.0 +# GNU Make 3.81 +version=`$make --version 2>&1 | sed -e 's/^.* \([0-9\.]\)/\1/g' -e 's/[-\ +].*//g' -e '1q'` + +major=`echo "$version" | cut -d. -f1` +minor=`echo "$version" | cut -d. -f2` + +if [ $major -lt $major_min ]; then + # echo nothing: no suitable make found + exit 1 +fi + +if [ $major -eq $major_min -a $minor -lt $minor_min ]; then + # echo nothing: no suitable make found + exit 1 +fi + +# valid +echo $make diff --git a/support/dependencies/check-host-python3.mk b/support/dependencies/check-host-python3.mk new file mode 100644 index 0000000000..50ed933a4e --- /dev/null +++ b/support/dependencies/check-host-python3.mk @@ -0,0 +1,8 @@ +# Since version 2.29, glibc requires python 3.4 or later to build the GNU C Library. +# https://www.sourceware.org/ml/libc-alpha/2019-01/msg00723.html + +BR2_PYTHON3_VERSION_MIN = 3.4 + +ifeq (,$(call suitable-host-package,python3,$(BR2_PYTHON3_VERSION_MIN) python3 python)) +BR2_PYTHON3_HOST_DEPENDENCY = host-python3 +endif diff --git a/support/dependencies/check-host-python3.sh b/support/dependencies/check-host-python3.sh new file mode 100755 index 0000000000..17cafd2883 --- /dev/null +++ b/support/dependencies/check-host-python3.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +# prevent shift error +[ $# -lt 2 ] && exit 1 + +version_min="$(echo ${1} | awk '{ split($1, v, "."); print v[1] v[2] }')" + +shift + +# The host python interpreter is already checked by dependencies.sh but +# it only check if the version is at least 2.7. +# We want to check the version number of the python3 interpreter even +# if Buildroot is able to use any version but some packages may require +# a more recent version. + +for candidate in "${@}" ; do + python3=`which $candidate 2>/dev/null` + if [ ! -x "$python3" ]; then + continue + fi + version=`$python3 -V 2>&1 | awk '{ split($2, v, "."); print v[1] v[2] }'` + + if [ $version -lt $version_min ]; then + # no suitable python3 found + continue + fi + + # suitable python3 found + echo $python3 + break +done diff --git a/support/dependencies/check-host-tar.mk b/support/dependencies/check-host-tar.mk index 010bde7a51..c690eb0b99 100644 --- a/support/dependencies/check-host-tar.mk +++ b/support/dependencies/check-host-tar.mk @@ -1,9 +1,6 @@ TAR ?= tar ifeq (,$(call suitable-host-package,tar,$(TAR))) - DEPENDENCIES_HOST_PREREQ += host-tar - TAR = $(HOST_DIR)/usr/bin/tar +TAR = $(HOST_DIR)/usr/bin/tar +BR2_TAR_HOST_DEPENDENCY = host-tar endif - -# Since TAR is at least 1.17, it will certainly support --strip-components -TAR_STRIP_COMPONENTS = --strip-components diff --git a/support/dependencies/check-host-xzcat.mk b/support/dependencies/check-host-xzcat.mk index 24286ef5c8..5653e3b24c 100644 --- a/support/dependencies/check-host-xzcat.mk +++ b/support/dependencies/check-host-xzcat.mk @@ -2,6 +2,6 @@ # If it is not present, build our own host-xzcat ifeq (,$(call suitable-host-package,xzcat,$(XZCAT))) - DEPENDENCIES_HOST_PREREQ += host-xz - XZCAT = $(HOST_DIR)/usr/bin/xzcat +BR2_XZCAT_HOST_DEPENDENCY = host-xz +XZCAT = $(HOST_DIR)/usr/bin/xzcat endif diff --git a/support/dependencies/dependencies.mk b/support/dependencies/dependencies.mk index 155a90988c..4fac5c731b 100644 --- a/support/dependencies/dependencies.mk +++ b/support/dependencies/dependencies.mk @@ -5,8 +5,11 @@ # ################################################################################ -DEPENDENCIES_HOST_PREREQ := - +ifeq ($(BR2_FORCE_HOST_BUILD),y) +# ignore all available host packages +define suitable-host-package +endef +else # suitable-host-pkg: calls check-host-$(1).sh shell script. Parameter (2) # can be the candidate to be checked. If not present, the check-host-$(1).sh # script should use 'which' to find a candidate. The script should return @@ -14,28 +17,19 @@ DEPENDENCIES_HOST_PREREQ := define suitable-host-package $(shell support/dependencies/check-host-$(1).sh $(2)) endef --include $(sort $(wildcard support/dependencies/check-host-*.mk)) - -ifeq ($(BR2_STRIP_sstrip),y) -DEPENDENCIES_HOST_PREREQ+=host-sstrip endif +# host utilities needs host-tar to extract the source code tarballs, so +# ensure check-host-tar.mk is included before the rest +include support/dependencies/check-host-tar.mk +-include $(sort $(filter-out %-tar.mk,$(wildcard support/dependencies/check-host-*.mk))) -ifeq ($(BR2_CCACHE),y) -DEPENDENCIES_HOST_PREREQ += host-ccache -endif - -core-dependencies: - @HOSTCC="$(firstword $(HOSTCC))" MAKE="$(MAKE)" \ - DL_TOOLS="$(sort $(DL_TOOLS_DEPENDENCIES))" \ +dependencies: + @MAKE="$(MAKE)" DL_TOOLS="$(sort $(DL_TOOLS_DEPENDENCIES))" \ $(TOPDIR)/support/dependencies/dependencies.sh -dependencies: HOSTCC=$(HOSTCC_NOCCACHE) -dependencies: HOSTCXX=$(HOSTCXX_NOCCACHE) -dependencies: core-dependencies $(DEPENDENCIES_HOST_PREREQ) - ################################################################################ # # Toplevel Makefile options # ################################################################################ -.PHONY: dependencies core-dependencies +.PHONY: dependencies diff --git a/support/dependencies/dependencies.sh b/support/dependencies/dependencies.sh index a9c5b311ce..98469bd70c 100755 --- a/support/dependencies/dependencies.sh +++ b/support/dependencies/dependencies.sh @@ -11,36 +11,37 @@ if test $? != 0 ; then exit 1 fi -# sanity check for CWD in LD_LIBRARY_PATH -# try not to rely on egrep.. -if test -n "$LD_LIBRARY_PATH" ; then - echo TRiGGER_start"$LD_LIBRARY_PATH"TRiGGER_end | grep ':\.:' >/dev/null 2>&1 || - echo TRiGGER_start"$LD_LIBRARY_PATH"TRiGGER_end | grep 'TRiGGER_start\.:' >/dev/null 2>&1 || - echo TRiGGER_start"$LD_LIBRARY_PATH"TRiGGER_end | grep ':\.TRiGGER_end' >/dev/null 2>&1 || - echo TRiGGER_start"$LD_LIBRARY_PATH"TRiGGER_end | grep 'TRiGGER_start\.TRiGGER_end' >/dev/null 2>&1 - if test $? = 0; then - echo - echo "You seem to have the current working directory in your" - echo "LD_LIBRARY_PATH environment variable. This doesn't work." - exit 1; - fi -fi; +# Sanity check for CWD in LD_LIBRARY_PATH +case ":${LD_LIBRARY_PATH:-unset}:" in +(*::*|*:.:*) + echo + echo "You seem to have the current working directory in your" + echo "LD_LIBRARY_PATH environment variable. This doesn't work." + exit 1 + ;; +esac -# sanity check for CWD in PATH. Having the current working directory -# in the PATH makes the toolchain build process break. -# try not to rely on egrep.. -if test -n "$PATH" ; then - echo TRiGGER_start"$PATH"TRiGGER_end | grep ':\.:' >/dev/null 2>&1 || - echo TRiGGER_start"$PATH"TRiGGER_end | grep 'TRiGGER_start\.:' >/dev/null 2>&1 || - echo TRiGGER_start"$PATH"TRiGGER_end | grep ':\.TRiGGER_end' >/dev/null 2>&1 || - echo TRiGGER_start"$PATH"TRiGGER_end | grep 'TRiGGER_start\.TRiGGER_end' >/dev/null 2>&1 - if test $? = 0; then - echo - echo "You seem to have the current working directory in your" - echo "PATH environment variable. This doesn't work." - exit 1; - fi -fi; +# Sanity check for CWD in PATH. Having the current working directory +# in the PATH makes various packages (e.g. toolchain, coreutils...) +# build process break. +# PATH should not contain a newline, otherwise it fails in spectacular +# ways as soon as PATH is referenced in a package rule +# An empty PATH is technically possible, but in practice we would not +# even arrive here if that was the case. +case ":${PATH:-unset}:" in +(*::*|*:.:*) + echo + echo "You seem to have the current working directory in your" + echo "PATH environment variable. This doesn't work." + exit 1 + ;; +(*" +"*) printf "\n" + printf "Your PATH contains a newline (\\\n) character.\n" + printf "This doesn't work. Fix you PATH.\n" + exit 1 + ;; +esac if test -n "$PERL_MM_OPT" ; then echo @@ -66,6 +67,10 @@ check_prog_host "which" # Verify that sed is installed check_prog_host "sed" +# 'file' must be present and must be exactly /usr/bin/file, +# otherwise libtool fails in incomprehensible ways. +check_prog_host "/usr/bin/file" + # Check make MAKE=$(which make 2> /dev/null) if [ -z "$MAKE" ] ; then @@ -107,9 +112,9 @@ if [ -z "$COMPILER_VERSION" ] ; then fi; COMPILER_MAJOR=$(echo $COMPILER_VERSION | sed -e "s/\..*//g") COMPILER_MINOR=$(echo $COMPILER_VERSION | sed -e "s/^$COMPILER_MAJOR\.//g" -e "s/\..*//g") -if [ $COMPILER_MAJOR -lt 3 -o $COMPILER_MAJOR -eq 2 -a $COMPILER_MINOR -lt 95 ] ; then +if [ $COMPILER_MAJOR -lt 4 -o $COMPILER_MAJOR -eq 4 -a $COMPILER_MINOR -lt 8 ] ; then echo - echo "You have gcc '$COMPILER_VERSION' installed. gcc >= 2.95 is required" + echo "You have gcc '$COMPILER_VERSION' installed. gcc >= 4.8 is required" exit 1; fi; @@ -130,12 +135,14 @@ if [ ! -z "$CXXCOMPILER" ] ; then echo echo "You may have to install 'g++' on your build machine" fi +fi +if [ -n "$CXXCOMPILER_VERSION" ] ; then CXXCOMPILER_MAJOR=$(echo $CXXCOMPILER_VERSION | sed -e "s/\..*//g") CXXCOMPILER_MINOR=$(echo $CXXCOMPILER_VERSION | sed -e "s/^$CXXCOMPILER_MAJOR\.//g" -e "s/\..*//g") - if [ $CXXCOMPILER_MAJOR -lt 3 -o $CXXCOMPILER_MAJOR -eq 2 -a $CXXCOMPILER_MINOR -lt 95 ] ; then + if [ $CXXCOMPILER_MAJOR -lt 4 -o $CXXCOMPILER_MAJOR -eq 4 -a $CXXCOMPILER_MINOR -lt 8 ] ; then echo - echo "You have g++ '$CXXCOMPILER_VERSION' installed. g++ >= 2.95 is required" + echo "You have g++ '$CXXCOMPILER_VERSION' installed. g++ >= 4.8 is required" exit 1 fi fi @@ -153,7 +160,7 @@ fi # Check that a few mandatory programs are installed missing_progs="no" -for prog in patch perl tar wget cpio python unzip rsync bc ${DL_TOOLS} ; do +for prog in patch perl tar wget cpio unzip rsync bc ${DL_TOOLS} ; do if ! which $prog > /dev/null ; then echo "You must install '$prog' on your build machine"; missing_progs="yes" @@ -173,14 +180,13 @@ if test "${missing_progs}" = "yes" ; then exit 1 fi -if grep ^BR2_TOOLCHAIN_BUILDROOT=y $BR2_CONFIG > /dev/null && \ - grep ^BR2_ENABLE_LOCALE=y $BR2_CONFIG > /dev/null ; then +if grep ^BR2_NEEDS_HOST_UTF8_LOCALE=y $BR2_CONFIG > /dev/null; then if ! which locale > /dev/null ; then echo echo "You need locale support on your build machine to build a toolchain supporting locales" exit 1 ; fi - if ! locale -a | grep -q -i utf8$ ; then + if ! locale -a | grep -q -i -E 'utf-?8$' ; then echo echo "You need at least one UTF8 locale to build a toolchain supporting locales" exit 1 ; @@ -198,23 +204,18 @@ if grep -q ^BR2_NEEDS_HOST_JAVA=y $BR2_CONFIG ; then fi fi -if grep -q ^BR2_NEEDS_HOST_JAVAC=y $BR2_CONFIG ; then - check_prog_host "javac" -fi - -if grep -q ^BR2_NEEDS_HOST_JAR=y $BR2_CONFIG ; then - check_prog_host "jar" -fi - if grep -q ^BR2_HOSTARCH_NEEDS_IA32_LIBS=y $BR2_CONFIG ; then if test ! -f /lib/ld-linux.so.2 ; then echo echo "Your Buildroot configuration uses pre-built tools for the x86 architecture," echo "but your build machine uses the x86-64 architecture without the 32 bits compatibility" echo "library." - echo "If you're running a Debian/Ubuntu distribution, install the libc6-386," + echo "If you're running a Debian/Ubuntu distribution, install the libc6-i386," echo "lib32stdc++6, and lib32z1 packages (or alternatively libc6:i386," echo "libstdc++6:i386, and zlib1g:i386)." + echo "If you're running a RedHat/Fedora distribution, install the glibc.i686 and" + echo "zlib.i686 packages." + echo "If you're running an ArchLinux distribution, install lib32-glibc." echo "For other distributions, refer to the documentation on how to install the 32 bits" echo "compatibility libraries." exit 1 @@ -222,19 +223,57 @@ if grep -q ^BR2_HOSTARCH_NEEDS_IA32_LIBS=y $BR2_CONFIG ; then fi if grep -q ^BR2_HOSTARCH_NEEDS_IA32_COMPILER=y $BR2_CONFIG ; then - if ! echo "int main(void) {}" | gcc -m32 -x c - -o /dev/null ; then + if ! echo "int main(void) {}" | gcc -m32 -x c - -o /dev/null 2>/dev/null; then echo echo "Your Buildroot configuration needs a compiler capable of building 32 bits binaries." echo "If you're running a Debian/Ubuntu distribution, install the gcc-multilib package." echo "For other distributions, refer to their documentation." exit 1 fi + + if ! echo "int main(void) {}" | g++ -m32 -x c++ - -o /dev/null 2>/dev/null; then + echo + echo "Your Buildroot configuration needs a compiler capable of building 32 bits binaries." + echo "If you're running a Debian/Ubuntu distribution, install the g++-multilib package." + echo "For other distributions, refer to their documentation." + exit 1 + fi +fi + +# Check that the Perl installation is complete enough for Buildroot. +required_perl_modules="Data::Dumper" # Needed to build host-autoconf +required_perl_modules="$required_perl_modules ExtUtils::MakeMaker" # Used by host-libxml-parser-perl +required_perl_modules="$required_perl_modules Thread::Queue" # Used by host-automake + +if grep -q ^BR2_PACKAGE_MPV=y $BR2_CONFIG ; then + required_perl_modules="$required_perl_modules Math::BigInt" + required_perl_modules="$required_perl_modules Math::BigRat" +fi + +if grep -q ^BR2_PACKAGE_WHOIS=y $BR2_CONFIG ; then + required_perl_modules="$required_perl_modules autodie" +fi + +if grep -q -E '^BR2_PACKAGE_(WEBKITGTK|WPEWEBKIT)=y' $BR2_CONFIG ; then + required_perl_modules="${required_perl_modules} JSON::PP" fi -# Check that the Perl installation is complete enough to build -# host-autoconf. -if ! perl -e "require Data::Dumper" > /dev/null 2>&1 ; then - echo "Your Perl installation is not complete enough, at least Data::Dumper is missing." - echo "On Debian/Ubuntu distributions, install the 'perl' package." +# This variable will keep the modules that are missing in your system. +missing_perl_modules="" + +for pm in $required_perl_modules ; do + if ! perl -e "require $pm" > /dev/null 2>&1 ; then + missing_perl_modules="$missing_perl_modules $pm" + fi +done + +if [ -n "$missing_perl_modules" ] ; then + echo "Your Perl installation is not complete enough; at least the following" + echo "modules are missing:" + echo + for pm in $missing_perl_modules ; do + printf "\t $pm\n" + done + echo exit 1 fi From 13a752c28e38c3579e49fe955d6854b946be0976 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 00:02:06 +0100 Subject: [PATCH 07/12] Copy utils/ from upstream --- utils/brmake | 42 ++ utils/check-package | 194 ++++++ utils/checkpackagelib/__init__.py | 0 utils/checkpackagelib/base.py | 18 + utils/checkpackagelib/lib.py | 68 ++ utils/checkpackagelib/lib_config.py | 234 +++++++ utils/checkpackagelib/lib_hash.py | 55 ++ utils/checkpackagelib/lib_mk.py | 330 +++++++++ utils/checkpackagelib/lib_patch.py | 62 ++ utils/checkpackagelib/readme.txt | 73 ++ utils/config | 206 ++++++ utils/diffconfig | 138 ++++ utils/genrandconfig | 450 +++++++++++++ utils/get-developers | 112 ++++ utils/getdeveloperlib.py | 299 +++++++++ utils/readme.txt | 45 ++ utils/scancpan | 997 ++++++++++++++++++++++++++++ utils/scanpypi | 754 +++++++++++++++++++++ utils/size-stats-compare | 130 ++++ utils/test-pkg | 269 ++++++++ 20 files changed, 4476 insertions(+) create mode 100755 utils/brmake create mode 100755 utils/check-package create mode 100644 utils/checkpackagelib/__init__.py create mode 100644 utils/checkpackagelib/base.py create mode 100644 utils/checkpackagelib/lib.py create mode 100644 utils/checkpackagelib/lib_config.py create mode 100644 utils/checkpackagelib/lib_hash.py create mode 100644 utils/checkpackagelib/lib_mk.py create mode 100644 utils/checkpackagelib/lib_patch.py create mode 100644 utils/checkpackagelib/readme.txt create mode 100755 utils/config create mode 100755 utils/diffconfig create mode 100755 utils/genrandconfig create mode 100755 utils/get-developers create mode 100644 utils/getdeveloperlib.py create mode 100644 utils/readme.txt create mode 100755 utils/scancpan create mode 100755 utils/scanpypi create mode 100755 utils/size-stats-compare create mode 100755 utils/test-pkg diff --git a/utils/brmake b/utils/brmake new file mode 100755 index 0000000000..e30119dd10 --- /dev/null +++ b/utils/brmake @@ -0,0 +1,42 @@ +#!/bin/bash +# (C) 2016, "Yann E. MORIN" +# License: WTFPL, https://spdx.org/licenses/WTFPL.html + +main() { + local found ret start d h m mf + + if ! which unbuffer >/dev/null 2>&1; then + printf "you need to install 'unbuffer' (from package expect or expect-dev)\n" >&2 + exit 1 + fi + + start=${SECONDS} + + ( exec 2>&1; unbuffer make "${@}"; ) \ + > >( while read line; do + printf "%(%Y-%m-%dT%H:%M:%S)T %s\n" -1 "${line}" + done \ + |tee -a br.log \ + |grep --colour=never -E '>>>' + ) + ret=${?} + + d=$((SECONDS-start)) + printf "Done in " + h=$((d/3600)) + d=$((d%3600)) + [ ${h} -eq 0 ] || { printf "%dh " ${h}; mf="02"; } + m=$((d/60)) + d=$((d%60)) + [ ${m} -eq 0 ] || { printf "%${mf}dmin " ${m}; sf="02"; } + printf "%${sf}ds" ${d} + + if [ ${ret} -ne 0 ]; then + printf " (error code: %s)" ${ret} + fi + printf "\n" + + return ${ret} +} + +main "${@}" diff --git a/utils/check-package b/utils/check-package new file mode 100755 index 0000000000..52317e02f4 --- /dev/null +++ b/utils/check-package @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# See utils/checkpackagelib/readme.txt before editing this file. + +from __future__ import print_function +import argparse +import inspect +import os +import re +import six +import sys + +import checkpackagelib.lib_config +import checkpackagelib.lib_hash +import checkpackagelib.lib_mk +import checkpackagelib.lib_patch + +VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES = 3 +flags = None # Command line arguments. + + +def parse_args(): + parser = argparse.ArgumentParser() + + # Do not use argparse.FileType("r") here because only files with known + # format will be open based on the filename. + parser.add_argument("files", metavar="F", type=str, nargs="*", + help="list of files") + + parser.add_argument("--br2-external", "-b", dest='intree_only', action="store_false", + help="do not apply the pathname filters used for intree files") + + parser.add_argument("--manual-url", action="store", + default="http://nightly.buildroot.org/", + help="default: %(default)s") + parser.add_argument("--verbose", "-v", action="count", default=0) + parser.add_argument("--quiet", "-q", action="count", default=0) + + # Now the debug options in the order they are processed. + parser.add_argument("--include-only", dest="include_list", action="append", + help="run only the specified functions (debug)") + parser.add_argument("--exclude", dest="exclude_list", action="append", + help="do not run the specified functions (debug)") + parser.add_argument("--dry-run", action="store_true", help="print the " + "functions that would be called for each file (debug)") + + return parser.parse_args() + + +CONFIG_IN_FILENAME = re.compile("Config\.\S*$") +DO_CHECK_INTREE = re.compile("|".join([ + "Config.in", + "arch/", + "boot/", + "fs/", + "linux/", + "package/", + "system/", + "toolchain/", + ])) +DO_NOT_CHECK_INTREE = re.compile("|".join([ + "boot/barebox/barebox\.mk$", + "fs/common\.mk$", + "package/doc-asciidoc\.mk$", + "package/pkg-\S*\.mk$", + "toolchain/helpers\.mk$", + "toolchain/toolchain-external/pkg-toolchain-external\.mk$", + ])) + + +def get_lib_from_filename(fname): + if flags.intree_only: + if DO_CHECK_INTREE.match(fname) is None: + return None + if DO_NOT_CHECK_INTREE.match(fname): + return None + else: + if os.path.basename(fname) == "external.mk" and \ + os.path.exists(fname[:-2] + "desc"): + return None + if CONFIG_IN_FILENAME.search(fname): + return checkpackagelib.lib_config + if fname.endswith(".hash"): + return checkpackagelib.lib_hash + if fname.endswith(".mk"): + return checkpackagelib.lib_mk + if fname.endswith(".patch"): + return checkpackagelib.lib_patch + return None + + +def is_a_check_function(m): + if not inspect.isclass(m): + return False + # do not call the base class + if m.__name__.startswith("_"): + return False + if flags.include_list and m.__name__ not in flags.include_list: + return False + if flags.exclude_list and m.__name__ in flags.exclude_list: + return False + return True + + +def print_warnings(warnings): + # Avoid the need to use 'return []' at the end of every check function. + if warnings is None: + return 0 # No warning generated. + + for level, message in enumerate(warnings): + if flags.verbose >= level: + print(message.replace("\t", "< tab >").rstrip()) + return 1 # One more warning to count. + + +def check_file_using_lib(fname): + # Count number of warnings generated and lines processed. + nwarnings = 0 + nlines = 0 + + lib = get_lib_from_filename(fname) + if not lib: + if flags.verbose >= VERBOSE_LEVEL_TO_SHOW_IGNORED_FILES: + print("{}: ignored".format(fname)) + return nwarnings, nlines + classes = inspect.getmembers(lib, is_a_check_function) + + if flags.dry_run: + functions_to_run = [c[0] for c in classes] + print("{}: would run: {}".format(fname, functions_to_run)) + return nwarnings, nlines + + objects = [c[1](fname, flags.manual_url) for c in classes] + + for cf in objects: + nwarnings += print_warnings(cf.before()) + if six.PY3: + f = open(fname, "r", errors="surrogateescape") + else: + f = open(fname, "r") + lastline = "" + for lineno, text in enumerate(f.readlines()): + nlines += 1 + for cf in objects: + if cf.disable.search(lastline): + continue + nwarnings += print_warnings(cf.check_line(lineno + 1, text)) + lastline = text + f.close() + for cf in objects: + nwarnings += print_warnings(cf.after()) + + return nwarnings, nlines + + +def __main__(): + global flags + flags = parse_args() + + if flags.intree_only: + # change all paths received to be relative to the base dir + base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + files_to_check = [os.path.relpath(os.path.abspath(f), base_dir) for f in flags.files] + # move current dir so the script find the files + os.chdir(base_dir) + else: + files_to_check = flags.files + + if len(files_to_check) == 0: + print("No files to check style") + sys.exit(1) + + # Accumulate number of warnings generated and lines processed. + total_warnings = 0 + total_lines = 0 + + for fname in files_to_check: + nwarnings, nlines = check_file_using_lib(fname) + total_warnings += nwarnings + total_lines += nlines + + # The warning messages are printed to stdout and can be post-processed + # (e.g. counted by 'wc'), so for stats use stderr. Wait all warnings are + # printed, for the case there are many of them, before printing stats. + sys.stdout.flush() + + if not flags.quiet: + print("{} lines processed".format(total_lines), file=sys.stderr) + print("{} warnings generated".format(total_warnings), file=sys.stderr) + + if total_warnings > 0: + sys.exit(1) + + +__main__() diff --git a/utils/checkpackagelib/__init__.py b/utils/checkpackagelib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/checkpackagelib/base.py b/utils/checkpackagelib/base.py new file mode 100644 index 0000000000..9544a64e5a --- /dev/null +++ b/utils/checkpackagelib/base.py @@ -0,0 +1,18 @@ +# See utils/checkpackagelib/readme.txt before editing this file. +import re + + +class _CheckFunction(object): + def __init__(self, filename, url_to_manual): + self.filename = filename + self.url_to_manual = url_to_manual + self.disable = re.compile(r"^\s*# check-package .*\b{}\b".format(self.__class__.__name__)) + + def before(self): + pass + + def check_line(self, lineno, text): + pass + + def after(self): + pass diff --git a/utils/checkpackagelib/lib.py b/utils/checkpackagelib/lib.py new file mode 100644 index 0000000000..457b3c3171 --- /dev/null +++ b/utils/checkpackagelib/lib.py @@ -0,0 +1,68 @@ +# See utils/checkpackagelib/readme.txt before editing this file. + +from checkpackagelib.base import _CheckFunction + + +class ConsecutiveEmptyLines(_CheckFunction): + def before(self): + self.lastline = "non empty" + + def check_line(self, lineno, text): + if text.strip() == "" == self.lastline.strip(): + return ["{}:{}: consecutive empty lines" + .format(self.filename, lineno)] + self.lastline = text + + +class EmptyLastLine(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "non empty" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline.strip() == "": + return ["{}:{}: empty line at end of file" + .format(self.filename, self.lastlineno)] + + +class NewlineAtEof(_CheckFunction): + def before(self): + self.lastlineno = 0 + self.lastline = "\n" + + def check_line(self, lineno, text): + self.lastlineno = lineno + self.lastline = text + + def after(self): + if self.lastline == self.lastline.rstrip("\r\n"): + return ["{}:{}: missing newline at end of file" + .format(self.filename, self.lastlineno), + self.lastline] + + +class TrailingSpace(_CheckFunction): + def check_line(self, lineno, text): + line = text.rstrip("\r\n") + if line != line.rstrip(): + return ["{}:{}: line contains trailing whitespace" + .format(self.filename, lineno), + text] + + +class Utf8Characters(_CheckFunction): + def is_ascii(self, s): + try: + return all(ord(c) < 128 for c in s) + except TypeError: + return False + + def check_line(self, lineno, text): + if not self.is_ascii(text): + return ["{}:{}: line contains UTF-8 characters" + .format(self.filename, lineno), + text] diff --git a/utils/checkpackagelib/lib_config.py b/utils/checkpackagelib/lib_config.py new file mode 100644 index 0000000000..55c8589d71 --- /dev/null +++ b/utils/checkpackagelib/lib_config.py @@ -0,0 +1,234 @@ +# See utils/checkpackagelib/readme.txt before editing this file. +# Kconfig generates errors if someone introduces a typo like "boool" instead of +# "bool", so below check functions don't need to check for things already +# checked by running "make menuconfig". + +import re + +from checkpackagelib.base import _CheckFunction +from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401 +from checkpackagelib.lib import EmptyLastLine # noqa: F401 +from checkpackagelib.lib import NewlineAtEof # noqa: F401 +from checkpackagelib.lib import TrailingSpace # noqa: F401 + + +def _empty_or_comment(text): + line = text.strip() + # ignore empty lines and comment lines indented or not + return line == "" or line.startswith("#") + + +def _part_of_help_text(text): + return text.startswith("\t ") + + +# used in more than one check +entries_that_should_not_be_indented = [ + "choice", "comment", "config", "endchoice", "endif", "endmenu", "if", + "menu", "menuconfig", "source"] + + +class AttributesOrder(_CheckFunction): + attributes_order_convention = { + "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3, + "select": 4, "help": 5} + + def before(self): + self.state = 0 + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + return + + attribute = text.split()[0] + + if attribute in entries_that_should_not_be_indented: + self.state = 0 + return + if attribute not in self.attributes_order_convention.keys(): + return + new_state = self.attributes_order_convention[attribute] + wrong_order = self.state > new_state + + # save to process next line + self.state = new_state + + if wrong_order: + return ["{}:{}: attributes order: type, default, depends on," + " select, help ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class CommentsMenusPackagesOrder(_CheckFunction): + def before(self): + self.level = 0 + self.menu_of_packages = ["The top level menu"] + self.new_package = "" + self.package = [""] + self.print_package_warning = [True] + self.state = "" + + def get_level(self): + return len(self.state.split('-')) - 1 + + def initialize_package_level_elements(self, text): + try: + self.menu_of_packages[self.level] = text[:-1] + self.package[self.level] = "" + self.print_package_warning[self.level] = True + except IndexError: + self.menu_of_packages.append(text[:-1]) + self.package.append("") + self.print_package_warning.append(True) + + def initialize_level_elements(self, text): + self.level = self.get_level() + self.initialize_package_level_elements(text) + + def check_line(self, lineno, text): + # We only want to force sorting for the top-level menus + if self.filename not in ["fs/Config.in", + "package/Config.in", + "package/Config.in.host", + "package/kodi/Config.in"]: + return + + source_line = re.match(r'^\s*source ".*/([^/]*)/Config.in(.host)?"', text) + + if text.startswith("comment "): + if not self.state.endswith("-comment"): + self.state += "-comment" + + self.initialize_level_elements(text) + + elif text.startswith("if "): + self.state += "-if" + + self.initialize_level_elements(text) + + elif text.startswith("menu "): + if self.state.endswith("-comment"): + self.state = self.state[:-8] + + self.state += "-menu" + + self.initialize_level_elements(text) + + elif text.startswith("endif") or text.startswith("endmenu"): + if self.state.endswith("-comment"): + self.state = self.state[:-8] + + if text.startswith("endif"): + self.state = self.state[:-3] + + elif text.startswith("endmenu"): + self.state = self.state[:-5] + + self.level = self.get_level() + + elif source_line: + self.new_package = source_line.group(1) + + # We order _ before A, so replace it with . + new_package_ord = self.new_package.replace('_', '.') + + if self.package[self.level] != "" and \ + self.print_package_warning[self.level] and \ + new_package_ord < self.package[self.level]: + self.print_package_warning[self.level] = False + prefix = "{}:{}: ".format(self.filename, lineno) + spaces = " " * len(prefix) + return ["{prefix}Packages in: {menu},\n" + "{spaces}are not alphabetically ordered;\n" + "{spaces}correct order: '-', '_', digits, capitals, lowercase;\n" + "{spaces}first incorrect package: {package}" + .format(prefix=prefix, spaces=spaces, + menu=self.menu_of_packages[self.level], + package=self.new_package), + text] + + self.package[self.level] = new_package_ord + + +class HelpText(_CheckFunction): + HELP_TEXT_FORMAT = re.compile("^\t .{,62}$") + URL_ONLY = re.compile("^(http|https|git)://\S*$") + + def before(self): + self.help_text = False + + def check_line(self, lineno, text): + if _empty_or_comment(text): + return + + entry = text.split()[0] + + if entry in entries_that_should_not_be_indented: + self.help_text = False + return + if text.strip() == "help": + self.help_text = True + return + + if not self.help_text: + return + + if self.HELP_TEXT_FORMAT.match(text.rstrip()): + return + if self.URL_ONLY.match(text.strip()): + return + return ["{}:{}: help text: <2 spaces><62 chars>" + " ({}#writing-rules-config-in)" + .format(self.filename, lineno, self.url_to_manual), + text, + "\t " + "123456789 " * 6 + "12"] + + +class Indent(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + entries_that_should_be_indented = [ + "bool", "default", "depends", "help", "prompt", "select", "string"] + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + if _empty_or_comment(text) or _part_of_help_text(text): + self.backslash = False + return + + entry = text.split()[0] + + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if last_line_ends_in_backslash: + if text.startswith("\t"): + return + return ["{}:{}: continuation line should be indented using tabs" + .format(self.filename, lineno), + text] + + if entry in self.entries_that_should_be_indented: + if not text.startswith("\t{}".format(entry)): + return ["{}:{}: should be indented with one tab" + " ({}#_config_files)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif entry in entries_that_should_not_be_indented: + if not text.startswith(entry): + # four Config.in files have a special but legitimate indentation rule + if self.filename in ["package/Config.in", + "package/Config.in.host", + "package/kodi/Config.in", + "package/x11r7/Config.in"]: + return + return ["{}:{}: should not be indented" + .format(self.filename, lineno), + text] diff --git a/utils/checkpackagelib/lib_hash.py b/utils/checkpackagelib/lib_hash.py new file mode 100644 index 0000000000..3e381119a5 --- /dev/null +++ b/utils/checkpackagelib/lib_hash.py @@ -0,0 +1,55 @@ +# See utils/checkpackagelib/readme.txt before editing this file. +# The validity of the hashes itself is checked when building, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-source". + +import re + +from checkpackagelib.base import _CheckFunction +from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401 +from checkpackagelib.lib import EmptyLastLine # noqa: F401 +from checkpackagelib.lib import NewlineAtEof # noqa: F401 +from checkpackagelib.lib import TrailingSpace # noqa: F401 + + +def _empty_line_or_comment(text): + return text.strip() == "" or text.startswith("#") + + +class HashNumberOfFields(_CheckFunction): + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) != 3: + return ["{}:{}: expected three fields ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class HashType(_CheckFunction): + len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, + "sha384": 96, "sha512": 128} + + def check_line(self, lineno, text): + if _empty_line_or_comment(text): + return + + fields = text.split() + if len(fields) < 2: + return + + htype, hexa = fields[:2] + if htype == "none": + return + if htype not in self.len_of_hash.keys(): + return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text] + if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa): + return ["{}:{}: hash size does not match type " + "({}#adding-packages-hash)" + .format(self.filename, lineno, self.url_to_manual), + text, + "expected {} hex digits".format(self.len_of_hash[htype])] diff --git a/utils/checkpackagelib/lib_mk.py b/utils/checkpackagelib/lib_mk.py new file mode 100644 index 0000000000..a0caf84636 --- /dev/null +++ b/utils/checkpackagelib/lib_mk.py @@ -0,0 +1,330 @@ +# See utils/checkpackagelib/readme.txt before editing this file. +# There are already dependency checks during the build, so below check +# functions don't need to check for things already checked by exploring the +# menu options using "make menuconfig" and by running "make" with appropriate +# packages enabled. + +import os +import re + +from checkpackagelib.base import _CheckFunction +from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401 +from checkpackagelib.lib import EmptyLastLine # noqa: F401 +from checkpackagelib.lib import NewlineAtEof # noqa: F401 +from checkpackagelib.lib import TrailingSpace # noqa: F401 +from checkpackagelib.lib import Utf8Characters # noqa: F401 + +# used in more than one check +start_conditional = ["ifdef", "ifeq", "ifndef", "ifneq"] +end_conditional = ["endif"] + + +class Indent(_CheckFunction): + COMMENT = re.compile("^\s*#") + CONDITIONAL = re.compile("^\s*({})\s".format("|".join(start_conditional + end_conditional))) + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + END_DEFINE = re.compile("^\s*endef\s") + MAKEFILE_TARGET = re.compile("^[^# \t]+:\s") + START_DEFINE = re.compile("^\s*define\s") + + def before(self): + self.define = False + self.backslash = False + self.makefile_target = False + + def check_line(self, lineno, text): + if self.START_DEFINE.search(text): + self.define = True + return + if self.END_DEFINE.search(text): + self.define = False + return + + expect_tabs = False + if self.define or self.backslash or self.makefile_target: + expect_tabs = True + if self.CONDITIONAL.search(text): + expect_tabs = False + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + else: + self.backslash = False + + if self.MAKEFILE_TARGET.search(text): + self.makefile_target = True + return + if text.strip() == "": + self.makefile_target = False + return + + # comment can be indented or not inside define ... endef, so ignore it + if self.define and self.COMMENT.search(text): + return + + if expect_tabs: + if not text.startswith("\t"): + return ["{}:{}: expected indent with tabs" + .format(self.filename, lineno), + text] + else: + if text.startswith("\t"): + return ["{}:{}: unexpected indent with tabs" + .format(self.filename, lineno), + text] + + +class OverriddenVariable(_CheckFunction): + CONCATENATING = re.compile("^([A-Z0-9_]+)\s*(\+|:|)=\s*\$\(\\1\)") + END_CONDITIONAL = re.compile("^\s*({})".format("|".join(end_conditional))) + OVERRIDING_ASSIGNMENTS = [':=', "="] + START_CONDITIONAL = re.compile("^\s*({})".format("|".join(start_conditional))) + VARIABLE = re.compile("^([A-Z0-9_]+)\s*((\+|:|)=)") + USUALLY_OVERRIDDEN = re.compile("^[A-Z0-9_]+({})".format("|".join([ + "_ARCH\s*=\s*", + "_CPU\s*=\s*", + "_SITE\s*=\s*", + "_SOURCE\s*=\s*", + "_VERSION\s*=\s*"]))) + + def before(self): + self.conditional = 0 + self.unconditionally_set = [] + self.conditionally_set = [] + + def check_line(self, lineno, text): + if self.START_CONDITIONAL.search(text): + self.conditional += 1 + return + if self.END_CONDITIONAL.search(text): + self.conditional -= 1 + return + + m = self.VARIABLE.search(text) + if m is None: + return + variable, assignment = m.group(1, 2) + + if self.conditional == 0: + if variable in self.conditionally_set: + self.unconditionally_set.append(variable) + if assignment in self.OVERRIDING_ASSIGNMENTS: + return ["{}:{}: unconditional override of variable {} previously conditionally set" + .format(self.filename, lineno, variable), + text] + + if variable not in self.unconditionally_set: + self.unconditionally_set.append(variable) + return + if assignment in self.OVERRIDING_ASSIGNMENTS: + return ["{}:{}: unconditional override of variable {}" + .format(self.filename, lineno, variable), + text] + else: + if variable not in self.unconditionally_set: + self.conditionally_set.append(variable) + return + if self.CONCATENATING.search(text): + return ["{}:{}: immediate assignment to append to variable {}" + .format(self.filename, lineno, variable), + text] + if self.USUALLY_OVERRIDDEN.search(text): + return + if assignment in self.OVERRIDING_ASSIGNMENTS: + return ["{}:{}: conditional override of variable {}" + .format(self.filename, lineno, variable), + text] + + +class PackageHeader(_CheckFunction): + def before(self): + self.skip = False + + def check_line(self, lineno, text): + if self.skip or lineno > 6: + return + + if lineno in [1, 5]: + if lineno == 1 and text.startswith("include "): + self.skip = True + return + if text.rstrip() != "#" * 80: + return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text, + "#" * 80] + elif lineno in [2, 4]: + if text.rstrip() != "#": + return ["{}:{}: should be 1 hash ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + elif lineno == 6: + if text.rstrip() != "": + return ["{}:{}: should be a blank line ({}#writing-rules-mk)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class RemoveDefaultPackageSourceVariable(_CheckFunction): + packages_that_may_contain_default_source = ["binutils", "gcc", "gdb"] + + def before(self): + package, _ = os.path.splitext(os.path.basename(self.filename)) + package_upper = package.replace("-", "_").upper() + self.package = package + self.FIND_SOURCE = re.compile( + "^{}_SOURCE\s*=\s*{}-\$\({}_VERSION\)\.tar\.gz" + .format(package_upper, package, package_upper)) + + def check_line(self, lineno, text): + if self.FIND_SOURCE.search(text): + + if self.package in self.packages_that_may_contain_default_source: + return + + return ["{}:{}: remove default value of _SOURCE variable " + "({}#generic-package-reference)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class SpaceBeforeBackslash(_CheckFunction): + TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*( |\t ?)\\$") + + def check_line(self, lineno, text): + if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()): + return ["{}:{}: use only one space before backslash" + .format(self.filename, lineno), + text] + + +class TrailingBackslash(_CheckFunction): + ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$") + + def before(self): + self.backslash = False + + def check_line(self, lineno, text): + last_line_ends_in_backslash = self.backslash + + # calculate for next line + if self.ENDS_WITH_BACKSLASH.search(text): + self.backslash = True + self.lastline = text + return + self.backslash = False + + if last_line_ends_in_backslash and text.strip() == "": + return ["{}:{}: remove trailing backslash" + .format(self.filename, lineno - 1), + self.lastline] + + +class TypoInPackageVariable(_CheckFunction): + ALLOWED = re.compile("|".join([ + "ACLOCAL_DIR", + "ACLOCAL_HOST_DIR", + "ACLOCAL_PATH", + "BR_CCACHE_INITIAL_SETUP", + "BR_LIBC", + "BR_NO_CHECK_HASH_FOR", + "LINUX_EXTENSIONS", + "LINUX_POST_PATCH_HOOKS", + "LINUX_TOOLS", + "LUA_RUN", + "MKFS_JFFS2", + "MKIMAGE_ARCH", + "PACKAGES_PERMISSIONS_TABLE", + "PKG_CONFIG_HOST_BINARY", + "SUMTOOL", + "TARGET_FINALIZE_HOOKS", + "TARGETS_ROOTFS", + "XTENSA_CORE_NAME"])) + VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=") + + def before(self): + package, _ = os.path.splitext(os.path.basename(self.filename)) + package = package.replace("-", "_").upper() + # linux tools do not use LINUX_TOOL_ prefix for variables + package = package.replace("LINUX_TOOL_", "") + # linux extensions do not use LINUX_EXT_ prefix for variables + package = package.replace("LINUX_EXT_", "") + self.package = package + self.REGEX = re.compile("^(HOST_|ROOTFS_)?({}_[A-Z0-9_]+)".format(package)) + self.FIND_VIRTUAL = re.compile( + "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package)) + self.virtual = [] + + def check_line(self, lineno, text): + m = self.VARIABLE.search(text) + if m is None: + return + + variable = m.group(1) + + # allow to set variables for virtual package this package provides + v = self.FIND_VIRTUAL.search(text) + if v: + self.virtual += v.group(2).upper().split() + return + for virtual in self.virtual: + if variable.startswith("{}_".format(virtual)): + return + + if self.ALLOWED.match(variable): + return + if self.REGEX.search(text) is None: + return ["{}:{}: possible typo: {} -> *{}*" + .format(self.filename, lineno, variable, self.package), + text] + + +class UselessFlag(_CheckFunction): + DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([ + "_AUTORECONF\s*=\s*NO", + "_LIBTOOL_PATCH\s*=\s*YES"]))) + DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([ + "_INSTALL_IMAGES\s*=\s*NO", + "_INSTALL_REDISTRIBUTE\s*=\s*YES", + "_INSTALL_STAGING\s*=\s*NO", + "_INSTALL_TARGET\s*=\s*YES"]))) + END_CONDITIONAL = re.compile("^\s*({})".format("|".join(end_conditional))) + START_CONDITIONAL = re.compile("^\s*({})".format("|".join(start_conditional))) + + def before(self): + self.conditional = 0 + + def check_line(self, lineno, text): + if self.START_CONDITIONAL.search(text): + self.conditional += 1 + return + if self.END_CONDITIONAL.search(text): + self.conditional -= 1 + return + + # allow non-default conditionally overridden by default + if self.conditional > 0: + return + + if self.DEFAULT_GENERIC_FLAG.search(text): + return ["{}:{}: useless default value ({}#" + "_infrastructure_for_packages_with_specific_build_systems)" + .format(self.filename, lineno, self.url_to_manual), + text] + + if self.DEFAULT_AUTOTOOLS_FLAG.search(text) and not text.lstrip().startswith("HOST_"): + return ["{}:{}: useless default value " + "({}#_infrastructure_for_autotools_based_packages)" + .format(self.filename, lineno, self.url_to_manual), + text] + + +class VariableWithBraces(_CheckFunction): + VARIABLE_WITH_BRACES = re.compile(r"^[^#].*[^$]\${\w+}") + + def check_line(self, lineno, text): + if self.VARIABLE_WITH_BRACES.match(text.rstrip()): + return ["{}:{}: use $() to delimit variables, not ${{}}" + .format(self.filename, lineno), + text] diff --git a/utils/checkpackagelib/lib_patch.py b/utils/checkpackagelib/lib_patch.py new file mode 100644 index 0000000000..438353ad3b --- /dev/null +++ b/utils/checkpackagelib/lib_patch.py @@ -0,0 +1,62 @@ +# See utils/checkpackagelib/readme.txt before editing this file. +# The format of the patch files is tested during the build, so below check +# functions don't need to check for things already checked by running +# "make package-dirclean package-patch". + +import os +import re + +from checkpackagelib.base import _CheckFunction +from checkpackagelib.lib import NewlineAtEof # noqa: F401 + + +class ApplyOrder(_CheckFunction): + APPLY_ORDER = re.compile("\d{1,4}-[^/]*$") + + def before(self): + if not self.APPLY_ORDER.match(os.path.basename(self.filename)): + return ["{}:0: use name -.patch " + "({}#_providing_patches)" + .format(self.filename, self.url_to_manual)] + + +class NumberedSubject(_CheckFunction): + NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]") + + def before(self): + self.git_patch = False + self.lineno = 0 + self.text = None + + def check_line(self, lineno, text): + if text.startswith("diff --git"): + self.git_patch = True + return + if self.NUMBERED_PATCH.search(text): + self.lineno = lineno + self.text = text + + def after(self): + if self.git_patch and self.text: + return ["{}:{}: generate your patches with 'git format-patch -N'" + .format(self.filename, self.lineno), + self.text] + + +class Sob(_CheckFunction): + SOB_ENTRY = re.compile("^Signed-off-by: .*$") + + def before(self): + self.found = False + + def check_line(self, lineno, text): + if self.found: + return + if self.SOB_ENTRY.search(text): + self.found = True + + def after(self): + if not self.found: + return ["{}:0: missing Signed-off-by in the header " + "({}#_format_and_licensing_of_the_package_patches)" + .format(self.filename, self.url_to_manual)] diff --git a/utils/checkpackagelib/readme.txt b/utils/checkpackagelib/readme.txt new file mode 100644 index 0000000000..3bfe289607 --- /dev/null +++ b/utils/checkpackagelib/readme.txt @@ -0,0 +1,73 @@ +How the scripts are structured: +- check-package is the main engine, called by the user. + For each input file, this script decides which parser should be used and it + collects all classes declared in the library file and instantiates them. + The main engine opens the input files and it serves each raw line (including + newline!) to the method check_line() of every check object. + Two special methods before() and after() are used to call the initialization + of variables (for the case it needs to keep data across calls) and the + equivalent finalization (e.g. for the case a warning must be issued if some + pattern is not in the input file). +- base.py contains the base class for all check functions. +- lib.py contains the classes for common check functions. + Each check function is explicitly included in a given type-parsing library. + Do not include every single check function in this file, a class that will + only parse hash files should be implemented in the hash-parsing library. + When a warning must be issued, the check function returns an array of strings. + Each string is a warning message and is displayed if the corresponding verbose + level is active. When the script is called without --verbose only the first + warning in the returned array is printed; when called with --verbose both + first and second warnings are printed; when called with -vv until the third + warning is printed; an so on. + Helper functions can be defined and will not be called by the main script. +- lib_type.py contains check functions specific to files of this type. + +Some hints when changing this code: +- prefer O(n) algorithms, where n is the total number of lines in the files + processed. +- when there is no other reason for ordering, use alphabetical order (e.g. keep + the check functions in alphabetical order, keep the imports in alphabetical + order, and so on). +- keep in mind that for every class the method before() will be called before + any line is served to be checked by the method check_line(). A class that + checks the filename should only implement the method before(). A function that + needs to keep data across calls (e.g. keep the last line before the one being + processed) should initialize all variables using this method. +- keep in mind that for every class the method after() will be called after all + lines were served to be checked by the method check_line(). A class that + checks the absence of a pattern in the file will need to use this method. +- try to avoid false warnings. It's better to not issue a warning message to a + corner case than have too many false warnings. The second can make users stop + using the script. +- do not check spacing in the input line in every single function. Trailing + whitespace and wrong indentation should be checked by separate functions. +- avoid duplicate tests. Try to test only one thing in each function. +- in the warning message, include the url to a section from the manual, when + applicable. It potentially will make more people know the manual. +- use short sentences in the warning messages. A complete explanation can be + added to show when --verbose is used. +- when testing, verify the error message is displayed when the error pattern is + found, but also verify the error message is not displayed for few + well-formatted packages... there are many of these, just pick your favorite + as golden package that should not trigger any warning message. +- check the url displayed by the warning message works. + +Usage examples: +- to get a list of check functions that would be called without actually + calling them you can use the --dry-run option: +$ utils/check-package --dry-run package/yourfavorite/* + +- when you just added a new check function, e.g. Something, check how it behaves + for all current packages: +$ utils/check-package --include-only Something $(find package -type f) + +- the effective processing time (when the .pyc were already generated and all + files to be processed are cached in the RAM) should stay in the order of few + seconds: +$ utils/check-package $(find package -type f) >/dev/null ; \ + time utils/check-package $(find package -type f) >/dev/null + +- vim users can navigate the warnings (most editors probably have similar + function) since warnings are generated in the form 'path/file:line: warning': +$ find package/ -name 'Config.*' > filelist && vim -c \ + 'set makeprg=utils/check-package\ $(cat\ filelist)' -c make -c copen diff --git a/utils/config b/utils/config new file mode 100755 index 0000000000..c5e2d723ea --- /dev/null +++ b/utils/config @@ -0,0 +1,206 @@ +#!/bin/bash +# Manipulate options in a .config file from the command line + +myname=${0##*/} + +# If no prefix forced, use the default BR2_ +BR2_PREFIX="${BR2_PREFIX-BR2_}" + +usage() { + cat >&2 <>"$FN" + fi +} + +undef_var() { + local name=$1 + + txt_delete "^$name=" "$FN" + txt_delete "^# $name is not set" "$FN" +} + +if [ "$1" = "--file" ]; then + FN="$2" + if [ "$FN" = "" ] ; then + usage + fi + shift 2 +else + FN=.config +fi + +if [ "$1" = "" ] ; then + usage +fi + +MUNGE_CASE=yes +while [ "$1" != "" ] ; do + CMD="$1" + shift + case "$CMD" in + --keep-case|-k) + MUNGE_CASE=no + continue + ;; + --package|-p) + BR2_PREFIX="BR2_PACKAGE_" + continue + ;; + --*-after|-E|-D|-M) + checkarg "$1" + A=$ARG + checkarg "$2" + B=$ARG + shift 2 + ;; + -*) + checkarg "$1" + shift + ;; + esac + case "$CMD" in + --enable|-e) + set_var "${BR2_PREFIX}$ARG" "${BR2_PREFIX}$ARG=y" + ;; + + --disable|-d) + set_var "${BR2_PREFIX}$ARG" "# ${BR2_PREFIX}$ARG is not set" + ;; + + --set-str) + # sed swallows one level of escaping, so we need double-escaping + set_var "${BR2_PREFIX}$ARG" "${BR2_PREFIX}$ARG=\"${1//\"/\\\\\"}\"" + shift + ;; + + --set-val) + set_var "${BR2_PREFIX}$ARG" "${BR2_PREFIX}$ARG=$1" + shift + ;; + --undefine|-u) + undef_var "${BR2_PREFIX}$ARG" + ;; + + --state|-s) + if grep -q "# ${BR2_PREFIX}$ARG is not set" $FN ; then + echo n + else + V="$(grep "^${BR2_PREFIX}$ARG=" $FN)" + if [ $? != 0 ] ; then + echo undef + else + V="${V/#${BR2_PREFIX}$ARG=/}" + V="${V/#\"/}" + V="${V/%\"/}" + V="${V//\\\"/\"}" + echo "${V}" + fi + fi + ;; + + --enable-after|-E) + set_var "${BR2_PREFIX}$B" "${BR2_PREFIX}$B=y" "${BR2_PREFIX}$A" + ;; + + --disable-after|-D) + set_var "${BR2_PREFIX}$B" "# ${BR2_PREFIX}$B is not set" "${BR2_PREFIX}$A" + ;; + + *) + usage + ;; + esac +done diff --git a/utils/diffconfig b/utils/diffconfig new file mode 100755 index 0000000000..f1af23cfce --- /dev/null +++ b/utils/diffconfig @@ -0,0 +1,138 @@ +#!/usr/bin/python +# +# diffconfig - a tool to compare .config files. +# +# originally written in 2006 by Matt Mackall +# (at least, this was in his bloatwatch source code) +# last worked on 2008 by Tim Bird for the Linux kernel +# Adapted to Buildroot 2017 by Marcus Folkesson +# + +import sys, os + +def usage(): + print("""Usage: diffconfig [-h] [-m] [ ] + +Diffconfig is a simple utility for comparing two .config files. +Using standard diff to compare .config files often includes extraneous and +distracting information. This utility produces sorted output with only the +changes in configuration values between the two files. + +Added and removed items are shown with a leading plus or minus, respectively. +Changed items show the old and new values on a single line. + +If -m is specified, then output will be in "merge" style, which has the +changed and new values in kernel config option format. + +If no config files are specified, .config and .config.old are used. + +Example usage: + $ diffconfig .config config-with-some-changes +-BR2_LINUX_KERNEL_INTREE_DTS_NAME "vexpress-v2p-ca9" + BR2_LINUX_KERNEL_DTS_SUPPORT y -> n + BR2_LINUX_KERNEL_USE_INTREE_DTS y -> n + BR2_PACKAGE_DFU_UTIL n -> y + BR2_PACKAGE_LIBUSB n -> y + BR2_TARGET_GENERIC_HOSTNAME "buildroot" -> "Tuxie" + BR2_TARGET_GENERIC_ISSUE "Welcome to Buildroot" -> "Welcome to CustomBoard" ++BR2_PACKAGE_LIBUSB_COMPAT n + +""") + sys.exit(0) + +# returns a dictionary of name/value pairs for config items in the file +def readconfig(config_file): + d = {} + for line in config_file: + line = line.strip() + if len(line) == 0: + continue + if line[-11:] == " is not set": + d[line[2:-11]] = "n" + elif line[0] != "#": + name, val = line.split("=", 1) + d[name] = val + return d + +def print_config(op, config, value, new_value): + global merge_style + + if merge_style: + if new_value: + if new_value=="n": + print("# %s is not set" % config) + else: + print("%s=%s" % (config, new_value)) + else: + if op=="-": + print("-%s %s" % (config, value)) + elif op=="+": + print("+%s %s" % (config, new_value)) + else: + print(" %s %s -> %s" % (config, value, new_value)) + +def main(): + global merge_style + + # parse command line args + if ("-h" in sys.argv or "--help" in sys.argv): + usage() + + merge_style = 0 + if "-m" in sys.argv: + merge_style = 1 + sys.argv.remove("-m") + + argc = len(sys.argv) + if not (argc==1 or argc == 3): + print("Error: incorrect number of arguments or unrecognized option") + usage() + + if argc == 1: + # if no filenames given, assume .config and .config.old + build_dir="" + if "KBUILD_OUTPUT" in os.environ: + build_dir = os.environ["KBUILD_OUTPUT"]+"/" + configa_filename = build_dir + ".config.old" + configb_filename = build_dir + ".config" + else: + configa_filename = sys.argv[1] + configb_filename = sys.argv[2] + + try: + a = readconfig(open(configa_filename)) + b = readconfig(open(configb_filename)) + except (IOError): + e = sys.exc_info()[1] + print("I/O error[%s]: %s\n" % (e.args[0],e.args[1])) + usage() + + # print items in a but not b (accumulate, sort and print) + old = [] + for config in a: + if config not in b: + old.append(config) + old.sort() + for config in old: + print_config("-", config, a[config], None) + del a[config] + + # print items that changed (accumulate, sort, and print) + changed = [] + for config in a: + if a[config] != b[config]: + changed.append(config) + else: + del b[config] + changed.sort() + for config in changed: + print_config("->", config, a[config], b[config]) + del b[config] + + # now print items in b but not in a + # (items from b that were in a were removed above) + new = sorted(b.keys()) + for config in new: + print_config("+", config, None, b[config]) + +main() diff --git a/utils/genrandconfig b/utils/genrandconfig new file mode 100755 index 0000000000..63ff32cbfb --- /dev/null +++ b/utils/genrandconfig @@ -0,0 +1,450 @@ +#!/usr/bin/env python + +# Copyright (C) 2014 by Thomas Petazzoni +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# This script generates a random configuration for testing Buildroot. + +from __future__ import print_function + +import contextlib +import csv +import os +from random import randint +import subprocess +import sys +from distutils.version import StrictVersion +import platform + +if sys.hexversion >= 0x3000000: + import urllib.request as _urllib +else: + import urllib2 as _urllib + + +def urlopen_closing(uri): + return contextlib.closing(_urllib.urlopen(uri)) + + +class SystemInfo: + DEFAULT_NEEDED_PROGS = ["make", "git", "gcc", "timeout"] + DEFAULT_OPTIONAL_PROGS = ["bzr", "java", "javac", "jar", "diffoscope"] + + def __init__(self): + self.needed_progs = list(self.__class__.DEFAULT_NEEDED_PROGS) + self.optional_progs = list(self.__class__.DEFAULT_OPTIONAL_PROGS) + self.progs = {} + + def find_prog(self, name, flags=os.X_OK, env=os.environ): + if not name or name[0] == os.sep: + raise ValueError(name) + + prog_path = env.get("PATH", None) + # for windows compatibility, we'd need to take PATHEXT into account + + if prog_path: + for prog_dir in filter(None, prog_path.split(os.pathsep)): + # os.join() not necessary: non-empty prog_dir + # and name[0] != os.sep + prog = prog_dir + os.sep + name + if os.access(prog, flags): + return prog + # -- + return None + + def has(self, prog): + """Checks whether a program is available. + Lazily evaluates missing entries. + + Returns: None if prog not found, else path to the program [evaluates + to True] + """ + try: + return self.progs[prog] + except KeyError: + pass + + have_it = self.find_prog(prog) + # java[c] needs special care + if have_it and prog in ('java', 'javac'): + with open(os.devnull, "w") as devnull: + if subprocess.call("%s -version | grep gcj" % prog, + shell=True, + stdout=devnull, stderr=devnull) != 1: + have_it = False + # -- + self.progs[prog] = have_it + return have_it + + def check_requirements(self): + """Checks program dependencies. + + Returns: True if all mandatory programs are present, else False. + """ + do_check_has_prog = self.has + + missing_requirements = False + for prog in self.needed_progs: + if not do_check_has_prog(prog): + print("ERROR: your system lacks the '%s' program" % prog) + missing_requirements = True + + # check optional programs here, + # else they'd get checked by each worker instance + for prog in self.optional_progs: + do_check_has_prog(prog) + + return not missing_requirements + + +def get_toolchain_configs(toolchains_csv, buildrootdir): + """Fetch and return the possible toolchain configurations + + This function returns an array of toolchain configurations. Each + toolchain configuration is itself an array of lines of the defconfig. + """ + + with open(toolchains_csv) as r: + # filter empty lines and comments + lines = [t for t in r.readlines() if len(t.strip()) > 0 and t[0] != '#'] + toolchains = lines + configs = [] + + (_, _, _, _, hostarch) = os.uname() + # ~2015 distros report x86 when on a 32bit install + if hostarch == 'i686' or hostarch == 'i386' or hostarch == 'x86': + hostarch = 'x86' + + for row in csv.reader(toolchains): + config = {} + configfile = row[0] + config_hostarch = row[1] + keep = False + + # Keep all toolchain configs that work regardless of the host + # architecture + if config_hostarch == "any": + keep = True + + # Keep all toolchain configs that can work on the current host + # architecture + if hostarch == config_hostarch: + keep = True + + # Assume that x86 32 bits toolchains work on x86_64 build + # machines + if hostarch == 'x86_64' and config_hostarch == "x86": + keep = True + + if not keep: + continue + + if not os.path.isabs(configfile): + configfile = os.path.join(buildrootdir, configfile) + + with open(configfile) as r: + config = r.readlines() + configs.append(config) + return configs + + +def is_toolchain_usable(configfile, config): + """Check if the toolchain is actually usable.""" + + with open(configfile) as configf: + configlines = configf.readlines() + + # Check that the toolchain configuration is still present + for toolchainline in config: + if toolchainline not in configlines: + print("WARN: toolchain can't be used", file=sys.stderr) + print(" Missing: %s" % toolchainline.strip(), file=sys.stderr) + return False + + # The latest Linaro toolchains on x86-64 hosts requires glibc + # 2.14+ on the host. + if platform.machine() == 'x86_64': + if 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARM=y\n' in configlines or \ + 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \ + 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \ + 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines: + ldd_version_output = subprocess.check_output(['ldd', '--version']) + glibc_version = ldd_version_output.splitlines()[0].split()[-1] + if StrictVersion('2.14') > StrictVersion(glibc_version): + print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr) + return False + + return True + + +def fixup_config(sysinfo, configfile): + """Finalize the configuration and reject any problematic combinations + + This function returns 'True' when the configuration has been + accepted, and 'False' when the configuration has not been accepted because + it is known to fail (in which case another random configuration will be + generated). + """ + + with open(configfile) as configf: + configlines = configf.readlines() + + BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/' + + if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"): + return False + # The ctng toolchain is affected by PR58854 + if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines: + return False + # The ctng toolchain tigger an assembler error with guile package when compiled with -Os (same issue as for CS ARM 2014.05-29) + if 'BR2_PACKAGE_GUILE=y\n' in configlines and \ + 'BR2_OPTIMIZE_S=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines: + return False + # The ctng toolchain is affected by PR58854 + if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'armv6-ctng-linux-uclibcgnueabi.tar.xz"\n' in configlines: + return False + # The ctng toolchain is affected by PR58854 + if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'armv7-ctng-linux-gnueabihf.tar.xz"\n' in configlines: + return False + # The ctng toolchain is affected by PR60155 + if 'BR2_PACKAGE_SDL=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # The ctng toolchain is affected by PR60155 + if 'BR2_PACKAGE_LIBMPEG2=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64 + if 'BR2_PACKAGE_STRONGSWAN=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: + return False + # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64 + if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: + return False + # libffi not available on sh2a and ARMv7-M, but propagating libffi + # arch dependencies in Buildroot is really too much work, so we + # handle this here. + if 'BR2_sh2a=y\n' in configlines and \ + 'BR2_PACKAGE_LIBFFI=y\n' in configlines: + return False + if 'BR2_ARM_CPU_ARMV7M=y\n' in configlines and \ + 'BR2_PACKAGE_LIBFFI=y\n' in configlines: + return False + if 'BR2_nds32=y\n' in configlines and \ + 'BR2_PACKAGE_LIBFFI=y\n' in configlines: + return False + if 'BR2_PACKAGE_SUNXI_BOARDS=y\n' in configlines: + configlines.remove('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE=""\n') + configlines.append('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE="a10/hackberry.fex"\n') + # This MIPS uClibc toolchain fails to build the gdb package + if 'BR2_PACKAGE_GDB=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS uClibc toolchain fails to build the rt-tests package + if 'BR2_PACKAGE_RT_TESTS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS uClibc toolchain fails to build the civetweb package + if 'BR2_PACKAGE_CIVETWEB=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS ctng toolchain fails to build the python3 package + if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: + return False + # This MIPS uClibc toolchain fails to build the strace package + if 'BR2_PACKAGE_STRACE=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS uClibc toolchain fails to build the cdrkit package + if 'BR2_PACKAGE_CDRKIT=y\n' in configlines and \ + 'BR2_STATIC_LIBS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # uClibc vfork static linking issue + if 'BR2_PACKAGE_ALSA_LIB=y\n' in configlines and \ + 'BR2_STATIC_LIBS=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'i486-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # This MIPS uClibc toolchain fails to build the weston package + if 'BR2_PACKAGE_WESTON=y\n' in configlines and \ + BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: + return False + # The cs nios2 2017.02 toolchain is affected by binutils PR19405 + if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ + 'BR2_PACKAGE_BOOST=y\n' in configlines: + return False + # The cs nios2 2017.02 toolchain is affected by binutils PR19405 + if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ + 'BR2_PACKAGE_QT5BASE_GUI=y\n' in configlines: + return False + # The cs nios2 2017.02 toolchain is affected by binutils PR19405 + if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ + 'BR2_PACKAGE_FLANN=y\n' in configlines: + return False + + with open(configfile, "w+") as configf: + configf.writelines(configlines) + + return True + + +def gen_config(args): + """Generate a new random configuration + + This function generates the configuration, by choosing a random + toolchain configuration and then generating a random selection of + packages. + """ + + sysinfo = SystemInfo() + + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + + configlines = list(toolchainconfig) + + # Combine with the minimal configuration + minimalconfigfile = os.path.join(args.buildrootdir, + 'support/config-fragments/minimal.config') + with open(minimalconfigfile) as minimalf: + configlines += minimalf.readlines() + + # Allow hosts with old certificates to download over https + configlines.append("BR2_WGET=\"wget --passive-ftp -nd -t 3 --no-check-certificate\"\n") + + # Per-package folder + if randint(0, 15) == 0: + configlines.append("BR2_PER_PACKAGE_DIRECTORIES=y\n") + + # Amend the configuration with a few things. + if randint(0, 20) == 0: + configlines.append("BR2_ENABLE_DEBUG=y\n") + if randint(0, 1) == 0: + configlines.append("BR2_INIT_BUSYBOX=y\n") + elif randint(0, 15) == 0: + configlines.append("BR2_INIT_SYSTEMD=y\n") + elif randint(0, 10) == 0: + configlines.append("BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y\n") + if randint(0, 20) == 0: + configlines.append("BR2_STATIC_LIBS=y\n") + if randint(0, 20) == 0: + configlines.append("BR2_PACKAGE_PYTHON_PY_ONLY=y\n") + if randint(0, 5) == 0: + configlines.append("BR2_OPTIMIZE_2=y\n") + if randint(0, 4) == 0: + configlines.append("BR2_SYSTEM_ENABLE_NLS=y\n") + if randint(0, 4) == 0: + configlines.append("BR2_PIC_PIE=y\n") + if randint(0, 4) == 0: + configlines.append("BR2_RELRO_FULL=y\n") + elif randint(0, 4) == 0: + configlines.append("BR2_RELRO_PARTIAL=y\n") + if randint(0, 4) == 0: + configlines.append("BR2_SSP_ALL=y\n") + elif randint(0, 4) == 0: + configlines.append("BR2_SSP_REGULAR=y\n") + elif randint(0, 4) == 0: + configlines.append("BR2_SSP_STRONG=y\n") + if randint(0, 4) == 0: + configlines.append("BR2_FORTIFY_SOURCE_2=y\n") + elif randint(0, 4) == 0: + configlines.append("BR2_FORTIFY_SOURCE_1=y\n") + + # Randomly enable BR2_REPRODUCIBLE 10% of times + # also enable tar filesystem images for testing + if sysinfo.has("diffoscope") and randint(0, 10) == 0: + configlines.append("BR2_REPRODUCIBLE=y\n") + configlines.append("BR2_TARGET_ROOTFS_TAR=y\n") + + # Write out the configuration file + if not os.path.exists(args.outputdir): + os.makedirs(args.outputdir) + if args.outputdir == os.path.abspath(os.path.join(args.buildrootdir, "output")): + configfile = os.path.join(args.buildrootdir, ".config") + else: + configfile = os.path.join(args.outputdir, ".config") + with open(configfile, "w+") as configf: + configf.writelines(configlines) + + subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, + "olddefconfig"]) + + if not is_toolchain_usable(configfile, toolchainconfig): + return 2 + + # Now, generate the random selection of packages, and fixup + # things if needed. + # Safe-guard, in case we can not quickly come to a valid + # configuration: allow at most 100 (arbitrary) iterations. + bounded_loop = 100 + while True: + if bounded_loop == 0: + print("ERROR: cannot generate random configuration after 100 iterations", + file=sys.stderr) + return 1 + bounded_loop -= 1 + subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, + "KCONFIG_PROBABILITY=%d" % randint(1, 30), + "randpackageconfig"]) + + if fixup_config(sysinfo, configfile): + break + + subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, + "olddefconfig"]) + + subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, + "savedefconfig"]) + + return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, + "dependencies"]) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description="Generate a random configuration") + parser.add_argument("--outputdir", "-o", + help="Output directory (relative to current directory)", + type=str, default='output') + parser.add_argument("--buildrootdir", "-b", + help="Buildroot directory (relative to current directory)", + type=str, default='.') + parser.add_argument("--toolchains-csv", + help="Path of the toolchain configuration file", + type=str, + default="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args() + + # We need the absolute path to use with O=, because the relative + # path to the output directory here is not relative to the + # Buildroot sources, but to the current directory. + args.outputdir = os.path.abspath(args.outputdir) + + try: + ret = gen_config(args) + except Exception as e: + print(str(e), file=sys.stderr) + parser.exit(1) + parser.exit(ret) diff --git a/utils/get-developers b/utils/get-developers new file mode 100755 index 0000000000..750fc556dc --- /dev/null +++ b/utils/get-developers @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +import argparse +import getdeveloperlib +import sys +import os + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('patches', metavar='P', type=argparse.FileType('r'), nargs='*', + help='list of patches (use - to read patches from stdin)') + parser.add_argument('-a', dest='architecture', action='store', + help='find developers in charge of this architecture') + parser.add_argument('-p', dest='package', action='store', + help='find developers in charge of this package') + parser.add_argument('-f', dest='files', nargs='*', + help='find developers in charge of these files') + parser.add_argument('-c', dest='check', action='store_const', + const=True, help='list files not handled by any developer') + parser.add_argument('-e', dest='email', action='store_const', + const=True, help='only list affected developer email addresses') + return parser.parse_args() + + +def __main__(): + args = parse_args() + + # Check that only one action is given + action = 0 + if args.architecture is not None: + action += 1 + if args.package is not None: + action += 1 + if args.files: + action += 1 + if args.check: + action += 1 + if len(args.patches) != 0: + action += 1 + if action > 1: + print("Cannot do more than one action") + return + if action == 0: + print("No action specified") + return + + # getdeveloperlib expects to be executed from the toplevel buildroot + # directory, which is one level up from this script + os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) + + devs = getdeveloperlib.parse_developers() + if devs is None: + sys.exit(1) + + # Handle the check action + if args.check: + files = getdeveloperlib.check_developers(devs) + for f in files: + print(f) + + # Handle the architecture action + if args.architecture is not None: + for dev in devs: + if args.architecture in dev.architectures: + print(dev.name) + return + + # Handle the package action + if args.package is not None: + for dev in devs: + if args.package in dev.packages: + print(dev.name) + return + + # Handle the files action + if args.files is not None: + args.files = [os.path.abspath(f) for f in args.files] + for dev in devs: + for devfile in dev.files: + commonfiles = [f for f in args.files if f.startswith(devfile)] + if commonfiles: + print(dev.name) + break + + # Handle the patches action + if len(args.patches) != 0: + (files, infras) = getdeveloperlib.analyze_patches(args.patches) + matching_devs = set() + for dev in devs: + # See if we have developers matching by package name + for f in files: + if dev.hasfile(f): + matching_devs.add(dev.name) + # See if we have developers matching by package infra + for i in infras: + if i in dev.infras: + matching_devs.add(dev.name) + + if args.email: + for dev in matching_devs: + print(dev) + else: + result = "--to buildroot@buildroot.org" + for dev in matching_devs: + result += " --cc \"%s\"" % dev + + if result != "": + print("git send-email %s" % result) + + +__main__() diff --git a/utils/getdeveloperlib.py b/utils/getdeveloperlib.py new file mode 100644 index 0000000000..239ffa340b --- /dev/null +++ b/utils/getdeveloperlib.py @@ -0,0 +1,299 @@ +from __future__ import print_function +import os +import re +import glob +import subprocess +import sys +import unittest + +# +# Patch parsing functions +# + +FIND_INFRA_IN_PATCH = re.compile("^\+\$\(eval \$\((host-)?([^-]*)-package\)\)$") + + +def analyze_patch(patch): + """Parse one patch and return the list of files modified, added or + removed by the patch.""" + files = set() + infras = set() + for line in patch: + # If the patch is adding a package, find which infra it is + m = FIND_INFRA_IN_PATCH.match(line) + if m: + infras.add(m.group(2)) + if not line.startswith("+++ "): + continue + line.strip() + fname = line[line.find("/") + 1:].strip() + if fname == "dev/null": + continue + files.add(fname) + return (files, infras) + + +FIND_INFRA_IN_MK = re.compile("^\$\(eval \$\((host-)?([^-]*)-package\)\)$") + + +def fname_get_package_infra(fname): + """Checks whether the file name passed as argument is a Buildroot .mk + file describing a package, and find the infrastructure it's using.""" + if not fname.endswith(".mk"): + return None + + if not os.path.exists(fname): + return None + + with open(fname, "r") as f: + for line in f: + line = line.strip() + m = FIND_INFRA_IN_MK.match(line) + if m: + return m.group(2) + return None + + +def get_infras(files): + """Search in the list of files for .mk files, and collect the package + infrastructures used by those .mk files.""" + infras = set() + for fname in files: + infra = fname_get_package_infra(fname) + if infra: + infras.add(infra) + return infras + + +def analyze_patches(patches): + """Parse a list of patches and returns the list of files modified, + added or removed by the patches, as well as the list of package + infrastructures used by those patches (if any)""" + allfiles = set() + allinfras = set() + for patch in patches: + (files, infras) = analyze_patch(patch) + allfiles = allfiles | files + allinfras = allinfras | infras + allinfras = allinfras | get_infras(allfiles) + return (allfiles, allinfras) + + +# +# Unit-test parsing functions +# + +def get_all_test_cases(suite): + """Generate all test-cases from a given test-suite. + :return: (test.module, test.name)""" + if issubclass(type(suite), unittest.TestSuite): + for test in suite: + for res in get_all_test_cases(test): + yield res + else: + yield (suite.__module__, suite.__class__.__name__) + + +def list_unittests(path): + """Use the unittest module to retreive all test cases from a given + directory""" + loader = unittest.TestLoader() + suite = loader.discover(path) + tests = {} + for module, test in get_all_test_cases(suite): + module_path = os.path.join(path, *module.split('.')) + tests.setdefault(module_path, []).append('%s.%s' % (module, test)) + return tests + + +unittests = {} + + +# +# DEVELOPERS file parsing functions +# + +class Developer: + def __init__(self, name, files): + self.name = name + self.files = files + self.packages = parse_developer_packages(files) + self.architectures = parse_developer_architectures(files) + self.infras = parse_developer_infras(files) + self.runtime_tests = parse_developer_runtime_tests(files) + self.defconfigs = parse_developer_defconfigs(files) + + def hasfile(self, f): + f = os.path.abspath(f) + for fs in self.files: + if f.startswith(fs): + return True + return False + + def __repr__(self): + name = '\'' + self.name.split(' <')[0][:20] + '\'' + things = [] + if len(self.files): + things.append('{} files'.format(len(self.files))) + if len(self.packages): + things.append('{} pkgs'.format(len(self.packages))) + if len(self.architectures): + things.append('{} archs'.format(len(self.architectures))) + if len(self.infras): + things.append('{} infras'.format(len(self.infras))) + if len(self.runtime_tests): + things.append('{} tests'.format(len(self.runtime_tests))) + if len(self.defconfigs): + things.append('{} defconfigs'.format(len(self.defconfigs))) + if things: + return 'Developer <{} ({})>'.format(name, ', '.join(things)) + else: + return 'Developer <' + name + '>' + + +def parse_developer_packages(fnames): + """Given a list of file patterns, travel through the Buildroot source + tree to find which packages are implemented by those file + patterns, and return a list of those packages.""" + packages = set() + for fname in fnames: + for root, dirs, files in os.walk(fname): + for f in files: + path = os.path.join(root, f) + if fname_get_package_infra(path): + pkg = os.path.splitext(f)[0] + packages.add(pkg) + return packages + + +def parse_arches_from_config_in(fname): + """Given a path to an arch/Config.in.* file, parse it to get the list + of BR2_ARCH values for this architecture.""" + arches = set() + with open(fname, "r") as f: + parsing_arches = False + for line in f: + line = line.strip() + if line == "config BR2_ARCH": + parsing_arches = True + continue + if parsing_arches: + m = re.match("^\s*default \"([^\"]*)\".*", line) + if m: + arches.add(m.group(1)) + else: + parsing_arches = False + return arches + + +def parse_developer_architectures(fnames): + """Given a list of file names, find the ones starting by + 'arch/Config.in.', and use that to determine the architecture a + developer is working on.""" + arches = set() + for fname in fnames: + if not re.match("^.*/arch/Config\.in\..*$", fname): + continue + arches = arches | parse_arches_from_config_in(fname) + return arches + + +def parse_developer_infras(fnames): + infras = set() + for fname in fnames: + m = re.match("^package/pkg-([^.]*).mk$", fname) + if m: + infras.add(m.group(1)) + return infras + + +def parse_developer_defconfigs(fnames): + """Given a list of file names, returns the config names + corresponding to defconfigs.""" + return {os.path.basename(fname[:-10]) + for fname in fnames + if fname.endswith('_defconfig')} + + +def parse_developer_runtime_tests(fnames): + """Given a list of file names, returns the runtime tests + corresponding to the file.""" + all_files = [] + # List all files recursively + for fname in fnames: + if os.path.isdir(fname): + for root, _dirs, files in os.walk(fname): + all_files += [os.path.join(root, f) for f in files] + else: + all_files.append(fname) + + # Get all runtime tests + runtimes = set() + for f in all_files: + name = os.path.splitext(f)[0] + if name in unittests: + runtimes |= set(unittests[name]) + return runtimes + + +def parse_developers(basepath=None): + """Parse the DEVELOPERS file and return a list of Developer objects.""" + developers = [] + linen = 0 + if basepath is None: + basepath = os.getcwd() + global unittests + unittests = list_unittests(os.path.join(basepath, 'support/testing')) + with open(os.path.join(basepath, "DEVELOPERS"), "r") as f: + files = [] + name = None + for line in f: + line = line.strip() + if line.startswith("#"): + continue + elif line.startswith("N:"): + if name is not None or len(files) != 0: + print("Syntax error in DEVELOPERS file, line %d" % linen, + file=sys.stderr) + name = line[2:].strip() + elif line.startswith("F:"): + fname = line[2:].strip() + dev_files = glob.glob(os.path.join(basepath, fname)) + if len(dev_files) == 0: + print("WARNING: '%s' doesn't match any file" % fname, + file=sys.stderr) + files += dev_files + elif line == "": + if not name: + continue + developers.append(Developer(name, files)) + files = [] + name = None + else: + print("Syntax error in DEVELOPERS file, line %d: '%s'" % (linen, line), + file=sys.stderr) + return None + linen += 1 + # handle last developer + if name is not None: + developers.append(Developer(name, files)) + return developers + + +def check_developers(developers, basepath=None): + """Look at the list of files versioned in Buildroot, and returns the + list of files that are not handled by any developer""" + if basepath is None: + basepath = os.getcwd() + cmd = ["git", "--git-dir", os.path.join(basepath, ".git"), "ls-files"] + files = subprocess.check_output(cmd).strip().split("\n") + unhandled_files = [] + for f in files: + handled = False + for d in developers: + if d.hasfile(os.path.join(basepath, f)): + handled = True + break + if not handled: + unhandled_files.append(f) + return unhandled_files diff --git a/utils/readme.txt b/utils/readme.txt new file mode 100644 index 0000000000..3064ecec32 --- /dev/null +++ b/utils/readme.txt @@ -0,0 +1,45 @@ +This directory contains various useful scripts and tools for working +with Buildroot. You need not add this directory in your PATH to use +any of those tools, but you may do so if you want. + +brmake + a script that can be run instead of make, that prepends the date in + front of each line, redirects all of the build output to a file + ("'br.log' in the current directory), and just outputs the Buildroot + messages (those lines starting with >>>) on stdout. + Do not run this script for interactive configuration (e.g. menuconfig) + or on an unconfigured directory. The output is redirected so you will see + nothing. + +check-package + a script that checks the coding style of a package's Config.in and + .mk files, and also tests them for various types of typoes. + +genrandconfig + a script that generates a random configuration, used by the autobuilders + (http://autobuild.buildroot.org). It selects a random toolchain from + support/config-fragments/autobuild and randomly selects packages to build. + +get-developpers + a script to return the list of people interested in a specific part + of Buildroot, so they can be Cc:ed on a mail. Accepts a patch as + input, a package name or and architecture name. + +scancpan + a script to create a Buildroot package by scanning a CPAN module + description. + +scanpypi + a script to create a Buildroot package by scanning a PyPI package + description. + +size-stats-compare + a script to compare the rootfs size between two different Buildroot + configurations. This can be used to identify the size impact of + a specific option, of a set of specific options, or of an update + to a newer Buildroot version... + +test-pkg + a script that tests a specific package against a set of various + toolchains, with the goal to detect toolchain-related dependencies + (wchar, threads...) diff --git a/utils/scancpan b/utils/scancpan new file mode 100755 index 0000000000..327a58fc78 --- /dev/null +++ b/utils/scancpan @@ -0,0 +1,997 @@ +#!/usr/bin/env perl + +# This chunk of stuff was generated by App::FatPacker. To find the original +# file's code, look for the end of this BEGIN block or the string 'FATPACK' +BEGIN { +my %fatpacked; + +$fatpacked{"MetaCPAN/API/Tiny.pm"} = <<'METACPAN_API_TINY'; + package MetaCPAN::API::Tiny; + { + $MetaCPAN::API::Tiny::VERSION = '1.131730'; + } + use strict; + use warnings; + # ABSTRACT: A Tiny API client for MetaCPAN + + use Carp; + use JSON::PP 'encode_json', 'decode_json'; + use HTTP::Tiny; + + + sub new { + my ($class, @args) = @_; + + $#_ % 2 == 0 + or croak 'Arguments must be provided as name/value pairs'; + + my %params = @args; + + die 'ua_args must be an array reference' + if $params{ua_args} && ref($params{ua_args}) ne 'ARRAY'; + + my $self = +{ + base_url => $params{base_url} || 'http://api.metacpan.org/v0', + ua => $params{ua} || HTTP::Tiny->new( + $params{ua_args} + ? @{$params{ua_args}} + : (agent => 'MetaCPAN::API::Tiny/' + . ($MetaCPAN::API::VERSION || 'xx'))), + }; + + return bless($self, $class); + } + + sub _build_extra_params { + my $self = shift; + + @_ % 2 == 0 + or croak 'Incorrect number of params, must be key/value'; + + my %extra = @_; + my $ua = $self->{ua}; + + foreach my $key (keys %extra) + { + # The implementation in HTTP::Tiny uses + instead of %20, fix that + $extra{$key} = $ua->_uri_escape($extra{$key}); + $extra{$key} =~ s/\+/%20/g; + } + + my $params = join '&', map { "$_=" . $extra{$_} } sort keys %extra; + + return $params; + } + + + # /source/{author}/{release}/{path} + sub source { + my $self = shift; + my %opts = @_ ? @_ : (); + my $url = ''; + my $error = "Provide 'author' and 'release' and 'path'"; + + %opts or croak $error; + + if ( + defined ( my $author = $opts{'author'} ) && + defined ( my $release = $opts{'release'} ) && + defined ( my $path = $opts{'path'} ) + ) { + $url = "source/$author/$release/$path"; + } else { + croak $error; + } + + $url = $self->{base_url} . "/$url"; + + my $result = $self->{ua}->get($url); + $result->{'success'} + or croak "Failed to fetch '$url': " . $result->{'reason'}; + + return $result->{'content'}; + } + + + # /release/{distribution} + # /release/{author}/{release} + sub release { + my $self = shift; + my %opts = @_ ? @_ : (); + my $url = ''; + my $error = "Either provide 'distribution', or 'author' and 'release', " . + "or 'search'"; + + %opts or croak $error; + + my %extra_opts = (); + + if ( defined ( my $dist = $opts{'distribution'} ) ) { + $url = "release/$dist"; + } elsif ( + defined ( my $author = $opts{'author'} ) && + defined ( my $release = $opts{'release'} ) + ) { + $url = "release/$author/$release"; + } elsif ( defined ( my $search_opts = $opts{'search'} ) ) { + ref $search_opts && ref $search_opts eq 'HASH' + or croak $error; + + %extra_opts = %{$search_opts}; + $url = 'release/_search'; + } else { + croak $error; + } + + return $self->fetch( $url, %extra_opts ); + } + + + # /pod/{module} + # /pod/{author}/{release}/{path} + sub pod { + my $self = shift; + my %opts = @_ ? @_ : (); + my $url = ''; + my $error = "Either provide 'module' or 'author and 'release' and 'path'"; + + %opts or croak $error; + + if ( defined ( my $module = $opts{'module'} ) ) { + $url = "pod/$module"; + } elsif ( + defined ( my $author = $opts{'author'} ) && + defined ( my $release = $opts{'release'} ) && + defined ( my $path = $opts{'path'} ) + ) { + $url = "pod/$author/$release/$path"; + } else { + croak $error; + } + + # check content-type + my %extra = (); + if ( defined ( my $type = $opts{'content-type'} ) ) { + $type =~ m{^ text/ (?: html|plain|x-pod|x-markdown ) $}x + or croak 'Incorrect content-type provided'; + + $extra{headers}{'content-type'} = $type; + } + + $url = $self->{base_url}. "/$url"; + + my $result = $self->{ua}->get( $url, \%extra ); + $result->{'success'} + or croak "Failed to fetch '$url': " . $result->{'reason'}; + + return $result->{'content'}; + } + + + # /module/{module} + sub module { + my $self = shift; + my $name = shift; + + $name or croak 'Please provide a module name'; + + return $self->fetch("module/$name"); + } + + + # file() is a synonym of module + sub file { goto &module } + + + # /author/{author} + sub author { + my $self = shift; + my ( $pause_id, $url, %extra_opts ); + + if ( @_ == 1 ) { + $url = 'author/' . shift; + } elsif ( @_ == 2 ) { + my %opts = @_; + + if ( defined $opts{'pauseid'} ) { + $url = "author/" . $opts{'pauseid'}; + } elsif ( defined $opts{'search'} ) { + my $search_opts = $opts{'search'}; + + ref $search_opts && ref $search_opts eq 'HASH' + or croak "'search' key must be hashref"; + + %extra_opts = %{$search_opts}; + $url = 'author/_search'; + } else { + croak 'Unknown option given'; + } + } else { + croak 'Please provide an author PAUSEID or a "search"'; + } + + return $self->fetch( $url, %extra_opts ); + } + + + + sub fetch { + my $self = shift; + my $url = shift; + my $extra = $self->_build_extra_params(@_); + my $base = $self->{base_url}; + my $req_url = $extra ? "$base/$url?$extra" : "$base/$url"; + + my $result = $self->{ua}->get($req_url); + return $self->_decode_result( $result, $req_url ); + } + + + sub post { + my $self = shift; + my $url = shift; + my $query = shift; + my $base = $self->{base_url}; + + defined $url + or croak 'First argument of URL must be provided'; + + ref $query and ref $query eq 'HASH' + or croak 'Second argument of query hashref must be provided'; + + my $query_json = encode_json( $query ); + my $result = $self->{ua}->request( + 'POST', + "$base/$url", + { + headers => { 'Content-Type' => 'application/json' }, + content => $query_json, + } + ); + + return $self->_decode_result( $result, $url, $query_json ); + } + + sub _decode_result { + my $self = shift; + my ( $result, $url, $original ) = @_; + my $decoded_result; + + ref $result and ref $result eq 'HASH' + or croak 'First argument must be hashref'; + + defined $url + or croak 'Second argument of a URL must be provided'; + + if ( defined ( my $success = $result->{'success'} ) ) { + my $reason = $result->{'reason'} || ''; + $reason .= ( defined $original ? " (request: $original)" : '' ); + + $success or croak "Failed to fetch '$url': $reason"; + } else { + croak 'Missing success in return value'; + } + + defined ( my $content = $result->{'content'} ) + or croak 'Missing content in return value'; + + eval { $decoded_result = decode_json $content; 1 } + or do { croak "Couldn't decode '$content': $@" }; + + return $decoded_result; + } + + 1; + + __END__ + + =pod + + =head1 NAME + + MetaCPAN::API::Tiny - A Tiny API client for MetaCPAN + + =head1 VERSION + + version 1.131730 + + =head1 DESCRIPTION + + This is the Tiny version of L. It implements a compatible API + with a few notable exceptions: + + =over 4 + + =item Attributes are direct hash access + + The attributes defined using Mo(o|u)se are now accessed via the blessed hash + directly. There are no accessors defined to access this elements. + + =item Exception handling + + Instead of using Try::Tiny, raw evals are used. This could potentially cause + issues, so just be aware. + + =item Testing + + Test::Fatal was replaced with an eval implementation of exception(). + Test::TinyMocker usage is retained, but may be absorbed since it is pure perl + + =back + + =head1 CLASS_METHODS + + =head2 new + + new is the constructor for MetaCPAN::API::Tiny. In the non-tiny version of this + module, this is provided via Any::Moose built from the attributes defined. In + the tiny version, we define our own constructor. It takes the same arguments + and provides similar checks to MetaCPAN::API with regards to arguments passed. + + =head1 PUBLIC_METHODS + + =head2 source + + my $source = $mcpan->source( + author => 'DOY', + release => 'Moose-2.0201', + path => 'lib/Moose.pm', + ); + + Searches MetaCPAN for a module or a specific release and returns the plain source. + + =head2 release + + my $result = $mcpan->release( distribution => 'Moose' ); + + # or + my $result = $mcpan->release( author => 'DOY', release => 'Moose-2.0001' ); + + Searches MetaCPAN for a dist. + + You can do complex searches using 'search' parameter: + + # example lifted from MetaCPAN docs + my $result = $mcpan->release( + search => { + author => "OALDERS AND ", + filter => "status:latest", + fields => "name", + size => 1, + }, + ); + + =head2 pod + + my $result = $mcpan->pod( module => 'Moose' ); + + # or + my $result = $mcpan->pod( + author => 'DOY', + release => 'Moose-2.0201', + path => 'lib/Moose.pm', + ); + + Searches MetaCPAN for a module or a specific release and returns the POD. + + =head2 module + + my $result = $mcpan->module('MetaCPAN::API'); + + Searches MetaCPAN and returns a module's ".pm" file. + + =head2 file + + A synonym of L + + =head2 author + + my $result1 = $mcpan->author('XSAWYERX'); + my $result2 = $mcpan->author( pauseid => 'XSAWYERX' ); + + Searches MetaCPAN for a specific author. + + You can do complex searches using 'search' parameter: + + # example lifted from MetaCPAN docs + my $result = $mcpan->author( + search => { + q => 'profile.name:twitter', + size => 1, + }, + ); + + =head2 fetch + + my $result = $mcpan->fetch('/release/distribution/Moose'); + + # with parameters + my $more = $mcpan->fetch( + '/release/distribution/Moose', + param => 'value', + ); + + This is a helper method for API implementations. It fetches a path from MetaCPAN, decodes the JSON from the content variable and returns it. + + You don't really need to use it, but you can in case you want to write your own extension implementation to MetaCPAN::API. + + It accepts an additional hash as "GET" parameters. + + =head2 post + + # /release&content={"query":{"match_all":{}},"filter":{"prefix":{"archive":"Cache-Cache-1.06"}}} + my $result = $mcpan->post( + 'release', + { + query => { match_all => {} }, + filter => { prefix => { archive => 'Cache-Cache-1.06' } }, + }, + ); + + The POST equivalent of the "fetch()" method. It gets the path and JSON request. + + =head1 THANKS + + Overall the tests and code were ripped directly from MetaCPAN::API and + tiny-fied. A big thanks to Sawyer X for writing the original module. + + =head1 AUTHOR + + Nicholas R. Perez + + =head1 COPYRIGHT AND LICENSE + + This software is copyright (c) 2013 by Nicholas R. Perez . + + This is free software; you can redistribute it and/or modify it under + the same terms as the Perl 5 programming language system itself. + + =cut +METACPAN_API_TINY + +s/^ //mg for values %fatpacked; + +unshift @INC, sub { + if (my $fat = $fatpacked{$_[1]}) { + if ($] < 5.008) { + return sub { + return 0 unless length $fat; + $fat =~ s/^([^\n]*\n?)//; + $_ = $1; + return 1; + }; + } + open my $fh, '<', \$fat + or die "FatPacker error loading $_[1] (could be a perl installation issue?)"; + return $fh; + } + return +}; + +} # END OF FATPACK CODE + + +use 5.010; +use strict; +use warnings; +use Fatal qw(open close); + +use Getopt::Long; +use Pod::Usage; +use File::Basename; +use File::Path qw(make_path); +use Module::CoreList; +use HTTP::Tiny; +use Safe; +use MetaCPAN::API::Tiny; +use Digest::SHA qw(sha256_hex); +use Text::Wrap; +$Text::Wrap::columns = 62; + +# Below, 5.030 should be aligned with the version of perl actually +# bundled in Buildroot: +die <<"MSG" if $] < 5.030; +This script needs a host perl with the same major version as Buildroot target perl. + +Your current host perl is: + $^X + version $] + +You may install a local one by running: + perlbrew install perl-5.30.0 +MSG + +my ($help, $man, $quiet, $force, $recommend, $test, $host); +my $target = 1; +GetOptions( 'help|?' => \$help, + 'man' => \$man, + 'quiet|q' => \$quiet, + 'force|f' => \$force, + 'host!' => \$host, + 'target!' => \$target, + 'recommend' => \$recommend, + 'test' => \$test +) or pod2usage(-exitval => 1); +pod2usage(-exitval => 0) if $help; +pod2usage(-exitval => 0, -verbose => 2) if $man; +pod2usage(-exitval => 1) if scalar @ARGV == 0; + +my %dist; # name -> metacpan data +my %need_target; # name -> 1 if target package is needed +my %need_host; # name -> 1 if host package is needed +my %need_dlopen; # name -> 1 if requires dynamic library +my %is_xs; # name -> 1 if XS module +my %deps_build; # name -> list of host dependencies +my %deps_runtime; # name -> list of target dependencies +my %license_files; # name -> hash of license files +my %checksum; # author -> list of checksum +my $mirror = 'http://cpan.metacpan.org'; # a CPAN mirror +my $mcpan = MetaCPAN::API::Tiny->new(base_url => 'http://fastapi.metacpan.org/v1'); +my $ua = HTTP::Tiny->new(); +my $new_pkgs; + +my %white_list = ( + 'ExtUtils-Config' => 1, + 'ExtUtils-InstallPaths' => 1, + 'ExtUtils-Helpers' => 1, + 'File-ShareDir-Install' => 1, + 'Module-Build' => 1, + 'Module-Build-Tiny' => 1, +); +my @info = (); + +sub get_checksum { + my ($url) = @_; + my ($path) = $url =~ m|^[^:/?#]+://[^/?#]*([^?#]*)|; + my ($basename, $dirname) = fileparse( $path ); + unless ($checksum{$dirname}) { + my $url = $mirror . $dirname . q{CHECKSUMS}; + my $response = $ua->get($url); + $checksum{$dirname} = $response->{content}; + } + my $chksum = Safe->new->reval($checksum{$dirname}); + return $chksum->{$basename}, $basename; +} + +sub is_xs { + my ($manifest) = @_; + # This heuristic determines if a module is a native extension, by searching + # some file extension types in the MANIFEST of the distribution. + # It was inspired by http://deps.cpantesters.org/static/purity.html + return $manifest =~ m/\.(swg|xs|c|h|i)[\n\s]/; +} + +sub find_license_files { + my ($manifest) = @_; + my @license_files; + foreach (split /\n/, $manifest) { + next if m|/|; + s|\s+.*$||; + push @license_files, $_ if m/(ARTISTIC|COPYING|COPYRIGHT|LICENSE|LICENCE)/i; + } + if (scalar @license_files == 0 && $manifest =~ m/(README)[\n\s]/i) { + @license_files = ($1); + } + if (scalar @license_files == 0 && $manifest =~ m/(README\.md)[\n\s]/i) { + @license_files = ($1); + } + return @license_files; +} + +sub want_test { + my ($distname) = @_; + return 1 if $need_dlopen{$distname} && scalar @{$deps_runtime{$distname}} > 0; +} + +sub get_dependencies { + my ($distname) = @_; + my %dep = map { $_ => 1 } @{$deps_runtime{$distname}}; + for my $direct (@{$deps_runtime{$distname}}) { + for (get_dependencies( $direct )) { + $dep{$_} = 1; + } + } + return keys %dep; +} + +sub get_indirect_dependencies { + my ($distname) = @_; + my %indirect; + my %direct = map { $_ => 1 } @{$deps_runtime{$distname}}; + for my $dep (get_dependencies( $distname )) { + $indirect{$dep} = 1 unless exists $direct{$dep}; + } + return keys %indirect; +} + +sub fetch { + my ($name, $need_target, $need_host, $top) = @_; + $need_target{$name} = $need_target if $need_target; + $need_host{$name} = $need_host if $need_host; + unless ($dist{$name} && !$top) { + say qq{fetch ${name}} unless $quiet; + my $result = $mcpan->release( distribution => $name ); + my $main_module = $result->{main_module}; + push @info, qq{[$name] $main_module is a core module} + if $top && Module::CoreList::is_core( $main_module, undef, $] ); + $dist{$name} = $result; + $license_files{$name} = {}; + eval { + my $author = $result->{author}; + my $release = $name . q{-} . $result->{version}; + my $manifest = $mcpan->source( author => $author, release => $release, path => 'MANIFEST' ); + $need_dlopen{$name} = $is_xs{$name} = is_xs( $manifest ); + foreach my $fname (find_license_files( $manifest )) { + my $license = $mcpan->source( author => $author, release => $release, path => $fname ); + $license_files{$name}->{$fname} = sha256_hex( $license ); + } + }; + if ($@) { + warn $@; + } + my %build = (); + my %runtime = (); + my %optional = (); + foreach my $dep (@{$result->{dependency}}) { + my $modname = ${$dep}{module}; + next if $modname eq q{perl}; + next if $modname =~ m|^Alien|; + next if $modname =~ m|^Win32|; + next if !($test && $top) && $modname =~ m|^Test|; + next if Module::CoreList::is_core( $modname, undef, $] ); + # we could use the host Module::CoreList data, because host perl and + # target perl have the same major version + next if ${$dep}{phase} eq q{develop}; + next if ${$dep}{phase} eq q{x_Dist_Zilla}; + next if !($test && $top) && ${$dep}{phase} eq q{test}; + my $distname = $mcpan->module( $modname )->{distribution}; + if (${$dep}{phase} eq q{runtime}) { + if (${$dep}{relationship} eq q{requires}) { + $runtime{$distname} = 1; + } + else { + $optional{$distname} = 1 if $recommend && $top; + } + } + else { # configure, build + $build{$distname} = 1; + push @info, qq{[$name] suspicious dependency on $distname} + unless exists $white_list{$distname}; + } + } + $deps_build{$name} = [keys %build]; + $deps_runtime{$name} = [keys %runtime]; + foreach my $distname (@{$deps_build{$name}}) { + fetch( $distname, 0, 1 ); + } + foreach my $distname (@{$deps_runtime{$name}}) { + fetch( $distname, $need_target, $need_host ); + $need_dlopen{$name} ||= $need_dlopen{$distname}; + } + foreach my $distname (keys %optional) { + fetch( $distname, $need_target, $need_host ); + } + } + return; +} + +foreach my $distname (@ARGV) { + # Command-line's distributions + fetch( $distname, !!$target, !!$host, 1 ); +} +say scalar keys %dist, q{ packages fetched.} unless $quiet; + +# Buildroot package name: lowercase +sub fsname { + my $name = shift; + $name =~ s|_|-|g; + return q{perl-} . lc $name; +} + +# Buildroot variable name: uppercase +sub brname { + my $name = shift; + $name =~ s|-|_|g; + return uc $name; +} + +# Buildroot requires license name as in http://spdx.org/licenses/ +sub brlicense { + my $license = shift; + $license =~ s|apache_1_1|Apache-1.1|; + $license =~ s|apache_2_0|Apache-2.0|; + $license =~ s|artistic_2|Artistic-2.0|; + $license =~ s|artistic|Artistic-1.0|; + $license =~ s|lgpl_2_1|LGPL-2.1|; + $license =~ s|lgpl_3_0|LGPL-3.0|; + $license =~ s|gpl_2|GPL-2.0|; + $license =~ s|gpl_3|GPL-3.0|; + $license =~ s|mit|MIT|; + $license =~ s|mozilla_1_1|Mozilla-1.1|; + $license =~ s|openssl|OpenSSL|; + $license =~ s|perl_5|Artistic or GPL-1.0+|; + return $license; +} + +while (my ($distname, $dist) = each %dist) { + my $fsname = fsname( $distname ); + my $dirname = q{package/} . $fsname; + my $cfgname = $dirname . q{/Config.in}; + my $mkname = $dirname . q{/} . $fsname . q{.mk}; + my $hashname = $dirname . q{/} . $fsname . q{.hash}; + my $brname = brname( $fsname ); + my $testdir = q{support/testing/tests/package}; + my $testname = $testdir . q{/test_} . lc $brname . q{.py}; + unless (-d $dirname) { + make_path $dirname; + $new_pkgs = 1; + } + if ($need_target{$distname} && ($force || !-f $cfgname)) { + $dist->{abstract} =~ s|\s+$||; + $dist->{abstract} .= q{.} unless $dist->{abstract} =~ m|\.$|; + my $abstract = wrap( q{}, qq{\t }, $dist->{abstract} ); + my $homepage = $dist->{resources}->{homepage} || qq{https://metacpan.org/release/${distname}}; + say qq{write ${cfgname}} unless $quiet; + open my $fh, q{>}, $cfgname; + say {$fh} qq{config BR2_PACKAGE_${brname}}; + say {$fh} qq{\tbool "${fsname}"}; + say {$fh} qq{\tdepends on !BR2_STATIC_LIBS} if $need_dlopen{$distname}; + foreach my $dep (sort @{$deps_runtime{$distname}}) { + my $brdep = brname( fsname( $dep ) ); + say {$fh} qq{\tselect BR2_PACKAGE_${brdep} # runtime}; + } + say {$fh} qq{\thelp}; + say {$fh} qq{\t ${abstract}\n} if $abstract; + say {$fh} qq{\t ${homepage}}; + if ($need_dlopen{$distname}) { + say {$fh} qq{\ncomment "${fsname} needs a toolchain w/ dynamic library"}; + say {$fh} qq{\tdepends on BR2_STATIC_LIBS}; + } + close $fh; + } + if ($force || !-f $mkname) { + my $version = $dist->{version}; + my ($path) = $dist->{download_url} =~ m|^[^:/?#]+://[^/?#]*([^?#]*)|; + # this URL contains only the scheme, auth and path parts (but no query and fragment parts) + # the scheme is not used, because the job is done by the BR download infrastructure + # the auth part is not used, because we use $(BR2_CPAN_MIRROR) + my ($filename, $directories, $suffix) = fileparse( $path, q{tar.gz}, q{tgz} ); + $directories =~ s|/$||; + my @dependencies = map( { q{host-} . fsname( $_ ); } sort @{$deps_build{$distname}} ); + my $dependencies = join qq{ \\\n\t}, @dependencies; + $dependencies = qq{\\\n\t} . $dependencies if scalar @dependencies > 1; + my @host_dependencies = map { q{host-} . fsname( $_ ); } sort( @{$deps_build{$distname}}, + @{$deps_runtime{$distname}} ); + my $host_dependencies = join qq{ \\\n\t}, @host_dependencies; + $host_dependencies = qq{\\\n\t} . $host_dependencies if scalar @host_dependencies > 1; + my $license = brlicense( ref $dist->{license} eq 'ARRAY' + ? join q{ or }, @{$dist->{license}} + : $dist->{license} ); + my $license_files = join q{ }, sort keys %{$license_files{$distname}}; + if ($license_files && (!$license || $license eq q{unknown})) { + push @info, qq{[$distname] undefined LICENSE, see $license_files}; + $license = q{???}; + } + say qq{write ${mkname}} unless $quiet; + open my $fh, q{>}, $mkname; + say {$fh} qq{################################################################################}; + say {$fh} qq{#}; + say {$fh} qq{# ${fsname}}; + say {$fh} qq{#}; + say {$fh} qq{################################################################################}; + say {$fh} qq{}; + say {$fh} qq{${brname}_VERSION = ${version}}; + say {$fh} qq{${brname}_SOURCE = ${distname}-\$(${brname}_VERSION).${suffix}}; + say {$fh} qq{${brname}_SITE = \$(BR2_CPAN_MIRROR)${directories}}; + say {$fh} qq{${brname}_DEPENDENCIES = ${dependencies}} if $need_target{$distname} && $dependencies; + say {$fh} qq{HOST_${brname}_DEPENDENCIES = ${host_dependencies}} if $need_host{$distname} && $host_dependencies; + say {$fh} qq{${brname}_LICENSE = ${license}} if $license; + say {$fh} qq{${brname}_LICENSE_FILES = ${license_files}} if $license_files; + say {$fh} qq{${brname}_DISTNAME = ${distname}}; + say {$fh} qq{}; + say {$fh} qq{\$(eval \$(perl-package))} if $need_target{$distname}; + say {$fh} qq{\$(eval \$(host-perl-package))} if $need_host{$distname}; + close $fh; + } + if ($force || !-f $hashname) { + my ($checksum, $filename) = get_checksum($dist->{download_url}); + my $md5 = $checksum->{md5}; + my $sha256 = $checksum->{sha256}; + say qq{write ${hashname}} unless $quiet; + open my $fh, q{>}, $hashname; + say {$fh} qq{# retrieved by scancpan from ${mirror}/}; + say {$fh} qq{md5 ${md5} ${filename}}; + say {$fh} qq{sha256 ${sha256} ${filename}}; + my %license_files = %{$license_files{$distname}}; + if (scalar keys %license_files) { + say {$fh} q{}; + say {$fh} qq{# computed by scancpan}; + foreach my $license (sort keys %license_files) { + my $digest = $license_files{$license}; + say {$fh} qq{sha256 ${digest} ${license}}; + } + } + close $fh; + } + if (want_test( $distname ) && ($force || !-f $testname)) { + my $classname = $distname; + $classname =~ s|-||g; + my $modname = $distname; + $modname =~ s|-|::|g; + my $mark = $is_xs{$distname} ? q{ XS} : q{}; + my @indirect = (get_indirect_dependencies( $distname )); + say qq{write ${testname}} unless $quiet; + make_path $testdir unless -d $testdir; + open my $fh, q{>}, $testname; + say {$fh} qq{from tests.package.test_perl import TestPerlBase}; + say {$fh} qq{}; + say {$fh} qq{}; + say {$fh} qq{class TestPerl${classname}(TestPerlBase):}; + say {$fh} qq{ """}; + say {$fh} qq{ package:}; + say {$fh} qq{ ${distname}${mark}}; + say {$fh} qq{ direct dependencies:}; + foreach my $dep (sort @{$deps_runtime{$distname}}) { + $mark = $is_xs{$dep} ? q{ XS} : q{}; + say {$fh} qq{ ${dep}${mark}}; + } + if (scalar @indirect > 0) { + say {$fh} qq{ indirect dependencies:}; + foreach my $dep (sort @indirect) { + $mark = $is_xs{$dep} ? q{ XS} : q{}; + say {$fh} qq{ ${dep}${mark}}; + } + } + say {$fh} qq{ """}; + say {$fh} qq{}; + say {$fh} qq{ config = TestPerlBase.config + \\}; + say {$fh} qq{ """}; + say {$fh} qq{ BR2_PACKAGE_PERL=y}; + say {$fh} qq{ BR2_PACKAGE_${brname}=y}; + say {$fh} qq{ """}; + say {$fh} qq{}; + say {$fh} qq{ def test_run(self):}; + say {$fh} qq{ self.login()}; + foreach my $dep (sort grep { $is_xs{$_} } @indirect) { + $dep =~ s|-|::|g; + say {$fh} qq{ self.module_test("${dep}")}; + } + foreach my $dep (sort grep { $is_xs{$_} } @{$deps_runtime{$distname}}) { + $dep =~ s|-|::|g; + say {$fh} qq{ self.module_test("${dep}")}; + } + say {$fh} qq{ self.module_test("${modname}")}; + close $fh; + } +} + +if ($new_pkgs) { + my %pkg; + my $cfgname = q{package/Config.in}; + if (-f $cfgname) { + open my $fh, q{<}, $cfgname; + while (<$fh>) { + chomp; + $pkg{$_} = 1 if m|package/perl-|; + } + close $fh; + } + + foreach my $distname (keys %need_target) { + my $fsname = fsname( $distname ); + $pkg{qq{\tsource "package/${fsname}/Config.in"}} = 1; + } + + say qq{${cfgname} must contain the following lines:}; + say join qq{\n}, sort keys %pkg; +} + +say join qq{\n}, @info; + +__END__ + +=head1 NAME + +utils/scancpan Try-Tiny Moo + +=head1 SYNOPSIS + +utils/scancpan [options] [distname ...] + + Options: + -help + -man + -quiet + -force + -target/-notarget + -host/-nohost + -recommend + -test + +=head1 OPTIONS + +=over 8 + +=item B<-help> + +Prints a brief help message and exits. + +=item B<-man> + +Prints the manual page and exits. + +=item B<-quiet> + +Executes without output + +=item B<-force> + +Forces the overwriting of existing files. + +=item B<-target/-notarget> + +Switches package generation for the target variant (the default is C<-target>). + +=item B<-host/-nohost> + +Switches package generation for the host variant (the default is C<-nohost>). + +=item B<-recommend> + +Adds I dependencies. + +=item B<-test> + +Adds dependencies for test. + +=back + +=head1 DESCRIPTION + +This script creates templates of the Buildroot package files for all the +Perl/CPAN distributions required by the specified distnames. The +dependencies and metadata are fetched from https://metacpan.org/. + +After running this script, it is necessary to check the generated files. +For distributions that link against a target library, you have to add the +buildroot package name for that library to the DEPENDENCIES variable. + +See the Buildroot documentation for details on the usage of the Perl +infrastructure. + +The major version of the host perl must be aligned on the target one, +in order to work with the right CoreList data. + +=head1 LICENSE + +Copyright (C) 2013-2019 by Francois Perrad + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +This script is a part of Buildroot. + +This script requires the module C (version 1.131730) +which was included at the beginning of this file by the tool C. + +See L. + +See L. + +These both libraries are free software and may be distributed under the same +terms as perl itself. + +And perl may be distributed under the terms of Artistic v1 or GPL v1 license. + +=cut diff --git a/utils/scanpypi b/utils/scanpypi new file mode 100755 index 0000000000..212dbea85e --- /dev/null +++ b/utils/scanpypi @@ -0,0 +1,754 @@ +#!/usr/bin/env python +""" + +Utility for building Buildroot packages for existing PyPI packages + +Any package built by scanpypi should be manually checked for +errors. +""" +from __future__ import print_function +from __future__ import absolute_import +import argparse +import json +import sys +import os +import shutil +import tarfile +import zipfile +import errno +import hashlib +import re +import textwrap +import tempfile +import imp +from functools import wraps +import six.moves.urllib.request +import six.moves.urllib.error +import six.moves.urllib.parse +from six.moves import map +from six.moves import zip +from six.moves import input +if six.PY2: + import StringIO +else: + import io + +BUF_SIZE = 65536 + +try: + import spdx_lookup as liclookup +except ImportError: + # spdx_lookup is not installed + print('spdx_lookup module is not installed. This can lead to an ' + 'inaccurate licence detection. Please install it via\n' + 'pip install spdx_lookup') + liclookup = None + + +def setup_decorator(func, method): + """ + Decorator for distutils.core.setup and setuptools.setup. + Puts the arguments with which setup is called as a dict + Add key 'method' which should be either 'setuptools' or 'distutils'. + + Keyword arguments: + func -- either setuptools.setup or distutils.core.setup + method -- either 'setuptools' or 'distutils' + """ + + @wraps(func) + def closure(*args, **kwargs): + # Any python packages calls its setup function to be installed. + # Argument 'name' of this setup function is the package's name + BuildrootPackage.setup_args[kwargs['name']] = kwargs + BuildrootPackage.setup_args[kwargs['name']]['method'] = method + return closure + +# monkey patch +import setuptools # noqa E402 +setuptools.setup = setup_decorator(setuptools.setup, 'setuptools') +import distutils # noqa E402 +distutils.core.setup = setup_decorator(setuptools.setup, 'distutils') + + +def find_file_upper_case(filenames, path='./'): + """ + List generator: + Recursively find files that matches one of the specified filenames. + Returns a relative path starting with path argument. + + Keyword arguments: + filenames -- List of filenames to be found + path -- Path to the directory to search + """ + for root, dirs, files in os.walk(path): + for file in files: + if file.upper() in filenames: + yield (os.path.join(root, file)) + + +def pkg_buildroot_name(pkg_name): + """ + Returns the Buildroot package name for the PyPI package pkg_name. + Remove all non alphanumeric characters except - + Also lowers the name and adds 'python-' suffix + + Keyword arguments: + pkg_name -- String to rename + """ + name = re.sub('[^\w-]', '', pkg_name.lower()) + name = name.replace('_', '-') + prefix = 'python-' + pattern = re.compile('^(?!' + prefix + ')(.+?)$') + name = pattern.sub(r'python-\1', name) + return name + + +class DownloadFailed(Exception): + pass + + +class BuildrootPackage(): + """This class's methods are not meant to be used individually please + use them in the correct order: + + __init__ + + download_package + + extract_package + + load_module + + get_requirements + + create_package_mk + + create_hash_file + + create_config_in + + """ + setup_args = {} + + def __init__(self, real_name, pkg_folder): + self.real_name = real_name + self.buildroot_name = pkg_buildroot_name(self.real_name) + self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name) + self.mk_name = self.buildroot_name.upper().replace('-', '_') + self.as_string = None + self.md5_sum = None + self.metadata = None + self.metadata_name = None + self.metadata_url = None + self.pkg_req = None + self.setup_metadata = None + self.tmp_extract = None + self.used_url = None + self.filename = None + self.url = None + self.version = None + self.license_files = [] + + def fetch_package_info(self): + """ + Fetch a package's metadata from the python package index + """ + self.metadata_url = 'https://pypi.org/pypi/{pkg}/json'.format( + pkg=self.real_name) + try: + pkg_json = six.moves.urllib.request.urlopen(self.metadata_url).read().decode() + except six.moves.urllib.error.HTTPError as error: + print('ERROR:', error.getcode(), error.msg, file=sys.stderr) + print('ERROR: Could not find package {pkg}.\n' + 'Check syntax inside the python package index:\n' + 'https://pypi.python.org/pypi/ ' + .format(pkg=self.real_name)) + raise + except six.moves.urllib.error.URLError: + print('ERROR: Could not find package {pkg}.\n' + 'Check syntax inside the python package index:\n' + 'https://pypi.python.org/pypi/ ' + .format(pkg=self.real_name)) + raise + self.metadata = json.loads(pkg_json) + self.version = self.metadata['info']['version'] + self.metadata_name = self.metadata['info']['name'] + + def download_package(self): + """ + Download a package using metadata from pypi + """ + download = None + try: + self.metadata['urls'][0]['filename'] + except IndexError: + print( + 'Non-conventional package, ', + 'please check carefully after creation') + self.metadata['urls'] = [{ + 'packagetype': 'sdist', + 'url': self.metadata['info']['download_url'], + 'digests': None}] + # In this case, we can't get the name of the downloaded file + # from the pypi api, so we need to find it, this should work + urlpath = six.moves.urllib.parse.urlparse( + self.metadata['info']['download_url']).path + # urlparse().path give something like + # /path/to/file-version.tar.gz + # We use basename to remove /path/to + self.metadata['urls'][0]['filename'] = os.path.basename(urlpath) + for download_url in self.metadata['urls']: + if 'bdist' in download_url['packagetype']: + continue + try: + print('Downloading package {pkg} from {url}...'.format( + pkg=self.real_name, url=download_url['url'])) + download = six.moves.urllib.request.urlopen(download_url['url']) + except six.moves.urllib.error.HTTPError as http_error: + download = http_error + else: + self.used_url = download_url + self.as_string = download.read() + if not download_url['digests']['md5']: + break + self.md5_sum = hashlib.md5(self.as_string).hexdigest() + if self.md5_sum == download_url['digests']['md5']: + break + + if download is None: + raise DownloadFailed('Failed to download package {pkg}: ' + 'No source archive available' + .format(pkg=self.real_name)) + elif download.__class__ == six.moves.urllib.error.HTTPError: + raise download + + self.filename = self.used_url['filename'] + self.url = self.used_url['url'] + + def check_archive(self, members): + """ + Check archive content before extracting + + Keyword arguments: + members -- list of archive members + """ + # Protect against https://github.com/snyk/zip-slip-vulnerability + # Older python versions do not validate that the extracted files are + # inside the target directory. Detect and error out on evil paths + evil = [e for e in members if os.path.relpath(e).startswith(('/', '..'))] + if evil: + print('ERROR: Refusing to extract {} with suspicious members {}'.format( + self.filename, evil)) + sys.exit(1) + + def extract_package(self, tmp_path): + """ + Extract the package contents into a directrory + + Keyword arguments: + tmp_path -- directory where you want the package to be extracted + """ + if six.PY2: + as_file = StringIO.StringIO(self.as_string) + else: + as_file = io.BytesIO(self.as_string) + if self.filename[-3:] == 'zip': + with zipfile.ZipFile(as_file) as as_zipfile: + tmp_pkg = os.path.join(tmp_path, self.buildroot_name) + try: + os.makedirs(tmp_pkg) + except OSError as exception: + if exception.errno != errno.EEXIST: + print("ERROR: ", exception.strerror, file=sys.stderr) + return + print('WARNING:', exception.strerror, file=sys.stderr) + print('Removing {pkg}...'.format(pkg=tmp_pkg)) + shutil.rmtree(tmp_pkg) + os.makedirs(tmp_pkg) + self.check_archive(as_zipfile.namelist()) + as_zipfile.extractall(tmp_pkg) + pkg_filename = self.filename.split(".zip")[0] + else: + with tarfile.open(fileobj=as_file) as as_tarfile: + tmp_pkg = os.path.join(tmp_path, self.buildroot_name) + try: + os.makedirs(tmp_pkg) + except OSError as exception: + if exception.errno != errno.EEXIST: + print("ERROR: ", exception.strerror, file=sys.stderr) + return + print('WARNING:', exception.strerror, file=sys.stderr) + print('Removing {pkg}...'.format(pkg=tmp_pkg)) + shutil.rmtree(tmp_pkg) + os.makedirs(tmp_pkg) + self.check_archive(as_tarfile.getnames()) + as_tarfile.extractall(tmp_pkg) + pkg_filename = self.filename.split(".tar")[0] + + tmp_extract = '{folder}/{name}' + self.tmp_extract = tmp_extract.format( + folder=tmp_pkg, + name=pkg_filename) + + def load_setup(self): + """ + Loads the corresponding setup and store its metadata + """ + current_dir = os.getcwd() + os.chdir(self.tmp_extract) + sys.path.append(self.tmp_extract) + s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract]) + setup = imp.load_module('setup', s_file, s_path, s_desc) + if self.metadata_name in self.setup_args: + pass + elif self.metadata_name.replace('_', '-') in self.setup_args: + self.metadata_name = self.metadata_name.replace('_', '-') + elif self.metadata_name.replace('-', '_') in self.setup_args: + self.metadata_name = self.metadata_name.replace('-', '_') + try: + self.setup_metadata = self.setup_args[self.metadata_name] + except KeyError: + # This means setup was not called which most likely mean that it is + # called through the if __name__ == '__main__' directive. + # In this case, we can only pray that it is called through a + # function called main() in setup.py. + setup.main() # Will raise AttributeError if not found + self.setup_metadata = self.setup_args[self.metadata_name] + # Here we must remove the module the hard way. + # We must do this because of a very specific case: if a package calls + # setup from the __main__ but does not come with a 'main()' function, + # for some reason setup.main() will successfully call the main + # function of a previous package... + sys.modules.pop('setup', None) + del setup + os.chdir(current_dir) + sys.path.remove(self.tmp_extract) + + def get_requirements(self, pkg_folder): + """ + Retrieve dependencies from the metadata found in the setup.py script of + a pypi package. + + Keyword Arguments: + pkg_folder -- location of the already created packages + """ + if 'install_requires' not in self.setup_metadata: + self.pkg_req = None + return set() + self.pkg_req = self.setup_metadata['install_requires'] + self.pkg_req = [re.sub('([-.\w]+).*', r'\1', req) + for req in self.pkg_req] + + # get rid of commented lines and also strip the package strings + self.pkg_req = [item.strip() for item in self.pkg_req + if len(item) > 0 and item[0] != '#'] + + req_not_found = self.pkg_req + self.pkg_req = list(map(pkg_buildroot_name, self.pkg_req)) + pkg_tuples = list(zip(req_not_found, self.pkg_req)) + # pkg_tuples is a list of tuples that looks like + # ('werkzeug','python-werkzeug') because I need both when checking if + # dependencies already exist or are already in the download list + req_not_found = set( + pkg[0] for pkg in pkg_tuples + if not os.path.isdir(pkg[1]) + ) + return req_not_found + + def __create_mk_header(self): + """ + Create the header of the .mk file + """ + header = ['#' * 80 + '\n'] + header.append('#\n') + header.append('# {name}\n'.format(name=self.buildroot_name)) + header.append('#\n') + header.append('#' * 80 + '\n') + header.append('\n') + return header + + def __create_mk_download_info(self): + """ + Create the lines refering to the download information of the + .mk file + """ + lines = [] + version_line = '{name}_VERSION = {version}\n'.format( + name=self.mk_name, + version=self.version) + lines.append(version_line) + + if self.buildroot_name != self.real_name: + targz = self.filename.replace( + self.version, + '$({name}_VERSION)'.format(name=self.mk_name)) + targz_line = '{name}_SOURCE = {filename}\n'.format( + name=self.mk_name, + filename=targz) + lines.append(targz_line) + + if self.filename not in self.url: + # Sometimes the filename is in the url, sometimes it's not + site_url = self.url + else: + site_url = self.url[:self.url.find(self.filename)] + site_line = '{name}_SITE = {url}'.format(name=self.mk_name, + url=site_url) + site_line = site_line.rstrip('/') + '\n' + lines.append(site_line) + return lines + + def __create_mk_setup(self): + """ + Create the line refering to the setup method of the package of the + .mk file + + There are two things you can use to make an installer + for a python package: distutils or setuptools + distutils comes with python but does not support dependencies. + distutils is mostly still there for backward support. + setuptools is what smart people use, + but it is not shipped with python :( + """ + lines = [] + setup_type_line = '{name}_SETUP_TYPE = {method}\n'.format( + name=self.mk_name, + method=self.setup_metadata['method']) + lines.append(setup_type_line) + return lines + + def __get_license_names(self, license_files): + """ + Try to determine the related license name. + + There are two possibilities. Either the script tries to + get license name from package's metadata or, if spdx_lookup + package is available, the script compares license files with + SPDX database. + """ + license_line = '' + if liclookup is None: + license_dict = { + 'Apache Software License': 'Apache-2.0', + 'BSD License': 'FIXME: please specify the exact BSD version', + 'European Union Public Licence 1.0': 'EUPL-1.0', + 'European Union Public Licence 1.1': 'EUPL-1.1', + "GNU General Public License": "GPL", + "GNU General Public License v2": "GPL-2.0", + "GNU General Public License v2 or later": "GPL-2.0+", + "GNU General Public License v3": "GPL-3.0", + "GNU General Public License v3 or later": "GPL-3.0+", + "GNU Lesser General Public License v2": "LGPL-2.1", + "GNU Lesser General Public License v2 or later": "LGPL-2.1+", + "GNU Lesser General Public License v3": "LGPL-3.0", + "GNU Lesser General Public License v3 or later": "LGPL-3.0+", + "GNU Library or Lesser General Public License": "LGPL-2.0", + "ISC License": "ISC", + "MIT License": "MIT", + "Mozilla Public License 1.0": "MPL-1.0", + "Mozilla Public License 1.1": "MPL-1.1", + "Mozilla Public License 2.0": "MPL-2.0", + "Zope Public License": "ZPL" + } + regexp = re.compile('^License :* *.* *:+ (.*)( \(.*\))?$') + classifiers_licenses = [regexp.sub(r"\1", lic) + for lic in self.metadata['info']['classifiers'] + if regexp.match(lic)] + licenses = [license_dict[x] if x in license_dict else x for x in classifiers_licenses] + if not len(licenses): + print('WARNING: License has been set to "{license}". It is most' + ' likely wrong, please change it if need be'.format( + license=', '.join(licenses))) + licenses = [self.metadata['info']['license']] + licenses = set(licenses) + license_line = '{name}_LICENSE = {license}\n'.format( + name=self.mk_name, + license=', '.join(licenses)) + else: + license_names = [] + for license_file in license_files: + with open(license_file) as lic_file: + match = liclookup.match(lic_file.read()) + if match is not None and match.confidence >= 90.0: + license_names.append(match.license.id) + else: + license_names.append("FIXME: license id couldn't be detected") + license_names = set(license_names) + + if len(license_names) > 0: + license_line = ('{name}_LICENSE =' + ' {names}\n'.format( + name=self.mk_name, + names=', '.join(license_names))) + + return license_line + + def __create_mk_license(self): + """ + Create the lines referring to the package's license informations of the + .mk file + + The license's files are found by searching the package (case insensitive) + for files named license, license.txt etc. If more than one license file + is found, the user is asked to select which ones he wants to use. + """ + lines = [] + + filenames = ['LICENCE', 'LICENSE', 'LICENSE.RST', 'LICENSE.TXT', + 'COPYING', 'COPYING.TXT'] + self.license_files = list(find_file_upper_case(filenames, self.tmp_extract)) + + lines.append(self.__get_license_names(self.license_files)) + + license_files = [license.replace(self.tmp_extract, '')[1:] + for license in self.license_files] + if len(license_files) > 0: + if len(license_files) > 1: + print('More than one file found for license:', + ', '.join(license_files)) + license_files = [filename + for index, filename in enumerate(license_files)] + license_file_line = ('{name}_LICENSE_FILES =' + ' {files}\n'.format( + name=self.mk_name, + files=' '.join(license_files))) + lines.append(license_file_line) + else: + print('WARNING: No license file found,' + ' please specify it manually afterwards') + license_file_line = '# No license file found\n' + + return lines + + def __create_mk_requirements(self): + """ + Create the lines referring to the dependencies of the of the + .mk file + + Keyword Arguments: + pkg_name -- name of the package + pkg_req -- dependencies of the package + """ + lines = [] + dependencies_line = ('{name}_DEPENDENCIES =' + ' {reqs}\n'.format( + name=self.mk_name, + reqs=' '.join(self.pkg_req))) + lines.append(dependencies_line) + return lines + + def create_package_mk(self): + """ + Create the lines corresponding to the .mk file + """ + pkg_mk = '{name}.mk'.format(name=self.buildroot_name) + path_to_mk = os.path.join(self.pkg_dir, pkg_mk) + print('Creating {file}...'.format(file=path_to_mk)) + lines = self.__create_mk_header() + lines += self.__create_mk_download_info() + lines += self.__create_mk_setup() + lines += self.__create_mk_license() + + lines.append('\n') + lines.append('$(eval $(python-package))') + lines.append('\n') + with open(path_to_mk, 'w') as mk_file: + mk_file.writelines(lines) + + def create_hash_file(self): + """ + Create the lines corresponding to the .hash files + """ + pkg_hash = '{name}.hash'.format(name=self.buildroot_name) + path_to_hash = os.path.join(self.pkg_dir, pkg_hash) + print('Creating {filename}...'.format(filename=path_to_hash)) + lines = [] + if self.used_url['digests']['md5'] and self.used_url['digests']['sha256']: + hash_header = '# md5, sha256 from {url}\n'.format( + url=self.metadata_url) + lines.append(hash_header) + hash_line = '{method} {digest} {filename}\n'.format( + method='md5', + digest=self.used_url['digests']['md5'], + filename=self.filename) + lines.append(hash_line) + hash_line = '{method} {digest} {filename}\n'.format( + method='sha256', + digest=self.used_url['digests']['sha256'], + filename=self.filename) + lines.append(hash_line) + + if self.license_files: + lines.append('# Locally computed sha256 checksums\n') + for license_file in self.license_files: + sha256 = hashlib.sha256() + with open(license_file, 'rb') as lic_f: + while True: + data = lic_f.read(BUF_SIZE) + if not data: + break + sha256.update(data) + hash_line = '{method} {digest} {filename}\n'.format( + method='sha256', + digest=sha256.hexdigest(), + filename=license_file.replace(self.tmp_extract, '')[1:]) + lines.append(hash_line) + + with open(path_to_hash, 'w') as hash_file: + hash_file.writelines(lines) + + def create_config_in(self): + """ + Creates the Config.in file of a package + """ + path_to_config = os.path.join(self.pkg_dir, 'Config.in') + print('Creating {file}...'.format(file=path_to_config)) + lines = [] + config_line = 'config BR2_PACKAGE_{name}\n'.format( + name=self.mk_name) + lines.append(config_line) + + bool_line = '\tbool "{name}"\n'.format(name=self.buildroot_name) + lines.append(bool_line) + if self.pkg_req: + self.pkg_req.sort() + for dep in self.pkg_req: + dep_line = '\tselect BR2_PACKAGE_{req} # runtime\n'.format( + req=dep.upper().replace('-', '_')) + lines.append(dep_line) + + lines.append('\thelp\n') + + help_lines = textwrap.wrap(self.metadata['info']['summary'], 62, + initial_indent='\t ', + subsequent_indent='\t ') + + # make sure a help text is terminated with a full stop + if help_lines[-1][-1] != '.': + help_lines[-1] += '.' + + # \t + two spaces is 3 char long + help_lines.append('') + help_lines.append('\t ' + self.metadata['info']['home_page']) + help_lines = [x + '\n' for x in help_lines] + lines += help_lines + + with open(path_to_config, 'w') as config_file: + config_file.writelines(lines) + + +def main(): + # Building the parser + parser = argparse.ArgumentParser( + description="Creates buildroot packages from the metadata of " + "an existing PyPI packages and include it " + "in menuconfig") + parser.add_argument("packages", + help="list of packages to be created", + nargs='+') + parser.add_argument("-o", "--output", + help=""" + Output directory for packages. + Default is ./package + """, + default='./package') + + args = parser.parse_args() + packages = list(set(args.packages)) + + # tmp_path is where we'll extract the files later + tmp_prefix = 'scanpypi-' + pkg_folder = args.output + tmp_path = tempfile.mkdtemp(prefix=tmp_prefix) + try: + for real_pkg_name in packages: + package = BuildrootPackage(real_pkg_name, pkg_folder) + print('buildroot package name for {}:'.format(package.real_name), + package.buildroot_name) + # First we download the package + # Most of the info we need can only be found inside the package + print('Package:', package.buildroot_name) + print('Fetching package', package.real_name) + try: + package.fetch_package_info() + except (six.moves.urllib.error.URLError, six.moves.urllib.error.HTTPError): + continue + if package.metadata_name.lower() == 'setuptools': + # setuptools imports itself, that does not work very well + # with the monkey path at the begining + print('Error: setuptools cannot be built using scanPyPI') + continue + + try: + package.download_package() + except six.moves.urllib.error.HTTPError as error: + print('Error: {code} {reason}'.format(code=error.code, + reason=error.reason)) + print('Error downloading package :', package.buildroot_name) + print() + continue + + # extract the tarball + try: + package.extract_package(tmp_path) + except (tarfile.ReadError, zipfile.BadZipfile): + print('Error extracting package {}'.format(package.real_name)) + print() + continue + + # Loading the package install info from the package + try: + package.load_setup() + except ImportError as err: + if 'buildutils' in err.message: + print('This package needs buildutils') + else: + raise + continue + except AttributeError as error: + print('Error: Could not install package {pkg}: {error}'.format( + pkg=package.real_name, error=error)) + continue + + # Package requirement are an argument of the setup function + req_not_found = package.get_requirements(pkg_folder) + req_not_found = req_not_found.difference(packages) + + packages += req_not_found + if req_not_found: + print('Added packages \'{pkgs}\' as dependencies of {pkg}' + .format(pkgs=", ".join(req_not_found), + pkg=package.buildroot_name)) + print('Checking if package {name} already exists...'.format( + name=package.pkg_dir)) + try: + os.makedirs(package.pkg_dir) + except OSError as exception: + if exception.errno != errno.EEXIST: + print("ERROR: ", exception.message, file=sys.stderr) + continue + print('Error: Package {name} already exists' + .format(name=package.pkg_dir)) + del_pkg = input( + 'Do you want to delete existing package ? [y/N]') + if del_pkg.lower() == 'y': + shutil.rmtree(package.pkg_dir) + os.makedirs(package.pkg_dir) + else: + continue + package.create_package_mk() + + package.create_hash_file() + + package.create_config_in() + print("NOTE: Remember to also make an update to the DEVELOPERS file") + print(" and include an entry for the pkg in packages/Config.in") + print() + # printing an empty line for visual confort + finally: + shutil.rmtree(tmp_path) + + +if __name__ == "__main__": + main() diff --git a/utils/size-stats-compare b/utils/size-stats-compare new file mode 100755 index 0000000000..a3d7f250c6 --- /dev/null +++ b/utils/size-stats-compare @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +# Copyright (C) 2016 Thomas De Schampheleire + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# TODO (improvements) +# - support K,M,G size suffixes for threshold +# - output CSV file in addition to stdout reporting + +import csv +import argparse +import sys + + +def read_file_size_csv(inputf, detail=None): + """Extract package or file sizes from CSV file into size dictionary""" + sizes = {} + reader = csv.reader(inputf) + + header = next(reader) + if header[0] != 'File name' or header[1] != 'Package name' or \ + header[2] != 'File size' or header[3] != 'Package size': + print(("Input file %s does not contain the expected header. Are you " + "sure this file corresponds to the file-size-stats.csv " + "file created by 'make graph-size'?") % inputf.name) + sys.exit(1) + + for row in reader: + if detail: + sizes[row[0]] = int(row[2]) + else: + sizes[row[1]] = int(row[3]) + + return sizes + + +def compare_sizes(old, new): + """Return delta/added/removed dictionaries based on two input size + dictionaries""" + delta = {} + oldkeys = set(old.keys()) + newkeys = set(new.keys()) + + # packages/files in both + for entry in newkeys.intersection(oldkeys): + delta[entry] = ('', new[entry] - old[entry]) + # packages/files only in new + for entry in newkeys.difference(oldkeys): + delta[entry] = ('added', new[entry]) + # packages/files only in old + for entry in oldkeys.difference(newkeys): + delta[entry] = ('removed', -old[entry]) + + return delta + + +def print_results(result, threshold): + """Print the given result dictionary sorted by size, ignoring any entries + below or equal to threshold""" + + from six import iteritems + list_result = list(iteritems(result)) + # result is a dictionary: name -> (flag, size difference) + # list_result is a list of tuples: (name, (flag, size difference)) + + for entry in sorted(list_result, key=lambda entry: entry[1][1]): + if threshold is not None and abs(entry[1][1]) <= threshold: + continue + print('%12s %7s %s' % (entry[1][1], entry[1][0], entry[0])) + + +# main ######################################################################### + +description = """ +Compare rootfs size between Buildroot compilations, for example after changing +configuration options or after switching to another Buildroot release. + +This script compares the file-size-stats.csv file generated by 'make graph-size' +with the corresponding file from another Buildroot compilation. +The size differences can be reported per package or per file. +Size differences smaller or equal than a given threshold can be ignored. +""" + +parser = argparse.ArgumentParser(description=description, + formatter_class=argparse.RawDescriptionHelpFormatter) + +parser.add_argument('-d', '--detail', action='store_true', + help='''report differences for individual files rather than + packages''') +parser.add_argument('-t', '--threshold', type=int, + help='''ignore size differences smaller or equal than this + value (bytes)''') +parser.add_argument('old_file_size_csv', type=argparse.FileType('r'), + metavar='old-file-size-stats.csv', + help="""old CSV file with file and package size statistics, + generated by 'make graph-size'""") +parser.add_argument('new_file_size_csv', type=argparse.FileType('r'), + metavar='new-file-size-stats.csv', + help='new CSV file with file and package size statistics') +args = parser.parse_args() + +if args.detail: + keyword = 'file' +else: + keyword = 'package' + +old_sizes = read_file_size_csv(args.old_file_size_csv, args.detail) +new_sizes = read_file_size_csv(args.new_file_size_csv, args.detail) + +delta = compare_sizes(old_sizes, new_sizes) + +print('Size difference per %s (bytes), threshold = %s' % (keyword, args.threshold)) +print(80*'-') +print_results(delta, args.threshold) +print(80*'-') +print_results({'TOTAL': ('', sum(new_sizes.values()) - sum(old_sizes.values()))}, + threshold=None) diff --git a/utils/test-pkg b/utils/test-pkg new file mode 100755 index 0000000000..a317d8c17a --- /dev/null +++ b/utils/test-pkg @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +set -e + +TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv' +TEMP_CONF="" + +do_clean() { + if [ ! -z "${TEMP_CONF}" ]; then + rm -f "${TEMP_CONF}" + fi +} + +main() { + local o O opts + local cfg dir pkg random toolchains_csv toolchain all number mode + local ret nb nb_skip nb_fail nb_legal nb_tc build_dir keep + local -a toolchains + local pkg_br_name + + o='hakc:d:n:p:r:t:' + O='help,all,keep,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:' + opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")" + eval set -- "${opts}" + + random=0 + all=0 + keep=0 + number=0 + mode=0 + toolchains_csv="${TOOLCHAINS_CSV}" + while [ ${#} -gt 0 ]; do + case "${1}" in + (-h|--help) + help; exit 0 + ;; + (-a|--all) + all=1; shift 1 + ;; + (-k|--keep) + keep=1; shift 1 + ;; + (-c|--config-snippet) + cfg="${2}"; shift 2 + ;; + (-d|--build-dir) + dir="${2}"; shift 2 + ;; + (-n|--number) + number="${2}"; shift 2 + ;; + (-p|--package) + pkg="${2}"; shift 2 + ;; + (-r|--random) + random="${2}"; shift 2 + ;; + (-t|--toolchains-csv) + toolchains_csv="${2}"; shift 2 + ;; + (--) + shift; break + ;; + esac + done + + trap do_clean INT TERM HUP EXIT + + if [ -z "${cfg}" ]; then + pkg_br_name="${pkg//-/_}" + pkg_br_name="BR2_PACKAGE_${pkg_br_name^^}" + TEMP_CONF=$(mktemp /tmp/test-${pkg}-config.XXXXXX) + echo "${pkg_br_name}=y" > ${TEMP_CONF} + cfg="${TEMP_CONF}" + fi + if [ ! -e "${cfg}" ]; then + printf "error: %s: no such file\n" "${cfg}" >&2; exit 1 + fi + if [ -z "${dir}" ]; then + dir="${HOME}/br-test-pkg" + fi + + if [ ${random} -gt 0 ]; then + mode=$((mode+1)) + fi + + if [ ${number} -gt 0 ]; then + mode=$((mode+1)) + fi + + if [ ${all} -eq 1 ]; then + mode=$((mode+1)) + fi + + # Default mode is to test the N first toolchains, which have been + # chosen to be a good selection of toolchains. + if [ ${mode} -eq 0 ] ; then + number=6 + elif [ ${mode} -gt 1 ] ; then + printf "error: --all, --number and --random are mutually exclusive\n" >&2; exit 1 + fi + + # Extract the URLs of the toolchains; drop internal toolchains + # E.g.: http://server/path/to/name.config,arch,libc + # --> http://server/path/to/name.config + toolchains=($(sed -r -e 's/,.*//; /internal/d; /^#/d; /^$/d;' "${toolchains_csv}" \ + |if [ ${random} -gt 0 ]; then \ + sort -R |head -n ${random} + elif [ ${number} -gt 0 ]; then \ + head -n ${number} + else + sort + fi + ) + ) + + nb_tc="${#toolchains[@]}" + if [ ${nb_tc} -eq 0 ]; then + printf "error: no toolchain found (networking issue?)\n" >&2; exit 1 + fi + + nb=0 + nb_skip=0 + nb_fail=0 + nb_legal=0 + for toolchainconfig in "${toolchains[@]}"; do + : $((nb++)) + toolchain="$(basename "${toolchainconfig}" .config)" + build_dir="${dir}/${toolchain}" + printf "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} ${nb} ${nb_tc} + build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" && ret=0 || ret=${?} + case ${ret} in + (0) printf "OK\n";; + (1) : $((nb_skip++)); printf "SKIPPED\n";; + (2) : $((nb_fail++)); printf "FAILED\n";; + (3) : $((nb_legal++)); printf "FAILED\n";; + esac + done + + printf "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \ + ${nb} ${nb_skip} ${nb_fail} ${nb_legal} + + return $((nb_fail + nb_legal)) +} + +build_one() { + local dir="${1}" + local toolchainconfig="${2}" + local cfg="${3}" + local pkg="${4}" + + mkdir -p "${dir}" + + CONFIG_= support/kconfig/merge_config.sh -O "${dir}" \ + "${toolchainconfig}" "support/config-fragments/minimal.config" "${cfg}" \ + >> "${dir}/logfile" 2>&1 + # We want all the options from the snippet to be present as-is (set + # or not set) in the actual .config; if one of them is not, it means + # some dependency from the toolchain or arch is not available, in + # which case this config is untestable and we skip it. + # We don't care about the locale to sort in, as long as both sort are + # done in the same locale. + comm -23 <(sort "${cfg}") <(sort "${dir}/.config") >"${dir}/missing.config" + if [ -s "${dir}/missing.config" ]; then + return 1 + fi + # Remove file, it's empty anyway. + rm -f "${dir}/missing.config" + + if [ -n "${pkg}" ]; then + if ! make O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then + return 2 + fi + fi + + # shellcheck disable=SC2086 + if ! BR_FORCE_CHECK_DEPENDENCIES=YES make O="${dir}" ${pkg} >> "${dir}/logfile" 2>&1; then + return 2 + fi + + # legal-info done systematically, because some packages have different + # sources depending on the configuration (e.g. lua-5.2 vs. lua-5.3) + if ! make O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then + return 3 + fi + + # If we get here, the build was successful. Clean up the build/host + # directories to save disk space, unless 'keep' was set. + if [ ${keep} -ne 1 ]; then + make O="${dir}" clean >> "${dir}/logfile" 2>&1 + fi +} + +help() { + cat <<_EOF_ +test-pkg: test-build a package against various toolchains and architectures + +The supplied config snippet is appended to each toolchain config, the +resulting configuration is checked to ensure it still contains all options +specified in the snippet; if any is missing, the build is skipped, on the +assumption that the package under test requires a toolchain or architecture +feature that is missing. + +In case failures are noticed, you can fix the package and just re-run the +same command again; it will re-run the test where it failed. If you did +specify a package (with -p), the package build dir will be removed first. + +The list of toolchains is retrieved from ${TOOLCHAINS_CSV}. +Only the external toolchains are tried, because building a Buildroot toolchain +would take too long. An alternative toolchains CSV file can be specified with +the -t option. This file should have lines consisting of the path to the +toolchain config fragment and the required host architecture, separated by a +comma. The config fragments should contain only the toolchain and architecture +settings. + +By default, a useful subset of toolchains is tested. If needed, all +toolchains can be tested (-a), an arbitrary number of toolchains (-n +in order, -r for random). + +Options: + + -h, --help + Print this help. + + -c CFG, --config-snippet CFG + Use the CFG file as the source for the config snippet. This file + should contain all the config options required to build a package. + + -d DIR, --build-dir DIR + Do the builds in directory DIR, one sub-dir per toolchain. + + -p PKG, --package PKG + Test-build the package PKG, by running 'make PKG'; if not specified, + just runs 'make'. + + -a, --all + Test all toolchains, instead of the default subset defined by + Buildroot developers. + + -n N, --number N + Test N toolchains, in the order defined in the toolchain CSV + file. + + -r N, --random N + Limit the tests to the N randomly selected toolchains. + + -t CSVFILE, --toolchains-csv CSVFILE + CSV file containing the paths to config fragments of toolchains to + try. If not specified, the toolchains in ${TOOLCHAINS_CSV} will be + used. + + -k, --keep + Keep the build directories even if the build succeeds. + Note: the logfile and configuration is always retained, even without + this option. + +Example: + + Testing libcec would require a config snippet that contains: + BR2_PACKAGE_LIBCEC=y + + Testing libcurl with openSSL support would require a snippet such as: + BR2_PACKAGE_OPENSSL=y + BR2_PACKAGE_LIBCURL=y + +_EOF_ +} + +my_name="${0##*/}" +main "${@}" From a9a5829a99b1ed5396defabe9392d287b31b5d64 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 00:18:30 +0100 Subject: [PATCH 08/12] Update all support scripts --- Makefile | 25 +- package/linux-headers/linux-headers.mk | 4 +- support/scripts/br2-external | 234 ++ support/scripts/brpkgutil.py | 50 + support/scripts/build-ext3-img | 152 - support/scripts/check-bin-arch | 93 + support/scripts/check-host-rpath | 111 + support/scripts/check-kernel-headers.sh | 46 +- support/scripts/check-merged-usr.sh | 39 + support/scripts/expunge-gconv-modules | 2 +- support/scripts/fix-configure-powerpc64.sh | 47 + support/scripts/fix-rpath | 162 + support/scripts/gen-manual-lists.py | 517 --- support/scripts/generate-gitlab-ci-yml | 17 + support/scripts/genimage.sh | 48 + support/scripts/graph-build-time | 51 +- support/scripts/graph-depends | 468 +-- support/scripts/hardlink-or-copy | 35 + support/scripts/kconfiglib.py | 3772 -------------------- support/scripts/mkmakefile | 15 +- support/scripts/mkusers | 120 +- support/scripts/pkg-stats | 1296 +++++-- support/scripts/pycompile.py | 69 + support/scripts/readme.kconfiglib | 30 - support/scripts/scancpan | 802 ----- support/scripts/setlocalversion | 4 +- support/scripts/size-stats | 308 ++ support/scripts/xorg-release | 180 - support/testing/.gitkeep | 0 toolchain/helpers.mk | 11 +- 30 files changed, 2659 insertions(+), 6049 deletions(-) create mode 100755 support/scripts/br2-external create mode 100644 support/scripts/brpkgutil.py delete mode 100755 support/scripts/build-ext3-img create mode 100755 support/scripts/check-bin-arch create mode 100755 support/scripts/check-host-rpath create mode 100755 support/scripts/check-merged-usr.sh create mode 100755 support/scripts/fix-configure-powerpc64.sh create mode 100755 support/scripts/fix-rpath delete mode 100644 support/scripts/gen-manual-lists.py create mode 100755 support/scripts/generate-gitlab-ci-yml create mode 100755 support/scripts/genimage.sh create mode 100755 support/scripts/hardlink-or-copy delete mode 100644 support/scripts/kconfiglib.py create mode 100644 support/scripts/pycompile.py delete mode 100644 support/scripts/readme.kconfiglib delete mode 100755 support/scripts/scancpan create mode 100755 support/scripts/size-stats delete mode 100755 support/scripts/xorg-release create mode 100644 support/testing/.gitkeep diff --git a/Makefile b/Makefile index c316d1e40d..eaff91f36f 100644 --- a/Makefile +++ b/Makefile @@ -381,17 +381,15 @@ ifneq ($(BR2_DEPRECATED),y) include Makefile.legacy endif +include system/system.mk include package/Makefile.in # arch/arch.mk must be after package/Makefile.in because it may need to # complement variables defined therein, like BR_NO_CHECK_HASH_FOR. include arch/arch.mk include support/dependencies/dependencies.mk -# We also need the various per-package makefiles, which also add -# each selected package to PACKAGES if that package was selected -# in the .config file. -include toolchain/*.mk -include toolchain/*/*.mk +include $(sort $(wildcard toolchain/*.mk)) +include $(sort $(wildcard toolchain/*/*.mk)) # Include the package override file if one has been provided in the # configuration. @@ -404,7 +402,6 @@ include $(sort $(wildcard package/*/*.mk)) include boot/common.mk include linux/linux.mk -include system/system.mk include fs/common.mk include $(BR2_EXTERNAL)/external.mk @@ -841,13 +838,21 @@ ifeq ($(NEED_WRAPPER),y) $(Q)$(TOPDIR)/support/scripts/mkmakefile $(TOPDIR) $(O) endif -# printvars prints all the variables currently defined in our Makefiles +# printvars prints all the variables currently defined in our +# Makefiles. Alternatively, if a non-empty VARS variable is passed, +# only the variables matching the make pattern passed in VARS are +# displayed. +.PHONY: printvars printvars: - @$(foreach V, \ - $(sort $(.VARIABLES)), \ + @: + $(foreach V, \ + $(sort $(filter $(VARS),$(.VARIABLES))), \ $(if $(filter-out environment% default automatic, \ $(origin $V)), \ - $(info $V=$($V) ($(value $V))))) + $(if $(QUOTED_VARS),\ + $(info $V='$(subst ','\'',$(if $(RAW_VARS),$(value $V),$($V)))'), \ + $(info $V=$(if $(RAW_VARS),$(value $V),$($V)))))) +# ' Syntax colouring... clean: rm -rf $(TARGET_DIR) $(BINARIES_DIR) $(HOST_DIR) \ diff --git a/package/linux-headers/linux-headers.mk b/package/linux-headers/linux-headers.mk index d7709273a3..57e8a042d4 100644 --- a/package/linux-headers/linux-headers.mk +++ b/package/linux-headers/linux-headers.mk @@ -54,8 +54,10 @@ endef ifeq ($(BR2_KERNEL_HEADERS_VERSION),y) define LINUX_HEADERS_CHECK_VERSION $(call check_kernel_headers_version,\ + $(BUILD_DIR),\ $(STAGING_DIR),\ - $(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST))) + $(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)),\ + loose) endef LINUX_HEADERS_POST_INSTALL_STAGING_HOOKS += LINUX_HEADERS_CHECK_VERSION endif diff --git a/support/scripts/br2-external b/support/scripts/br2-external new file mode 100755 index 0000000000..171526f8c8 --- /dev/null +++ b/support/scripts/br2-external @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +set -e + +# This script must be able to run with bash-3.1, so it can't use +# associative arrays. Instead, it emulates them using 'eval'. It +# can however use indexed arrays, supported since at least bash-3.0. + +# The names of the br2-external trees, once validated. +declare -a BR2_EXT_NAMES + +# URL to manual for help in converting old br2-external trees. +# Escape '#' so that make does not consider it a comment. +MANUAL_URL='https://buildroot.org/manual.html\#br2-external-converting' + +main() { + local OPT OPTARG + local br2_ext outputdir + + while getopts :d: OPT; do + case "${OPT}" in + d) outputdir="${OPTARG}";; + :) error "option '%s' expects a mandatory argument\n" "${OPTARG}";; + \?) error "unknown option '%s'\n" "${OPTARG}";; + esac + done + # Forget options; keep only positional args + shift $((OPTIND-1)) + + if [ -z "${outputdir}" ]; then + error "no output directory specified (-d)\n" + fi + + # Trap any unexpected error to generate a meaningful error message + trap "error 'unexpected error while generating ${ofile}\n'" ERR + + do_validate ${@//:/ } + + mkdir -p "${outputdir}" + do_mk "${outputdir}" + do_kconfig "${outputdir}" +} + +# Validates the br2-external trees passed as arguments. Makes each of +# them canonical and store them in the global arrays BR2_EXT_NAMES +# and BR2_EXT_PATHS. +# +# Note: since this script is always first called from Makefile context +# to generate the Makefile fragment before it is called to generate the +# Kconfig snippet, we're sure that any error in do_validate will be +# interpreted in Makefile context. Going up to generating the Kconfig +# snippet means that there were no error. +# +do_validate() { + local br2_ext + + if [ ${#} -eq 0 ]; then + # No br2-external tree is valid + return + fi + + for br2_ext in "${@}"; do + do_validate_one "${br2_ext}" + done +} + +do_validate_one() { + local br2_ext="${1}" + local br2_name br2_desc n d + + if [ ! -d "${br2_ext}" ]; then + error "'%s': no such file or directory\n" "${br2_ext}" + fi + if [ ! -r "${br2_ext}" -o ! -x "${br2_ext}" ]; then + error "'%s': permission denied\n" "${br2_ext}" + fi + if [ ! -f "${br2_ext}/external.desc" ]; then + error "'%s': does not have a name (in 'external.desc'). See %s\n" \ + "${br2_ext}" "${MANUAL_URL}" + fi + br2_name="$(sed -r -e '/^name: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")" + if [ -z "${br2_name}" ]; then + error "'%s/external.desc': does not define the name\n" "${br2_ext}" + fi + # Only ASCII chars in [A-Za-z0-9_] are permitted + n="$(sed -r -e 's/[A-Za-z0-9_]//g' <<<"${br2_name}" )" + if [ -n "${n}" ]; then + # Escape '$' so that it gets printed + error "'%s': name '%s' contains invalid chars: '%s'\n" \ + "${br2_ext}" "${br2_name//\$/\$\$}" "${n//\$/\$\$}" + fi + eval d="\"\${BR2_EXT_PATHS_${br2_name}}\"" + if [ -n "${d}" ]; then + error "'%s': name '%s' is already used in '%s'\n" \ + "${br2_ext}" "${br2_name}" "${d}" + fi + br2_desc="$(sed -r -e '/^desc: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")" + if [ ! -f "${br2_ext}/external.mk" ]; then + error "'%s/external.mk': no such file or directory\n" "${br2_ext}" + fi + if [ ! -f "${br2_ext}/Config.in" ]; then + error "'%s/Config.in': no such file or directory\n" "${br2_ext}" + fi + + # Register this br2-external tree, use an absolute canonical path + br2_ext="$( cd "${br2_ext}"; pwd )" + BR2_EXT_NAMES+=( "${br2_name}" ) + eval BR2_EXT_PATHS_${br2_name}="\"\${br2_ext}\"" + eval BR2_EXT_DESCS_${br2_name}="\"\${br2_desc:-\${br2_name}}\"" +} + +# Generate the .mk snippet that defines makefile variables +# for the br2-external tree +do_mk() { + local outputdir="${1}" + local br2_name br2_desc br2_ext + + { + printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n' + printf '\n' + + printf 'BR2_EXTERNAL ?=' + for br2_name in "${BR2_EXT_NAMES[@]}"; do + eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\"" + printf ' %s' "${br2_ext}" + done + printf '\n' + + printf 'BR2_EXTERNAL_NAMES = \n' + printf 'BR2_EXTERNAL_DIRS = \n' + printf 'BR2_EXTERNAL_MKS = \n' + + if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then + printf '\n' + printf '# No br2-external tree defined.\n' + return + fi + + for br2_name in "${BR2_EXT_NAMES[@]}"; do + eval br2_desc="\"\${BR2_EXT_DESCS_${br2_name}}\"" + eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\"" + printf '\n' + printf 'BR2_EXTERNAL_NAMES += %s\n' "${br2_name}" + printf 'BR2_EXTERNAL_DIRS += %s\n' "${br2_ext}" + printf 'BR2_EXTERNAL_MKS += %s/external.mk\n' "${br2_ext}" + printf 'export BR2_EXTERNAL_%s_PATH = %s\n' "${br2_name}" "${br2_ext}" + printf 'export BR2_EXTERNAL_%s_DESC = %s\n' "${br2_name}" "${br2_desc}" + done + } >"${outputdir}/.br2-external.mk" +} + +# Generate the kconfig snippets for the br2-external tree. +do_kconfig() { + local outputdir="${1}" + local br2_name br2_desc br2_ext br2 + local -a items + + items=( + paths + menus + toolchains + jpeg + openssl + ) + + for br2 in "${items[@]}"; do + { + printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n' + printf '\n' + if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then + printf '# No br2-external tree defined.\n' + fi + } >"${outputdir}/.br2-external.in.${br2}" + done + if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then + return + fi + + printf 'menu "External options"\n\n' >>"${outputdir}/.br2-external.in.menus" + + for br2_name in "${BR2_EXT_NAMES[@]}"; do + eval br2_desc="\"\${BR2_EXT_DESCS_${br2_name}}\"" + eval br2_ext="\"\${BR2_EXT_PATHS_${br2_name}}\"" + + { + printf 'config BR2_EXTERNAL_%s_PATH\n' "${br2_name}" + printf '\tstring\n' + printf '\tdefault "%s"\n' "${br2_ext}" + printf '\n' + } >>"${outputdir}/.br2-external.in.paths" + + { + if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then + printf 'menu "%s"\n' "${br2_desc}" + fi + printf 'comment "%s (in %s)"\n' "${br2_desc}" "${br2_ext}" + printf 'source "%s/Config.in"\n' "${br2_ext}" + if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then + printf 'endmenu # %s\n' "${br2_name}" + fi + printf '\n' + } >>"${outputdir}/.br2-external.in.menus" + + if [ -f "${br2_ext}/provides/toolchains.in" ]; then + printf 'comment "Toolchains from: %s"\n' "${br2_desc}" + printf 'source "%s/provides/toolchains.in"\n' "${br2_ext}" + printf '\n' + else + printf '# No toolchain from: %s\n\n' "${br2_desc}" + fi >>"${outputdir}/.br2-external.in.toolchains" + + if [ -f "${br2_ext}/provides/jpeg.in" ]; then + printf 'comment "jpeg from: %s"\n' "${br2_desc}" + printf 'source "%s/provides/jpeg.in"\n' "${br2_ext}" + printf '\n' + else + printf '# No jpeg from: %s\n\n' "${br2_desc}" + fi >>"${outputdir}/.br2-external.in.jpeg" + + if [ -f "${br2_ext}/provides/openssl.in" ]; then + printf 'comment "openssl from: %s"\n' "${br2_desc}" + printf 'source "%s/provides/openssl.in"\n' "${br2_ext}" + printf '\n' + else + printf '# No openssl from: %s\n\n' "${br2_desc}" + fi >>"${outputdir}/.br2-external.in.openssl" + done + + printf 'endmenu\n' >>"${outputdir}/.br2-external.in.menus" +} + +error() { local fmt="${1}"; shift; printf "BR2_EXTERNAL_ERROR = ${fmt}" "${@}"; exit 1; } + +my_name="${0##*/}" +main "${@}" diff --git a/support/scripts/brpkgutil.py b/support/scripts/brpkgutil.py new file mode 100644 index 0000000000..73d8fbe931 --- /dev/null +++ b/support/scripts/brpkgutil.py @@ -0,0 +1,50 @@ +# Copyright (C) 2010-2013 Thomas Petazzoni +# Copyright (C) 2019 Yann E. MORIN + +import json +import logging +import os +import subprocess +from collections import defaultdict + + +# This function returns a tuple of four dictionaries, all using package +# names as keys: +# - a dictionary which values are the lists of packages that are the +# dependencies of the package used as key; +# - a dictionary which values are the lists of packages that are the +# reverse dependencies of the package used as key; +# - a dictionary which values are the type of the package used as key; +# - a dictionary which values are the version of the package used as key, +# 'virtual' for a virtual package, or the empty string for a rootfs. +def get_dependency_tree(): + logging.info("Getting dependency tree...") + + deps = {} + rdeps = defaultdict(list) + types = {} + versions = {} + + # Special case for the 'all' top-level fake package + deps['all'] = [] + types['all'] = 'target' + versions['all'] = '' + + cmd = ["make", "-s", "--no-print-directory", "show-info"] + with open(os.devnull, 'wb') as devnull: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull, + universal_newlines=True) + pkg_list = json.loads(p.communicate()[0]) + + for pkg in pkg_list: + deps['all'].append(pkg) + types[pkg] = pkg_list[pkg]["type"] + deps[pkg] = pkg_list[pkg].get("dependencies", []) + for p in deps[pkg]: + rdeps[p].append(pkg) + versions[pkg] = \ + None if pkg_list[pkg]["type"] == "rootfs" \ + else "virtual" if pkg_list[pkg]["virtual"] \ + else pkg_list[pkg]["version"] + + return (deps, rdeps, types, versions) diff --git a/support/scripts/build-ext3-img b/support/scripts/build-ext3-img deleted file mode 100755 index 1cab710c25..0000000000 --- a/support/scripts/build-ext3-img +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/sh - -BLOCKSIZE=516096 -WORKING_DIR=`pwd` - -echo "This script will create a bootable ext3 image from buildroot." - -echo "Enter the path to the image (${WORKING_DIR})" -read IMG_PATH - -if [ "${IMAGE_PATH}" = "" ]; then - IMAGE_PATH=${WORKING_DIR} -fi - -echo "Enter the name of the image file (buildroot.img)" -read IMG_NAME - -if [ "${IMAGE_NAME}" = "" ]; then - IMAGE_NAME="buildroot.img" -fi - -IMAGE=${IMAGE_PATH}/${IMAGE_NAME} - -echo "Enter the path and filename for the root filesystem" -echo "tarball that you want to install into the image" -read ROOT_PATH - -if [ "${ROOT_PATH}" = "" ]; then - echo "Error: you must specify a path." - exit 1 -fi - -CYLINDERS=`du --summarize --block-size=${BLOCKSIZE} ${ROOT_PATH}` -BYTE_SIZE=`du --summarize --block-size=${BLOCKSIZE} --human-readable ${ROOT_PATH}` - -CYLINDERS=${CYLINDERS%${ROOT_PATH}} -BYTE_SIZE=${BYTE_SIZE%${ROOT_PATH}} - -CYLINDERS=`expr ${CYLINDERS} "*" 2` - -echo "Now I will create an ext3 image file" -echo "using ${CYLINDERS} cylinders, with ${BLOCKSIZE} bytes per block" -echo "in other words, ${BYTE_SIZE}bytes..." - - dd if=/dev/zero of=${IMAGE} bs=${BLOCKSIZE}c count=${CYLINDERS} - -# Create file partition and filesystem - - # STEP 1. create partition - /sbin/losetup /dev/loop3 ${IMAGE} - # probably should figure out how to use GNU parted to do this non-interactively - /sbin/fdisk -u -C${CYLINDERS} -S63 -H16 /dev/loop3 - /sbin/losetup -d /dev/loop3 - - # STEP 2. make file system (ext3) - /sbin/losetup -o 32256 /dev/loop3 ${IMAGE} - /sbin/mkfs.ext3 /dev/loop3 - /sbin/losetup -d /dev/loop3 - -# Install Software to the image - mkdir -p ${IMAGE_PATH}/temp - mount -o offset=32256,loop ${IMAGE} ${IMAGE_PATH}/temp - tar -xvf ${ROOT_PATH} --directory ${IMAGE_PATH}/temp - # make sure to unmount the image - umount ${IMAGE_PATH}/temp - rm -rf ${IMAGE_PATH}/temp - -# Create a VMware .vmx file -cat > ${IMAGE_PATH}/buildroot.vmx < -l -r -a [-i PATH ...]" + exit 1 +fi + +exitcode=0 + +# Only split on new lines, for filenames-with-spaces +IFS=" +" + +while read f; do + for ignore in "${IGNORES[@]}"; do + if [[ "${f}" =~ ^"${ignore}" ]]; then + continue 2 + fi + done + + # Skip symlinks. Some symlinks may have absolute paths as + # target, pointing to host binaries while we're building. + if [[ -L "${TARGET_DIR}/${f}" ]]; then + continue + fi + + # Get architecture using readelf. We pipe through 'head -1' so + # that when the file is a static library (.a), we only take + # into account the architecture of the first object file. + arch=$(LC_ALL=C ${readelf} -h "${TARGET_DIR}/${f}" 2>&1 | \ + sed -r -e '/^ Machine: +(.+)/!d; s//\1/;' | head -1) + + # If no architecture found, assume it was not an ELF file + if test "${arch}" = "" ; then + continue + fi + + # Architecture is correct + if test "${arch}" = "${arch_name}" ; then + continue + fi + + printf 'ERROR: architecture for "%s" is "%s", should be "%s"\n' \ + "${f}" "${arch}" "${arch_name}" + + exitcode=1 +done < <( sed -r -e "/^${package},\.(.+)$/!d; s//\1/;" ${pkg_list} ) + +exit ${exitcode} diff --git a/support/scripts/check-host-rpath b/support/scripts/check-host-rpath new file mode 100755 index 0000000000..9a3866982b --- /dev/null +++ b/support/scripts/check-host-rpath @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# This script scans $(HOST_DIR)/{bin,sbin} for all ELF files, and checks +# they have an RPATH to $(HOST_DIR)/lib if they need libraries from +# there. + +# Override the user's locale so we are sure we can parse the output of +# readelf(1) and file(1) +export LC_ALL=C + +main() { + local pkg="${1}" + local hostdir="${2}" + local perpackagedir="${3}" + local file ret + + # Remove duplicate and trailing '/' for proper match + hostdir="$( sed -r -e 's:/+:/:g; s:/$::;' <<<"${hostdir}" )" + + ret=0 + while read file; do + is_elf "${file}" || continue + elf_needs_rpath "${file}" "${hostdir}" || continue + check_elf_has_rpath "${file}" "${hostdir}" "${perpackagedir}" && continue + if [ ${ret} -eq 0 ]; then + ret=1 + printf "***\n" + printf "*** ERROR: package %s installs executables without proper RPATH:\n" "${pkg}" + fi + printf "*** %s\n" "${file}" + done < <( find "${hostdir}"/{bin,sbin} -type f 2>/dev/null ) + + return ${ret} +} + +is_elf() { + local f="${1}" + + readelf -l "${f}" 2>/dev/null \ + |grep -E 'Requesting program interpreter:' >/dev/null 2>&1 +} + +# This function tells whether a given ELF executable (first argument) +# needs a RPATH pointing to the host library directory or not. It +# needs such an RPATH if at least of the libraries used by the ELF +# executable is available in the host library directory. This function +# returns 0 when a RPATH is needed, 1 otherwise. +# +# With per-package directory support, ${hostdir} will point to the +# current package per-package host directory, and this is where this +# function will check if the libraries needed by the executable are +# located (or not). In practice, the ELF executable RPATH may point to +# another package per-package host directory, but that is fine because +# if such an executable is within the current package per-package host +# directory, its libraries will also have been copied into the current +# package per-package host directory. +elf_needs_rpath() { + local file="${1}" + local hostdir="${2}" + local lib + + while read lib; do + [ -e "${hostdir}/lib/${lib}" ] && return 0 + done < <( readelf -d "${file}" \ + |sed -r -e '/^.* \(NEEDED\) .*Shared library: \[(.+)\]$/!d;' \ + -e 's//\1/;' \ + ) + + return 1 +} + +# This function checks whether at least one of the RPATH of the given +# ELF executable (first argument) properly points to the host library +# directory (second argument), either through an absolute RPATH or a +# relative RPATH. In the context of per-package directory support, +# ${hostdir} (second argument) points to the current package host +# directory. However, it is perfectly valid for an ELF binary to have +# a RPATH pointing to another package per-package host directory, +# which is why such RPATH is also accepted (the per-package directory +# gets passed as third argument). Having a RPATH pointing to the host +# directory will make sure the ELF executable will find at runtime the +# shared libraries it depends on. This function returns 0 when a +# proper RPATH was found, or 1 otherwise. +check_elf_has_rpath() { + local file="${1}" + local hostdir="${2}" + local perpackagedir="${3}" + local rpath dir + + while read rpath; do + for dir in ${rpath//:/ }; do + # Remove duplicate and trailing '/' for proper match + dir="$( sed -r -e 's:/+:/:g; s:/$::;' <<<"${dir}" )" + [ "${dir}" = "${hostdir}/lib" ] && return 0 + [ "${dir}" = "\$ORIGIN/../lib" ] && return 0 + # This check is done even for builds where + # BR2_PER_PACKAGE_DIRECTORIES is disabled. In this case, + # PER_PACKAGE_DIR and therefore ${perpackagedir} points to + # a non-existent directory, and this check will always be + # false. + [[ ${dir} =~ ${perpackagedir}/[^/]+/host/lib ]] && return 0 + done + done < <( readelf -d "${file}" \ + |sed -r -e '/.* \(R(UN)?PATH\) +Library r(un)?path: \[(.+)\]$/!d' \ + -e 's//\3/;' \ + ) + + return 1 +} + +main "${@}" diff --git a/support/scripts/check-kernel-headers.sh b/support/scripts/check-kernel-headers.sh index 82cf9ac274..4e6dce5487 100755 --- a/support/scripts/check-kernel-headers.sh +++ b/support/scripts/check-kernel-headers.sh @@ -1,14 +1,40 @@ #!/bin/sh -SYSROOT="${1}" +# This script (and the embedded C code) will check that the actual +# headers version match the user told us they were: +# +# - if both versions are the same, all is well. +# +# - if the actual headers are older than the user told us, this is +# an error. +# +# - if the actual headers are more recent than the user told us, and +# we are doing a strict check, then this is an error. +# +# - if the actual headers are more recent than the user told us, and +# we are doing a loose check, then a warning is printed, but this is +# not an error. + +BUILDDIR="${1}" +SYSROOT="${2}" # Make sure we have enough version components -HDR_VER="${2}.0.0" +HDR_VER="${3}.0.0" +CHECK="${4}" # 'strict' or 'loose' HDR_M="${HDR_VER%%.*}" HDR_V="${HDR_VER#*.}" HDR_m="${HDR_V%%.*}" -EXEC="$(mktemp --tmpdir check-headers.XXXXXX)" +# Exit on any error, so we don't try to run an unexisting program if the +# compilation fails. +set -e + +# Set the clean-up trap in advance to prevent a race condition in which we +# create the file but get a SIGTERM before setting it. Notice that we don't +# need to care about EXEC being empty, since 'rm -f ""' does nothing. +trap 'rm -f "${EXEC}"' EXIT + +EXEC="$(mktemp -p "${BUILDDIR}" -t .check-headers.XXXXXX)" # We do not want to account for the patch-level, since headers are # not supposed to change for different patchlevels, so we mask it out. @@ -18,13 +44,18 @@ ${HOSTCC} -imacros "${SYSROOT}/usr/include/linux/version.h" \ -x c -o "${EXEC}" - <<_EOF_ #include #include +#include int main(int argc __attribute__((unused)), char** argv __attribute__((unused))) { - if((LINUX_VERSION_CODE & ~0xFF) - != KERNEL_VERSION(${HDR_M},${HDR_m},0)) - { + int l = LINUX_VERSION_CODE & ~0xFF; + int h = KERNEL_VERSION(${HDR_M},${HDR_m},0); + + if ((l >= h) && !strcmp("${CHECK}", "loose")) + return 0; + + if (l != h) { printf("Incorrect selection of kernel headers: "); printf("expected %d.%d.x, got %d.%d.x\n", ${HDR_M}, ${HDR_m}, ((LINUX_VERSION_CODE>>16) & 0xFF), @@ -36,6 +67,3 @@ int main(int argc __attribute__((unused)), _EOF_ "${EXEC}" -ret=${?} -rm -f "${EXEC}" -exit ${ret} diff --git a/support/scripts/check-merged-usr.sh b/support/scripts/check-merged-usr.sh new file mode 100755 index 0000000000..433857cd8c --- /dev/null +++ b/support/scripts/check-merged-usr.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# +# Check if a given custom skeleton or overlay complies to the merged /usr +# requirements: +# / +# /bin -> usr/bin +# /lib -> usr/lib +# /sbin -> usr/sbin +# /usr/bin/ +# /usr/lib/ +# /usr/sbin/ +# +# Output: the list of non-compliant paths (empty if compliant). +# + +# Extract the inode numbers for all of those directories. In case any is +# a symlink, we want to get the inode of the pointed-to directory, so we +# append '/.' to be sure we get the target directory. Since the symlinks +# can be anyway (/bin -> /usr/bin or /usr/bin -> /bin), we do that for +# all of them. +# +lib_inode=$(stat -c '%i' "${1}/lib/." 2>/dev/null) +bin_inode=$(stat -c '%i' "${1}/bin/." 2>/dev/null) +sbin_inode=$(stat -c '%i' "${1}/sbin/." 2>/dev/null) +usr_lib_inode=$(stat -c '%i' "${1}/usr/lib/." 2>/dev/null) +usr_bin_inode=$(stat -c '%i' "${1}/usr/bin/." 2>/dev/null) +usr_sbin_inode=$(stat -c '%i' "${1}/usr/sbin/." 2>/dev/null) + +not_merged_dirs="" +test -z "$lib_inode" || \ + test "$lib_inode" = "$usr_lib_inode" || \ + not_merged_dirs="/lib" +test -z "$bin_inode" || \ + test "$bin_inode" = "$usr_bin_inode" || \ + not_merged_dirs="$not_merged_dirs /bin" +test -z "$sbin_inode" || \ + test "$sbin_inode" = "$usr_sbin_inode" || \ + not_merged_dirs="$not_merged_dirs /sbin" +echo "${not_merged_dirs# }" diff --git a/support/scripts/expunge-gconv-modules b/support/scripts/expunge-gconv-modules index a77b063cac..03012c1ce3 100755 --- a/support/scripts/expunge-gconv-modules +++ b/support/scripts/expunge-gconv-modules @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used to generate a gconv-modules file that takes into # account only the gconv modules installed by Buildroot. It receives diff --git a/support/scripts/fix-configure-powerpc64.sh b/support/scripts/fix-configure-powerpc64.sh new file mode 100755 index 0000000000..ff2b283ce9 --- /dev/null +++ b/support/scripts/fix-configure-powerpc64.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# This is a script to find, and correct, a problem with old versions of +# configure that affect powerpc64 and powerpc64le. + +# The issue causes configure to incorrectly determine that shared library +# support is not present in the linker. This causes the package to build a +# static library rather than a dynamic one and although the build will succeed, +# it may cause packages that link with the static library it to fail due to +# undefined symbols. + +# This script searches for files named 'configure' that appear to have this +# issue (by searching for a known bad pattern) and patching them. + +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 2 +fi + +srcdir="$1" +files=$(cd "$srcdir" && find . -name configure \ +-exec grep -qF 'Generated by GNU Autoconf' {} \; \ +-exec grep -qF 'ppc*-*linux*|powerpc*-*linux*)' {} \; -print) + +# --ignore-whitespace is needed because some packages have included +# copies of configure scripts where tabs have been replaced with spaces. +for c in $files; do + patch --ignore-whitespace "$srcdir"/"$c" <<'EOF' +--- a/configure 2016-11-16 15:31:46.097447271 +1100 ++++ b/configure 2008-07-21 12:17:23.000000000 +1000 +@@ -4433,7 +4433,10 @@ + x86_64-*linux*) + LD="${LD-ld} -m elf_x86_64" + ;; +- ppc*-*linux*|powerpc*-*linux*) ++ powerpcle-*linux*) ++ LD="${LD-ld} -m elf64lppc" ++ ;; ++ powerpc-*linux*) + LD="${LD-ld} -m elf64ppc" + ;; + s390*-*linux*) +EOF +done + diff --git a/support/scripts/fix-rpath b/support/scripts/fix-rpath new file mode 100755 index 0000000000..b4ede000dd --- /dev/null +++ b/support/scripts/fix-rpath @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +# Copyright (C) 2016 Samuel Martin +# Copyright (C) 2017 Wolfgang Grandegger +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +usage() { + cat <&2 +Usage: ${0} TREE_KIND + +Description: + + This script scans a tree and sanitize ELF files' RPATH found in there. + + Sanitization behaves the same whatever the kind of the processed tree, + but the resulting RPATH differs. The rpath sanitization is done using + "patchelf --make-rpath-relative". + +Arguments: + + TREE_KIND Kind of tree to be processed. + Allowed values: host, target, staging + +Environment: + + PATCHELF patchelf program to use + (default: HOST_DIR/bin/patchelf) + + HOST_DIR host directory + STAGING_DIR staging directory + TARGET_DIR target directory + + TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR + (default HOST_DIR/opt/ext-toolchain) + +Returns: 0 if success or 1 in case of error + +EOF +} + +: ${PATCHELF:=${HOST_DIR}/usr/bin/patchelf} + +# ELF files should not be in these sub-directories +HOST_EXCLUDEPATHS="/share/terminfo" +STAGING_EXCLUDEPATHS="/usr/include /usr/share/terminfo" +TARGET_EXCLUDEPATHS="/lib/firmware" + +main() { + local rootdir + local tree="${1}" + local find_args=( ) + local sanitize_extra_args=( ) + + if ! "${PATCHELF}" --version > /dev/null 2>&1; then + echo "Error: can't execute patchelf utility '${PATCHELF}'" + exit 1 + fi + + case "${tree}" in + host) + rootdir="${HOST_DIR}" + + # do not process the sysroot (only contains target binaries) + find_args+=( "-path" "${STAGING_DIR}" "-prune" "-o" ) + + # do not process the external toolchain installation directory to + # avoid breaking it. + test "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" != "" && \ + find_args+=( "-path" "${TOOLCHAIN_EXTERNAL_DOWNLOAD_INSTALL_DIR}" "-prune" "-o" ) + + for excludepath in ${HOST_EXCLUDEPATHS}; do + find_args+=( "-path" "${HOST_DIR}""${excludepath}" "-prune" "-o" ) + done + + # do not process the patchelf binary but a copy to work-around "file in use" + find_args+=( "-path" "${PATCHELF}" "-prune" "-o" ) + cp "${PATCHELF}" "${PATCHELF}.__to_be_patched" + + # we always want $ORIGIN-based rpaths to make it relocatable. + sanitize_extra_args+=( "--relative-to-file" ) + ;; + + staging) + rootdir="${STAGING_DIR}" + + # ELF files should not be in these sub-directories + for excludepath in ${STAGING_EXCLUDEPATHS}; do + find_args+=( "-path" "${STAGING_DIR}""${excludepath}" "-prune" "-o" ) + done + + # should be like for the target tree below + sanitize_extra_args+=( "--no-standard-lib-dirs" ) + ;; + + target) + rootdir="${TARGET_DIR}" + + for excludepath in ${TARGET_EXCLUDEPATHS}; do + find_args+=( "-path" "${TARGET_DIR}""${excludepath}" "-prune" "-o" ) + done + + # we don't want $ORIGIN-based rpaths but absolute paths without rootdir. + # we also want to remove rpaths pointing to /lib or /usr/lib. + sanitize_extra_args+=( "--no-standard-lib-dirs" ) + ;; + + *) + usage + exit 1 + ;; + esac + + find_args+=( "-type" "f" "-print" ) + + while read file ; do + # check if it's an ELF file + rpath=$(${PATCHELF} --print-rpath "${file}" 2>&1) + if test $? -ne 0 ; then + continue + fi + + # make files writable if necessary + changed=$(chmod -c u+w "${file}") + + # With per-package directory support, most RPATH of host + # binaries will point to per-package directories. This won't + # work with the --make-rpath-relative ${rootdir} invocation as + # the per-package host directory is not within ${rootdir}. So, + # we rewrite all RPATHs pointing to per-package directories so + # that they point to the global host directry. + changed_rpath=$(echo ${rpath} | sed "s@${PER_PACKAGE_DIR}/[^/]\+/host@${HOST_DIR}@") + if test "${rpath}" != "${changed_rpath}" ; then + ${PATCHELF} --set-rpath ${changed_rpath} "${file}" + fi + + # call patchelf to sanitize the rpath + ${PATCHELF} --make-rpath-relative "${rootdir}" ${sanitize_extra_args[@]} "${file}" + # restore the original permission + test "${changed}" != "" && chmod u-w "${file}" + done < <(find "${rootdir}" ${find_args[@]}) + + # Restore patched patchelf utility + test "${tree}" = "host" && mv "${PATCHELF}.__to_be_patched" "${PATCHELF}" + + # ignore errors + return 0 +} + +main ${@} diff --git a/support/scripts/gen-manual-lists.py b/support/scripts/gen-manual-lists.py deleted file mode 100644 index 95c10dcbc8..0000000000 --- a/support/scripts/gen-manual-lists.py +++ /dev/null @@ -1,517 +0,0 @@ -## gen-manual-lists.py -## -## This script generates the following Buildroot manual appendices: -## - the package tables (one for the target, the other for host tools); -## - the deprecated items. -## -## Author(s): -## - Samuel Martin -## -## Copyright (C) 2013 Samuel Martin -## -## This program is free software; you can redistribute it and/or modify -## it under the terms of the GNU General Public License as published by -## the Free Software Foundation; either version 2 of the License, or -## (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -## - -## Note about python2. -## -## This script can currently only be run using python2 interpreter due to -## its kconfiglib dependency (which is not yet python3 friendly). - -from __future__ import print_function -from __future__ import unicode_literals - -import os -import re -import sys -import datetime -from argparse import ArgumentParser - -try: - import kconfiglib -except ImportError: - message = """ -Could not find the module 'kconfiglib' in the PYTHONPATH: -""" - message += "\n".join([" {0}".format(path) for path in sys.path]) - message += """ - -Make sure the Kconfiglib directory is in the PYTHONPATH, then relaunch the -script. - -You can get kconfiglib from: - https://github.com/ulfalizer/Kconfiglib - - -""" - sys.stderr.write(message) - raise - - -def get_symbol_subset(root, filter_func): - """ Return a generator of kconfig items. - - :param root_item: Root item of the generated subset of items - :param filter_func: Filter function - - """ - if hasattr(root, "get_items"): - get_items = root.get_items - elif hasattr(root, "get_top_level_items"): - get_items = root.get_top_level_items - else: - message = "The symbol does not contain any subset of symbols" - raise Exception(message) - for item in get_items(): - if item.is_symbol(): - if not filter_func(item): - continue - yield item - elif item.is_menu() or item.is_choice(): - for i in get_symbol_subset(item, filter_func): - yield i - - -def get_symbol_parents(item, root=None, enable_choice=False): - """ Return the list of the item's parents. The last item of the list is - the closest parent, the first the furthest. - - :param item: Item from which the the parent list is generated - :param root: Root item stopping the search (not included in the - parent list) - :param enable_choice: Flag enabling choices to appear in the parent list - - """ - parent = item.get_parent() - parents = [] - while parent and parent != root: - if parent.is_menu(): - parents.append(parent.get_title()) - elif enable_choice and parent.is_choice(): - parents.append(parent.prompts[0][0]) - parent = parent.get_parent() - if isinstance(root, kconfiglib.Menu) or \ - (enable_choice and isinstance(root, kconfiglib.Choice)): - parents.append("") # Dummy empty parent to get a leading arrow -> - parents.reverse() - return parents - - -def format_asciidoc_table(root, get_label_func, filter_func=lambda x: True, - format_func=lambda x: x, - enable_choice=False, sorted=True, - item_label=None): - """ Return the asciidoc formatted table of the items and their location. - - :param root: Root item of the item subset - :param get_label_func: Item's label getter function - :param filter_func: Filter function to apply on the item subset - :param format_func: Function to format a symbol and the table header - :param enable_choice: Enable choices to appear as part of the item's - location - :param sorted: Flag to alphabetically sort the table - - """ - - lines = [] - for item in get_symbol_subset(root, filter_func): - lines.append(format_func(what="symbol", symbol=item, root=root, - get_label_func=get_label_func, - enable_choice=enable_choice)) - if sorted: - lines.sort(key=lambda x: x.lower()) - table = ":halign: center\n\n" - width, columns = format_func(what="layout") - table = "[width=\"{0}\",cols=\"{1}\",options=\"header\"]\n".format(width, columns) - table += "|===================================================\n" - table += format_func(what="header", header=item_label, root=root) - table += "\n" + "".join(lines) + "\n" - table += "|===================================================\n" - return table - - -class Buildroot: - """ Buildroot configuration object. - - """ - root_config = "Config.in" - package_dirname = "package" - package_prefixes = ["BR2_PACKAGE_", "BR2_PACKAGE_HOST_"] - re_pkg_prefix = re.compile(r"^(" + "|".join(package_prefixes) + ").*") - deprecated_symbol = "BR2_DEPRECATED" - list_in = """\ -// -// Automatically generated list for Buildroot manual. -// - -{table} -""" - - list_info = { - 'target-packages': { - 'filename': "package-list", - 'root_menu': "Target packages", - 'filter': "_is_real_package", - 'format': "_format_symbol_prompt_location", - 'sorted': True, - }, - 'host-packages': { - 'filename': "host-package-list", - 'root_menu': "Host utilities", - 'filter': "_is_real_package", - 'format': "_format_symbol_prompt", - 'sorted': True, - }, - 'virtual-packages': { - 'filename': "virtual-package-list", - 'root_menu': "Target packages", - 'filter': "_is_virtual_package", - 'format': "_format_symbol_virtual", - 'sorted': True, - }, - 'deprecated': { - 'filename': "deprecated-list", - 'root_menu': None, - 'filter': "_is_deprecated", - 'format': "_format_symbol_prompt_location", - 'sorted': False, - }, - } - - def __init__(self): - self.base_dir = os.environ.get("TOPDIR") - self.output_dir = os.environ.get("O") - self.package_dir = os.path.join(self.base_dir, self.package_dirname) - # The kconfiglib requires an environment variable named "srctree" to - # load the configuration, so set it. - os.environ.update({'srctree': self.base_dir}) - self.config = kconfiglib.Config(os.path.join(self.base_dir, - self.root_config)) - self._deprecated = self.config.get_symbol(self.deprecated_symbol) - - self.gen_date = datetime.datetime.utcnow() - self.br_version_full = os.environ.get("BR2_VERSION_FULL") - if self.br_version_full and self.br_version_full.endswith("-git"): - self.br_version_full = self.br_version_full[:-4] - if not self.br_version_full: - self.br_version_full = "undefined" - - def _get_package_symbols(self, package_name): - """ Return a tuple containing the target and host package symbol. - - """ - symbols = re.sub("[-+.]", "_", package_name) - symbols = symbols.upper() - symbols = tuple([prefix + symbols for prefix in self.package_prefixes]) - return symbols - - def _is_deprecated(self, symbol): - """ Return True if the symbol is marked as deprecated, otherwise False. - - """ - # This also catches BR2_DEPRECATED_SINCE_xxxx_xx - return bool([ symbol for x in symbol.get_referenced_symbols() - if x.get_name().startswith(self._deprecated.get_name()) ]) - - def _is_package(self, symbol, type='real'): - """ Return True if the symbol is a package or a host package, otherwise - False. - - :param symbol: The symbol to check - :param type: Limit to 'real' or 'virtual' types of packages, - with 'real' being the default. - Note: only 'real' is (implictly) handled for now - - """ - if not symbol.is_symbol(): - return False - if type == 'real' and not symbol.prompts: - return False - if type == 'virtual' and symbol.prompts: - return False - if not self.re_pkg_prefix.match(symbol.get_name()): - return False - pkg_name = self._get_pkg_name(symbol) - - pattern = "^(HOST_)?" + pkg_name + "$" - pattern = re.sub("_", ".", pattern) - pattern = re.compile(pattern, re.IGNORECASE) - # Here, we cannot just check for the location of the Config.in because - # of the "virtual" package. - # - # So, to check that a symbol is a package (not a package option or - # anything else), we check for the existence of the package *.mk file. - # - # By the way, to actually check for a package, we should grep all *.mk - # files for the following regex: - # "\$\(eval \$\((host-)?(generic|autotools|cmake)-package\)\)" - # - # Implementation details: - # - # * The package list is generated from the *.mk file existence, the - # first time this function is called. Despite the memory consumption, - # this list is stored because the execution time of this script is - # noticeably shorter than rescanning the package sub-tree for each - # symbol. - if not hasattr(self, "_package_list"): - pkg_list = [] - for _, _, files in os.walk(self.package_dir): - for file_ in (f for f in files if f.endswith(".mk")): - pkg_list.append(re.sub(r"(.*?)\.mk", r"\1", file_)) - setattr(self, "_package_list", pkg_list) - for pkg in getattr(self, "_package_list"): - if type == 'real': - if pattern.match(pkg) and not self._exists_virt_symbol(pkg): - return True - if type == 'virtual': - if pattern.match('has_' + pkg): - return True - return False - - def _is_real_package(self, symbol): - return self._is_package(symbol, 'real') - - def _is_virtual_package(self, symbol): - return self._is_package(symbol, 'virtual') - - def _exists_virt_symbol(self, pkg_name): - """ Return True if a symbol exists that defines the package as - a virtual package, False otherwise - - :param pkg_name: The name of the package, for which to check if - a symbol exists defining it as a virtual package - - """ - virt_pattern = "BR2_PACKAGE_HAS_" + pkg_name + "$" - virt_pattern = re.sub("_", ".", virt_pattern) - virt_pattern = re.compile(virt_pattern, re.IGNORECASE) - for sym in self.config: - if virt_pattern.match(sym.get_name()): - return True - return False - - def _get_pkg_name(self, symbol): - """ Return the package name of the specified symbol. - - :param symbol: The symbol to get the package name of - - """ - - return re.sub("BR2_PACKAGE_(HOST_)?(.*)", r"\2", symbol.get_name()) - - def _get_symbol_label(self, symbol, mark_deprecated=True): - """ Return the label (a.k.a. prompt text) of the symbol. - - :param symbol: The symbol - :param mark_deprecated: Append a 'deprecated' to the label - - """ - label = symbol.prompts[0][0] - if self._is_deprecated(symbol) and mark_deprecated: - label += " *(deprecated)*" - return label - - def _format_symbol_prompt(self, what=None, symbol=None, root=None, - enable_choice=False, header=None, - get_label_func=lambda x: x): - if what == "layout": - return ( "30%", "^1" ) - - if what == "header": - return "| {0:<40}\n".format(header) - - if what == "symbol": - return "| {0:<40}\n".format(get_label_func(symbol)) - - message = "Invalid argument 'what': '%s'\n" % str(what) - message += "Allowed values are: 'layout', 'header' and 'symbol'" - raise Exception(message) - - def _format_symbol_prompt_location(self, what=None, symbol=None, root=None, - enable_choice=False, header=None, - get_label_func=lambda x: x): - if what == "layout": - return ( "100%", "^1,4" ) - - if what == "header": - if hasattr(root, "get_title"): - loc_label = get_symbol_parents(root, None, enable_choice=enable_choice) - loc_label += [root.get_title(), "..."] - else: - loc_label = ["Location"] - return "| {0:<40} <| {1}\n".format(header, " -> ".join(loc_label)) - - if what == "symbol": - parents = get_symbol_parents(symbol, root, enable_choice) - return "| {0:<40} <| {1}\n".format(get_label_func(symbol), - " -> ".join(parents)) - - message = "Invalid argument 'what': '%s'\n" % str(what) - message += "Allowed values are: 'layout', 'header' and 'symbol'" - raise Exception(message) - - def _format_symbol_virtual(self, what=None, symbol=None, root=None, - enable_choice=False, header=None, - get_label_func=lambda x: "?"): - def _symbol_is_legacy(symbol): - selects = [ s.get_name() for s in symbol.get_selected_symbols() ] - return ("BR2_LEGACY" in selects) - - def _get_parent_package(sym): - if self._is_real_package(sym): - return None - # Trim the symbol name from its last component (separated with - # underscores), until we either find a symbol which is a real - # package, or until we have no component (i.e. just 'BR2') - name = sym.get_name() - while name != "BR2": - name = name.rsplit("_", 1)[0] - s = self.config.get_symbol(name) - if s is None: - continue - if self._is_real_package(s): - return s - return None - - def _get_providers(symbol): - providers = list() - for sym in self.config: - if not sym.is_symbol(): - continue - if _symbol_is_legacy(sym): - continue - selects = sym.get_selected_symbols() - if not selects: - continue - for s in selects: - if s == symbol: - if sym.prompts: - l = self._get_symbol_label(sym,False) - parent_pkg = _get_parent_package(sym) - if parent_pkg is not None: - l = self._get_symbol_label(parent_pkg, False) \ - + " (w/ " + l + ")" - providers.append(l) - else: - providers.extend(_get_providers(sym)) - return providers - - if what == "layout": - return ( "100%", "^1,4,4" ) - - if what == "header": - return "| {0:<20} <| {1:<32} <| Providers\n".format("Virtual packages", "Symbols") - - if what == "symbol": - pkg = re.sub(r"^BR2_PACKAGE_HAS_(.+)$", r"\1", symbol.get_name()) - providers = _get_providers(symbol) - - return "| {0:<20} <| {1:<32} <| {2}\n".format(pkg.lower(), - '+' + symbol.get_name() + '+', - ", ".join(providers)) - - message = "Invalid argument 'what': '%s'\n" % str(what) - message += "Allowed values are: 'layout', 'header' and 'symbol'" - raise Exception(message) - - - def print_list(self, list_type, enable_choice=True, enable_deprecated=True, - dry_run=False, output=None): - """ Print the requested list. If not dry run, then the list is - automatically written in its own file. - - :param list_type: The list type to be generated - :param enable_choice: Flag enabling choices to appear in the list - :param enable_deprecated: Flag enabling deprecated items to appear in - the package lists - :param dry_run: Dry run (print the list in stdout instead of - writing the list file - - """ - def _get_menu(title): - """ Return the first symbol menu matching the given title. - - """ - menus = self.config.get_menus() - menu = [m for m in menus if m.get_title().lower() == title.lower()] - if not menu: - message = "No such menu: '{0}'".format(title) - raise Exception(message) - return menu[0] - - list_config = self.list_info[list_type] - root_title = list_config.get('root_menu') - if root_title: - root_item = _get_menu(root_title) - else: - root_item = self.config - filter_ = getattr(self, list_config.get('filter')) - filter_func = lambda x: filter_(x) - format_func = getattr(self, list_config.get('format')) - if not enable_deprecated and list_type != "deprecated": - filter_func = lambda x: filter_(x) and not self._is_deprecated(x) - mark_depr = list_type != "deprecated" - get_label = lambda x: self._get_symbol_label(x, mark_depr) - item_label = "Features" if list_type == "deprecated" else "Packages" - - table = format_asciidoc_table(root_item, get_label, - filter_func=filter_func, - format_func=format_func, - enable_choice=enable_choice, - sorted=list_config.get('sorted'), - item_label=item_label) - - content = self.list_in.format(table=table) - - if dry_run: - print(content) - return - - if not output: - output_dir = self.output_dir - if not output_dir: - print("Warning: Undefined output directory.") - print("\tUse source directory as output location.") - output_dir = self.base_dir - output = os.path.join(output_dir, - list_config.get('filename') + ".txt") - if not os.path.exists(os.path.dirname(output)): - os.makedirs(os.path.dirname(output)) - print("Writing the {0} list in:\n\t{1}".format(list_type, output)) - with open(output, 'w') as fout: - fout.write(content) - - -if __name__ == '__main__': - list_types = ['target-packages', 'host-packages', 'virtual-packages', 'deprecated'] - parser = ArgumentParser() - parser.add_argument("list_type", nargs="?", choices=list_types, - help="""\ -Generate the given list (generate all lists if unspecified)""") - parser.add_argument("-n", "--dry-run", dest="dry_run", action='store_true', - help="Output the generated list to stdout") - parser.add_argument("--output-target", dest="output_target", - help="Output target package file") - parser.add_argument("--output-host", dest="output_host", - help="Output host package file") - parser.add_argument("--output-virtual", dest="output_virtual", - help="Output virtual package file") - parser.add_argument("--output-deprecated", dest="output_deprecated", - help="Output deprecated file") - args = parser.parse_args() - lists = [args.list_type] if args.list_type else list_types - buildroot = Buildroot() - for list_name in lists: - output = getattr(args, "output_" + list_name.split("-", 1)[0]) - buildroot.print_list(list_name, dry_run=args.dry_run, output=output) diff --git a/support/scripts/generate-gitlab-ci-yml b/support/scripts/generate-gitlab-ci-yml new file mode 100755 index 0000000000..262a7649b2 --- /dev/null +++ b/support/scripts/generate-gitlab-ci-yml @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e +set -o pipefail + +input="${1}" + +cat "${input}" + +( + cd configs + LC_ALL=C ls -1 *_defconfig +) \ + | sed 's/$/: { extends: .defconfig }/' + +./support/testing/run-tests -l 2>&1 \ + | sed -r -e '/^test_run \((.*)\).*/!d; s//\1: { extends: .runtime_test }/' \ + | LC_ALL=C sort diff --git a/support/scripts/genimage.sh b/support/scripts/genimage.sh new file mode 100755 index 0000000000..2796e19eb7 --- /dev/null +++ b/support/scripts/genimage.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +die() { + cat <&2 +Error: $@ + +Usage: ${0} -c GENIMAGE_CONFIG_FILE +EOF + exit 1 +} + +# Parse arguments and put into argument list of the script +opts="$(getopt -n "${0##*/}" -o c: -- "$@")" || exit $? +eval set -- "$opts" + +GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp" + +while true ; do + case "$1" in + -c) + GENIMAGE_CFG="${2}"; + shift 2 ;; + --) # Discard all non-option parameters + shift 1; + break ;; + *) + die "unknown option '${1}'" ;; + esac +done + +[ -n "${GENIMAGE_CFG}" ] || die "Missing argument" + +# Pass an empty rootpath. genimage makes a full copy of the given rootpath to +# ${GENIMAGE_TMP}/root so passing TARGET_DIR would be a waste of time and disk +# space. We don't rely on genimage to build the rootfs image, just to insert a +# pre-built one in the disk image. + +trap 'rm -rf "${ROOTPATH_TMP}"' EXIT +ROOTPATH_TMP="$(mktemp -d)" + +rm -rf "${GENIMAGE_TMP}" + +genimage \ + --rootpath "${ROOTPATH_TMP}" \ + --tmppath "${GENIMAGE_TMP}" \ + --inputpath "${BINARIES_DIR}" \ + --outputpath "${BINARIES_DIR}" \ + --config "${GENIMAGE_CFG}" diff --git a/support/scripts/graph-build-time b/support/scripts/graph-build-time index 524c30d478..ba3cdad85b 100755 --- a/support/scripts/graph-build-time +++ b/support/scripts/graph-build-time @@ -49,25 +49,37 @@ # * argparse (by default in Python 2.7, requires python-argparse if # Python 2.6 is used) -import matplotlib -import numpy - -import matplotlib.pyplot as plt -import matplotlib.font_manager as fm -import csv -import argparse import sys -steps = [ 'extract', 'patch', 'configure', 'build', - 'install-target', 'install-staging', 'install-images', - 'install-host'] +try: + import matplotlib as mpl + import numpy +except ImportError: + sys.stderr.write("You need python-matplotlib and python-numpy to generate build graphs\n") + exit(1) + +# Use the Agg backend (which produces a PNG output, see +# http://matplotlib.org/faq/usage_faq.html#what-is-a-backend), +# otherwise an incorrect backend is used on some host machines). +# Note: matplotlib.use() must be called *before* matplotlib.pyplot. +mpl.use('Agg') -default_colors = ['#e60004', '#009836', '#2e1d86', '#ffed00', +import matplotlib.pyplot as plt # noqa: E402 +import matplotlib.font_manager as fm # noqa: E402 +import csv # noqa: E402 +import argparse # noqa: E402 + +steps = ['download', 'extract', 'patch', 'configure', 'build', + 'install-target', 'install-staging', 'install-images', + 'install-host'] + +default_colors = ['#8d02ff', '#e60004', '#009836', '#2e1d86', '#ffed00', '#0068b5', '#f28e00', '#940084', '#97c000'] -alternate_colors = ['#00e0e0', '#3f7f7f', '#ff0000', '#00c000', +alternate_colors = ['#ffbe0a', '#96bdff', '#3f7f7f', '#ff0000', '#00c000', '#0080ff', '#c000ff', '#00eeee', '#e0e000'] + class Package: def __init__(self, name): self.name = name @@ -93,6 +105,7 @@ class Package: return self.steps_duration[step] return 0 + # Generate an histogram of the time spent in each step of each # package. def pkg_histogram(data, output, order="build"): @@ -121,10 +134,10 @@ def pkg_histogram(data, output, order="build"): for i in range(0, len(vals)): b = plt.bar(ind+0.1, vals[i], width=0.8, color=colors[i], bottom=bottom, linewidth=0.25) legenditems.append(b[0]) - bottom = [ bottom[j] + vals[i][j] for j in range(0, len(vals[i])) ] + bottom = [bottom[j] + vals[i][j] for j in range(0, len(vals[i]))] # Draw the package names - plt.xticks(ind + .6, [ p.name for p in data ], rotation=-60, rotation_mode="anchor", fontsize=8, ha='left') + plt.xticks(ind + .6, [p.name for p in data], rotation=-60, rotation_mode="anchor", fontsize=8, ha='left') # Adjust size of graph depending on the number of packages # Ensure a minimal size twice as the default @@ -161,6 +174,7 @@ def pkg_histogram(data, output, order="build"): # Save graph plt.savefig(output) + # Generate a pie chart with the time spent building each package. def pkg_pie_time_per_package(data, output): # Compute total build duration @@ -173,7 +187,7 @@ def pkg_pie_time_per_package(data, output): labels = [] values = [] other_value = 0 - for p in data: + for p in sorted(data, key=lambda p: p.get_duration()): if p.get_duration() < (total * 0.01): other_value += p.get_duration() else: @@ -199,6 +213,7 @@ def pkg_pie_time_per_package(data, output): plt.title('Build time per package') plt.savefig(output) + # Generate a pie chart with a portion for the overall time spent in # each step for all packages. def pkg_pie_time_per_step(data, output): @@ -225,6 +240,7 @@ def pkg_pie_time_per_step(data, output): plt.title('Build time per step') plt.savefig(output) + # Parses the csv file passed on standard input and returns a list of # Package objects, filed with the duration of each step and the total # duration of the package. @@ -244,7 +260,7 @@ def read_data(input_file): return None for row in reader: - time = int(row[0].strip()) + time = float(row[0].strip()) state = row[1].strip() step = row[2].strip() pkg = row[3].strip() @@ -258,6 +274,7 @@ def read_data(input_file): return pkgs + parser = argparse.ArgumentParser(description='Draw build time graphs') parser.add_argument("--type", '-t', metavar="GRAPH_TYPE", help="Type of graph (histogram, pie-packages, pie-steps)") @@ -265,7 +282,7 @@ parser.add_argument("--order", '-O', metavar="GRAPH_ORDER", help="Ordering of packages: build or duration (for histogram only)") parser.add_argument("--alternate-colors", '-c', action="store_true", help="Use alternate colour-scheme") -parser.add_argument("--input", '-i', metavar="OUTPUT", +parser.add_argument("--input", '-i', metavar="INPUT", help="Input file (usually $(O)/build/build-time.log)") parser.add_argument("--output", '-o', metavar="OUTPUT", required=True, help="Output file (.pdf or .png extension)") diff --git a/support/scripts/graph-depends b/support/scripts/graph-depends index 1ecfedac7a..d42bebce9d 100755 --- a/support/scripts/graph-depends +++ b/support/scripts/graph-depends @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Usage (the graphviz package must be installed in your distribution) # ./support/scripts/graph-depends [-p package-name] > test.dot @@ -20,205 +20,76 @@ # configuration. # # Copyright (C) 2010-2013 Thomas Petazzoni +# Copyright (C) 2019 Yann E. MORIN +import logging import sys -import subprocess import argparse +from fnmatch import fnmatch + +import brpkgutil # Modes of operation: MODE_FULL = 1 # draw full dependency graph for all selected packages -MODE_PKG = 2 # draw dependency graph for a given package -mode = 0 - -# Limit drawing the dependency graph to this depth. 0 means 'no limit'. -max_depth = 0 - -# Whether to draw the transitive dependencies -transitive = True - -parser = argparse.ArgumentParser(description="Graph pacakges dependencies") -parser.add_argument("--package", '-p', metavar="PACKAGE", - help="Graph the dependencies of PACKAGE") -parser.add_argument("--depth", '-d', metavar="DEPTH", dest="depth", type=int, default=0, - help="Limit the dependency graph to DEPTH levels; 0 means no limit.") -parser.add_argument("--colours", "-c", metavar="COLOR_LIST", dest="colours", - default="lightblue,grey,gainsboro", - help="Comma-separated list of the three colours to use" \ - + " to draw the top-level package, the target" \ - + " packages, and the host packages, in this order." \ - + " Defaults to: 'lightblue,grey,gainsboro'") -parser.add_argument("--transitive", dest="transitive", action='store_true', - default=False) -parser.add_argument("--no-transitive", dest="transitive", action='store_false', - help="Draw (do not draw) transitive dependencies") -args = parser.parse_args() - -if args.package is None: - mode = MODE_FULL -else: - mode = MODE_PKG - rootpkg = args.package - -max_depth = args.depth - -transitive = args.transitive - -# Get the colours: we need exactly three colours, -# so no need not split more than 4 -# We'll let 'dot' validate the colours... -colours = args.colours.split(',',4) -if len(colours) != 3: - sys.stderr.write("Error: incorrect colour list '%s'\n" % args.colours) - sys.exit(1) -root_colour = colours[0] -target_colour = colours[1] -host_colour = colours[2] +MODE_PKG = 2 # draw dependency graph for a given package allpkgs = [] -# Execute the "make show-targets" command to get the list of the main -# Buildroot TARGETS and return it formatted as a Python list. This -# list is used as the starting point for full dependency graphs -def get_targets(): - sys.stderr.write("Getting targets\n") - cmd = ["make", "-s", "--no-print-directory", "show-targets"] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) - output = p.communicate()[0].strip() - if p.returncode != 0: - return None - if output == '': - return [] - return output.split(' ') - -# Execute the "make -show-depends" command to get the list of -# dependencies of a given list of packages, and return the list of -# dependencies formatted as a Python dictionary. -def get_depends(pkgs): - sys.stderr.write("Getting dependencies for %s\n" % pkgs) - cmd = ["make", "-s", "--no-print-directory" ] - for pkg in pkgs: - cmd.append("%s-show-depends" % pkg) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) - output = p.communicate()[0] - if p.returncode != 0: - sys.stderr.write("Error getting dependencies %s\n" % pkgs) - sys.exit(1) - output = output.split("\n") - if len(output) != len(pkgs) + 1: - sys.stderr.write("Error getting dependencies\n") - sys.exit(1) - deps = {} - for i in range(0, len(pkgs)): - pkg = pkgs[i] - pkg_deps = output[i].split(" ") - if pkg_deps == ['']: - deps[pkg] = [] - else: - deps[pkg] = pkg_deps - return deps - -# Recursive function that builds the tree of dependencies for a given -# list of packages. The dependencies are built in a list called -# 'dependencies', which contains tuples of the form (pkg1 -> -# pkg2_on_which_pkg1_depends, pkg3 -> pkg4_on_which_pkg3_depends) and -# the function finally returns this list. -def get_all_depends(pkgs): - dependencies = [] - # Filter the packages for which we already have the dependencies - filtered_pkgs = [] - for pkg in pkgs: - if pkg in allpkgs: - continue - filtered_pkgs.append(pkg) - allpkgs.append(pkg) - - if len(filtered_pkgs) == 0: - return [] +# The Graphviz "dot" utility doesn't like dashes in node names. So for +# node names, we strip all dashes. Also, nodes can't start with a number, +# so we prepend an underscore. +def pkg_node_name(pkg): + return "_" + pkg.replace("-", "") - depends = get_depends(filtered_pkgs) - deps = set() - for pkg in filtered_pkgs: - pkg_deps = depends[pkg] +# Basic cache for the results of the is_dep() function, in order to +# optimize the execution time. The cache is a dict of dict of boolean +# values. The key to the primary dict is "pkg", and the key of the +# sub-dicts is "pkg2". +is_dep_cache = {} - # This package has no dependency. - if pkg_deps == []: - continue - # Add dependencies to the list of dependencies - for dep in pkg_deps: - dependencies.append((pkg, dep)) - deps.add(dep) +def is_dep_cache_insert(pkg, pkg2, val): + try: + is_dep_cache[pkg].update({pkg2: val}) + except KeyError: + is_dep_cache[pkg] = {pkg2: val} - if len(deps) != 0: - newdeps = get_all_depends(deps) - if newdeps is not None: - dependencies += newdeps - return dependencies +# Retrieves from the cache whether pkg2 is a transitive dependency +# of pkg. +# Note: raises a KeyError exception if the dependency is not known. +def is_dep_cache_lookup(pkg, pkg2): + return is_dep_cache[pkg][pkg2] -# The Graphviz "dot" utility doesn't like dashes in node names. So for -# node names, we strip all dashes. -def pkg_node_name(pkg): - return pkg.replace("-","") - -TARGET_EXCEPTIONS = [ - "target-generic-securetty", - "target-generic-issue", - "target-generic-getty-busybox", - "target-generic-do-remount-rw", - "target-generic-dont-remount-rw", - "target-finalize", - "erase-fakeroots", - "target-generic-hostname", - "target-root-passwd", - "target-post-image", - "target-purgelocales", -] - -# In full mode, start with the result of get_targets() to get the main -# targets and then use get_all_depends() for all targets -if mode == MODE_FULL: - targets = get_targets() - dependencies = [] - allpkgs.append('all') - filtered_targets = [] - for tg in targets: - # Skip uninteresting targets - if tg in TARGET_EXCEPTIONS: - continue - dependencies.append(('all', tg)) - filtered_targets.append(tg) - deps = get_all_depends(filtered_targets) - if deps is not None: - dependencies += deps - rootpkg = 'all' - -# In pkg mode, start directly with get_all_depends() on the requested -# package -elif mode == MODE_PKG: - dependencies = get_all_depends([rootpkg]) - -# Make the dependencies a dictionnary { 'pkg':[dep1, dep2, ...] } -dict_deps = {} -for dep in dependencies: - if dep[0] not in dict_deps: - dict_deps[dep[0]] = [] - dict_deps[dep[0]].append(dep[1]) # This function return True if pkg is a dependency (direct or # transitive) of pkg2, dependencies being listed in the deps # dictionary. Returns False otherwise. -def is_dep(pkg,pkg2,deps): - if pkg2 in deps: +# This is the un-cached version. +def is_dep_uncached(pkg, pkg2, deps): + try: for p in deps[pkg2]: if pkg == p: return True - if is_dep(pkg,p,deps): + if is_dep(pkg, p, deps): return True + except KeyError: + pass return False + +# See is_dep_uncached() above; this is the cached version. +def is_dep(pkg, pkg2, deps): + try: + return is_dep_cache_lookup(pkg, pkg2) + except KeyError: + val = is_dep_uncached(pkg, pkg2, deps) + is_dep_cache_insert(pkg, pkg2, val) + return val + + # This function eliminates transitive dependencies; for example, given # these dependency chain: A->{B,C} and B->{C}, the A->{C} dependency is # already covered by B->{C}, so C is a transitive dependency of A, via B. @@ -227,73 +98,260 @@ def is_dep(pkg,pkg2,deps): # - if d[i] is a dependency of any of the other dependencies d[j] # - do not keep d[i] # - otherwise keep d[i] -def remove_transitive_deps(pkg,deps): +def remove_transitive_deps(pkg, deps): d = deps[pkg] new_d = [] for i in range(len(d)): keep_me = True for j in range(len(d)): - if j==i: + if j == i: continue - if is_dep(d[i],d[j],deps): + if is_dep(d[i], d[j], deps): keep_me = False if keep_me: new_d.append(d[i]) return new_d -# This function removes the dependency on the 'toolchain' package -def remove_toolchain_deps(pkg,deps): - return [p for p in deps[pkg] if not p == 'toolchain'] + +# List of dependencies that all/many packages have, and that we want +# to trim when generating the dependency graph. +MANDATORY_DEPS = ['toolchain', 'skeleton', 'host-skeleton', 'host-tar', 'host-gzip', 'host-ccache'] + + +# This function removes the dependency on some 'mandatory' package, like the +# 'toolchain' package, or the 'skeleton' package +def remove_mandatory_deps(pkg, deps): + return [p for p in deps[pkg] if p not in MANDATORY_DEPS] + + +# This function returns all dependencies of pkg that are part of the +# mandatory dependencies: +def get_mandatory_deps(pkg, deps): + return [p for p in deps[pkg] if p in MANDATORY_DEPS] + + +# This function will check that there is no loop in the dependency chain +# As a side effect, it builds up the dependency cache. +def check_circular_deps(deps): + def recurse(pkg): + if pkg not in list(deps.keys()): + return + if pkg in not_loop: + return + not_loop.append(pkg) + chain.append(pkg) + for p in deps[pkg]: + if p in chain: + logging.warning("\nRecursion detected for : %s" % (p)) + while True: + _p = chain.pop() + logging.warning("which is a dependency of: %s" % (_p)) + if p == _p: + sys.exit(1) + recurse(p) + chain.pop() + + not_loop = [] + chain = [] + for pkg in list(deps.keys()): + recurse(pkg) + # This functions trims down the dependency list of all packages. # It applies in sequence all the dependency-elimination methods. -def remove_extra_deps(deps): +def remove_extra_deps(deps, rootpkg, transitive, arrow_dir): + # For the direct dependencies, find and eliminate mandatory + # deps, and add them to the root package. Don't do it for a + # reverse graph, because mandatory deps are only direct deps. + if arrow_dir == "forward": + for pkg in list(deps.keys()): + if not pkg == rootpkg: + for d in get_mandatory_deps(pkg, deps): + if d not in deps[rootpkg]: + deps[rootpkg].append(d) + deps[pkg] = remove_mandatory_deps(pkg, deps) for pkg in list(deps.keys()): - if not pkg == 'all': - deps[pkg] = remove_toolchain_deps(pkg,deps) - for pkg in list(deps.keys()): - if not transitive or pkg == 'all': - deps[pkg] = remove_transitive_deps(pkg,deps) + if not transitive or pkg == rootpkg: + deps[pkg] = remove_transitive_deps(pkg, deps) return deps -dict_deps = remove_extra_deps(dict_deps) # Print the attributes of a node: label and fill-color -def print_attrs(pkg): +def print_attrs(outfile, pkg, pkg_type, pkg_version, depth, colors): name = pkg_node_name(pkg) if pkg == 'all': label = 'ALL' else: label = pkg - if pkg == 'all' or (mode == MODE_PKG and pkg == rootpkg): - color = root_colour + if depth == 0: + color = colors[0] else: - if pkg.startswith('host') \ - or pkg.startswith('toolchain') \ - or pkg.startswith('rootfs'): - color = host_colour + if pkg_type == "host": + color = colors[2] else: - color = target_colour - print("%s [label = \"%s\"]" % (name, label)) - print("%s [color=%s,style=filled]" % (name, color)) + color = colors[1] + if pkg_version == "virtual": + outfile.write("%s [label = <%s>]\n" % (name, label)) + else: + outfile.write("%s [label = \"%s\"]\n" % (name, label)) + outfile.write("%s [color=%s,style=filled]\n" % (name, color)) + + +done_deps = [] + # Print the dependency graph of a package -def print_pkg_deps(depth, pkg): +def print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list, + arrow_dir, draw_graph, depth, max_depth, pkg, colors): if pkg in done_deps: return done_deps.append(pkg) - print_attrs(pkg) + if draw_graph: + print_attrs(outfile, pkg, dict_types[pkg], dict_versions[pkg], depth, colors) + elif depth != 0: + outfile.write("%s " % pkg) if pkg not in dict_deps: return + for p in stop_list: + if fnmatch(pkg, p): + return + if dict_versions[pkg] == "virtual" and "virtual" in stop_list: + return + if dict_types[pkg] == "host" and "host" in stop_list: + return if max_depth == 0 or depth < max_depth: for d in dict_deps[pkg]: - print("%s -> %s" % (pkg_node_name(pkg), pkg_node_name(d))) - print_pkg_deps(depth+1, d) + if dict_versions[d] == "virtual" and "virtual" in exclude_list: + continue + if dict_types[d] == "host" and "host" in exclude_list: + continue + add = True + for p in exclude_list: + if fnmatch(d, p): + add = False + break + if add: + if draw_graph: + outfile.write("%s -> %s [dir=%s]\n" % (pkg_node_name(pkg), pkg_node_name(d), arrow_dir)) + print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list, + arrow_dir, draw_graph, depth + 1, max_depth, d, colors) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Graph packages dependencies") + parser.add_argument("--check-only", "-C", dest="check_only", action="store_true", default=False, + help="Only do the dependency checks (circular deps...)") + parser.add_argument("--outfile", "-o", metavar="OUT_FILE", dest="outfile", + help="File in which to generate the dot representation") + parser.add_argument("--package", '-p', metavar="PACKAGE", + help="Graph the dependencies of PACKAGE") + parser.add_argument("--depth", '-d', metavar="DEPTH", dest="depth", type=int, default=0, + help="Limit the dependency graph to DEPTH levels; 0 means no limit.") + parser.add_argument("--stop-on", "-s", metavar="PACKAGE", dest="stop_list", action="append", + help="Do not graph past this package (can be given multiple times)." + + " Can be a package name or a glob, " + + " 'virtual' to stop on virtual packages, or " + + "'host' to stop on host packages.") + parser.add_argument("--exclude", "-x", metavar="PACKAGE", dest="exclude_list", action="append", + help="Like --stop-on, but do not add PACKAGE to the graph.") + parser.add_argument("--exclude-mandatory", "-X", action="store_true", + help="Like if -x was passed for all mandatory dependencies.") + parser.add_argument("--colors", "-c", metavar="COLOR_LIST", dest="colors", + default="lightblue,grey,gainsboro", + help="Comma-separated list of the three colors to use" + + " to draw the top-level package, the target" + + " packages, and the host packages, in this order." + + " Defaults to: 'lightblue,grey,gainsboro'") + parser.add_argument("--transitive", dest="transitive", action='store_true', + default=False) + parser.add_argument("--no-transitive", dest="transitive", action='store_false', + help="Draw (do not draw) transitive dependencies") + parser.add_argument("--direct", dest="direct", action='store_true', default=True, + help="Draw direct dependencies (the default)") + parser.add_argument("--reverse", dest="direct", action='store_false', + help="Draw reverse dependencies") + parser.add_argument("--quiet", '-q', dest="quiet", action='store_true', + help="Quiet") + parser.add_argument("--flat-list", '-f', dest="flat_list", action='store_true', default=False, + help="Do not draw graph, just print a flat list") + return parser.parse_args() + + +def main(): + args = parse_args() + + check_only = args.check_only + + logging.basicConfig(stream=sys.stderr, format='%(message)s', + level=logging.WARNING if args.quiet else logging.INFO) + + if args.outfile is None: + outfile = sys.stdout + else: + if check_only: + logging.error("don't specify outfile and check-only at the same time") + sys.exit(1) + outfile = open(args.outfile, "w") + + if args.package is None: + mode = MODE_FULL + rootpkg = 'all' + else: + mode = MODE_PKG + rootpkg = args.package -# Start printing the graph data -print("digraph G {") + if args.stop_list is None: + stop_list = [] + else: + stop_list = args.stop_list + + if args.exclude_list is None: + exclude_list = [] + else: + exclude_list = args.exclude_list + + if args.exclude_mandatory: + exclude_list += MANDATORY_DEPS + + if args.direct: + arrow_dir = "forward" + else: + if mode == MODE_FULL: + logging.error("--reverse needs a package") + sys.exit(1) + arrow_dir = "back" + + draw_graph = not args.flat_list + + # Get the colors: we need exactly three colors, + # so no need not split more than 4 + # We'll let 'dot' validate the colors... + colors = args.colors.split(',', 4) + if len(colors) != 3: + logging.error("Error: incorrect color list '%s'" % args.colors) + sys.exit(1) + + deps, rdeps, dict_types, dict_versions = brpkgutil.get_dependency_tree() + dict_deps = deps if args.direct else rdeps + + check_circular_deps(dict_deps) + if check_only: + sys.exit(0) + + dict_deps = remove_extra_deps(dict_deps, rootpkg, args.transitive, arrow_dir) + + # Start printing the graph data + if draw_graph: + outfile.write("digraph G {\n") + + print_pkg_deps(outfile, dict_deps, dict_types, dict_versions, stop_list, exclude_list, + arrow_dir, draw_graph, 0, args.depth, rootpkg, colors) + + if draw_graph: + outfile.write("}\n") + else: + outfile.write("\n") -done_deps = [] -print_pkg_deps(0, rootpkg) -print("}") +if __name__ == "__main__": + sys.exit(main()) diff --git a/support/scripts/hardlink-or-copy b/support/scripts/hardlink-or-copy new file mode 100755 index 0000000000..a052318ffa --- /dev/null +++ b/support/scripts/hardlink-or-copy @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Try to hardlink a file into a directory, fallback to copy on failure. +# +# Hardlink-or-copy the source file in the first argument into the +# destination directory in the second argument, using the basename in +# the third argument as basename for the destination file. If the third +# argument is missing, use the basename of the source file as basename +# for the destination file. +# +# In either case, remove the destination prior to doing the +# hardlink-or-copy. +# +# Note that this is NOT an atomic operation. + +set -e + +main() { + local src_file="${1}" + local dst_dir="${2}" + local dst_file="${3}" + + if [ -n "${dst_file}" ]; then + dst_file="${dst_dir}/${dst_file}" + else + dst_file="${dst_dir}/${src_file##*/}" + fi + + mkdir -p "${dst_dir}" + rm -f "${dst_file}" + ln -f "${src_file}" "${dst_file}" 2>/dev/null \ + || cp -f "${src_file}" "${dst_file}" +} + +main "${@}" diff --git a/support/scripts/kconfiglib.py b/support/scripts/kconfiglib.py deleted file mode 100644 index e40947ce21..0000000000 --- a/support/scripts/kconfiglib.py +++ /dev/null @@ -1,3772 +0,0 @@ -# This is Kconfiglib, a Python library for scripting, debugging, and extracting -# information from Kconfig-based configuration systems. To view the -# documentation, run -# -# $ pydoc kconfiglib -# -# or, if you prefer HTML, -# -# $ pydoc -w kconfiglib -# -# The examples/ subdirectory contains examples, to be run with e.g. -# -# $ make scriptconfig SCRIPT=Kconfiglib/examples/print_tree.py -# -# Look in testsuite.py for the test suite. - -""" -Kconfiglib is a Python library for scripting and extracting information from -Kconfig-based configuration systems. Features include the following: - - - Symbol values and properties can be looked up and values assigned - programmatically. - - .config files can be read and written. - - Expressions can be evaluated in the context of a Kconfig configuration. - - Relations between symbols can be quickly determined, such as finding all - symbols that reference a particular symbol. - - Highly compatible with the scripts/kconfig/*conf utilities. The test suite - automatically compares outputs between Kconfiglib and the C implementation - for a large number of cases. - -For the Linux kernel, scripts are run using - - $ make scriptconfig SCRIPT= [SCRIPT_ARG=] - -Running scripts via the 'scriptconfig' target ensures that required environment -variables (SRCARCH, ARCH, srctree, KERNELVERSION, etc.) are set up correctly. -Alternative architectures can be specified like for other 'make *config' -targets: - - $ make scriptconfig ARCH=mips SCRIPT= [SCRIPT_ARG=] - -The script will receive the name of the Kconfig file to load in sys.argv[1]. -(As of Linux 3.7.0-rc8 this is always "Kconfig" from the kernel top-level -directory.) If an argument is provided with SCRIPT_ARG, it will appear in -sys.argv[2]. - -To get an interactive Python prompt with Kconfiglib preloaded and a Config -object 'c' created, use - - $ make iscriptconfig [ARCH=] - -Kconfiglib requires Python 2. For (i)scriptconfig the command to run the Python -interpreter can be passed in the environment variable PYTHONCMD (defaults to -'python'; PyPy works too and is a bit faster). - -Look in the examples/ subdirectory for examples, which can be run with e.g. - - $ make scriptconfig SCRIPT=Kconfiglib/examples/print_tree.py - -or - - $ make scriptconfig SCRIPT=Kconfiglib/examples/help_grep.py SCRIPT_ARG="kernel" - -Look in testsuite.py for the test suite. - -Credits: Written by Ulf "Ulfalizer" Magnusson - -Send bug reports, suggestions and other feedback to kconfiglib@gmail.com . -Don't wrestle with internal APIs. Tell me what you need and I might add it in a -safe way as a client API instead.""" - -# If you have Psyco installed (32-bit installations, Python <= 2.6 only), -# setting this to True (right here, not at runtime) might give a nice speedup. -# (22% faster for parsing arch/x86/Kconfig and 58% faster for evaluating all -# symbols in it without a .config on my Core Duo.) -use_psyco = False - -import os -import re -import string -import sys - -class Config(): - - """Represents a Kconfig configuration, e.g. for i386 or ARM. This is the - set of symbols and other items appearing in the configuration together with - their values. Creating any number of Config objects -- including for - different architectures -- is safe; Kconfiglib has no global state.""" - - # - # Public interface - # - - def __init__(self, - filename = "Kconfig", - base_dir = "$srctree", - print_warnings = True, - print_undef_assign = False): - """Creates a new Config object, representing a Kconfig configuration. - Raises Kconfig_Syntax_Error on syntax errors. - - filename (default: "Kconfig") -- The base Kconfig file of the - configuration. For the Linux kernel, this should usually be be - "Kconfig" from the top-level directory, as environment - variables will make sure the right Kconfig is included from - there (usually arch//Kconfig). If you are using - kconfiglib via 'make scriptconfig' the filename of the - correct Kconfig will be in sys.argv[1]. - - base_dir (default: "$srctree") -- The base directory relative to which - 'source' statements within Kconfig files will work. For the - Linux kernel this should be the top-level directory of the - kernel tree. $-references to environment variables will be - expanded. - - The environment variable 'srctree' is set by the Linux makefiles - to the top-level kernel directory. A default of "." would not - work if an alternative build directory is used. - - print_warnings (default: True) -- Set to True if warnings related to - this configuration should be printed to stderr. This can - be changed later with Config.set_print_warnings(). It is - provided as a constructor argument since warnings might - be generated during parsing. - - print_undef_assign (default: False) -- Set to True if informational - messages related to assignments to undefined symbols - should be printed to stderr for this configuration. - Can be changed later with - Config.set_print_undef_assign().""" - - # The set of all symbols, indexed by name (a string) - self.syms = {} - - # The set of all defined symbols in the configuration in the order they - # appear in the Kconfig files. This excludes the special symbols n, m, - # and y as well as symbols that are referenced but never defined. - self.kconfig_syms = [] - - # The set of all named choices (yes, choices can have names), indexed - # by name (a string) - self.named_choices = {} - - def register_special_symbol(type, name, value): - sym = Symbol() - sym.is_special_ = True - sym.is_defined_ = True - sym.config = self - sym.name = name - sym.type = type - sym.cached_value = value - self.syms[name] = sym - return sym - - # The special symbols n, m and y, used as shorthand for "n", "m" and - # "y" - self.n = register_special_symbol(TRISTATE, "n", "n") - self.m = register_special_symbol(TRISTATE, "m", "m") - self.y = register_special_symbol(TRISTATE, "y", "y") - - # DEFCONFIG_LIST uses this - register_special_symbol(STRING, "UNAME_RELEASE", os.uname()[2]) - - # The symbol with "option defconfig_list" set, containing a list of - # default .config files - self.defconfig_sym = None - - # See Symbol.get_(src)arch() - self.arch = os.environ.get("ARCH") - self.srcarch = os.environ.get("SRCARCH") - - # See Config.__init__(). We need this for get_defconfig_filename(). - self.srctree = os.environ.get("srctree") - if self.srctree is None: - self.srctree = "." - - self.filename = filename - self.base_dir = _strip_trailing_slash(os.path.expandvars(base_dir)) - - # The 'mainmenu' text - self.mainmenu_text = None - - # The filename of the most recently loaded .config file - self.config_filename = None - - # The textual header of the most recently loaded .config, uncommented - self.config_header = None - - self.print_warnings = print_warnings - self.print_undef_assign = print_undef_assign - - # Lists containing all choices, menus and comments in the configuration - - self.choices = [] - self.menus = [] - self.comments = [] - - # For parsing routines that stop when finding a line belonging to a - # different construct, these holds that line and the tokenized version - # of that line. The purpose is to avoid having to re-tokenize the line, - # which is inefficient and causes problems when recording references to - # symbols. - self.end_line = None - self.end_line_tokens = None - - # See the comment in _parse_expr(). - self.parse_expr_cur_sym_or_choice = None - self.parse_expr_line = None - self.parse_expr_filename = None - self.parse_expr_linenr = None - self.parse_expr_transform_m = None - - # Parse the Kconfig files - self.top_block = self._parse_file(filename, None, None, None) - - # Build Symbol.dep for all symbols - self._build_dep() - - def load_config(self, filename, replace = True): - """Loads symbol values from a file in the familiar .config format. - Equivalent to calling Symbol.set_user_value() to set each of the - values. - - filename -- The .config file to load. $-references to environment - variables will be expanded. For scripts to work even - when an alternative build directory is used with the - Linux kernel, you need to refer to the top-level kernel - directory with "$srctree". - - replace (default: True) -- True if the configuration should replace - the old configuration; False if it should add to it.""" - - def warn_override(filename, linenr, name, old_user_val, new_user_val): - self._warn("overriding the value of {0}. " - 'Old value: "{1}", new value: "{2}".' - .format(name, old_user_val, new_user_val), - filename, - linenr) - - filename = os.path.expandvars(filename) - - # Put this first so that a missing file doesn't screw up our state - line_feeder = _FileFeed(_get_lines(filename), filename) - - self.config_filename = filename - - # Invalidate everything. This is usually faster than finding the - # minimal set of symbols that needs to be invalidated, as nearly all - # symbols will tend to be affected anyway. - if replace: - self.unset_user_values() - else: - self._invalidate_all() - - # Read header - - self.config_header = None - - def is_header_line(line): - return line.startswith("#") and \ - not unset_re.match(line) - - first_line = line_feeder.get_next() - - if first_line is None: - return - - if not is_header_line(first_line): - line_feeder.go_back() - else: - self.config_header = first_line[1:] - - # Read remaining header lines - while 1: - line = line_feeder.get_next() - - if line is None: - break - - if not is_header_line(line): - line_feeder.go_back() - break - - self.config_header += line[1:] - - # Remove trailing newline - if self.config_header.endswith("\n"): - self.config_header = self.config_header[:-1] - - # Read assignments - - filename = line_feeder.get_filename() - - while 1: - line = line_feeder.get_next() - if line is None: - return - - linenr = line_feeder.get_linenr() - - line = line.strip() - - set_re_match = set_re.match(line) - if set_re_match: - name, val = set_re_match.groups() - # The unescaping producedure below should be safe since " can - # only appear as \" inside the string - val = _strip_quotes(val, line, filename, linenr)\ - .replace('\\"', '"').replace("\\\\", "\\") - if name in self.syms: - sym = self.syms[name] - - old_user_val = sym.user_val - if old_user_val is not None: - warn_override(filename, linenr, name, old_user_val, val) - - if sym.is_choice_symbol_: - user_mode = sym.parent.user_mode - if user_mode is not None and user_mode != val: - self._warn("assignment to {0} changes mode of containing " - 'choice from "{1}" to "{2}".' - .format(name, val, user_mode), - filename, - linenr) - - sym._set_user_value_no_invalidate(val, True) - - else: - self._undef_assign('attempt to assign the value "{0}" to the ' - "undefined symbol {1}." - .format(val, name), - filename, - linenr) - - else: - unset_re_match = unset_re.match(line) - if unset_re_match: - name = unset_re_match.group(1) - if name in self.syms: - sym = self.syms[name] - - old_user_val = sym.user_val - if old_user_val is not None: - warn_override(filename, linenr, name, old_user_val, "n") - - sym._set_user_value_no_invalidate("n", True) - - def write_config(self, filename, header = None): - """Writes out symbol values in the familiar .config format. - - filename -- The filename under which to save the configuration. - - header (default: None) -- A textual header that will appear at the - beginning of the file, with each line commented out - automatically. None means no header.""" - - # already_written is set when _make_conf() is called on a symbol, so - # that symbols defined in multiple locations only get one entry in the - # .config. We need to reset it prior to writing out a new .config. - for sym in self.syms.itervalues(): - sym.already_written = False - - with open(filename, "w") as f: - # Write header - if header is not None: - f.write(_comment(header)) - f.write("\n") - - # Write configuration. - # (You'd think passing a list around to all the nodes and appending - # to it to avoid copying would be faster, but it's actually a lot - # slower with PyPy, and about as fast with Python. Passing the file - # around is slower too.) - f.write("\n".join(self.top_block._make_conf())) - f.write("\n") - - def get_kconfig_filename(self): - """Returns the name of the (base) kconfig file this configuration was - loaded from.""" - return self.filename - - def get_arch(self): - """Returns the value the environment variable ARCH had at the time the - Config instance was created, or None if ARCH was not set. For the - kernel, this corresponds to the architecture being built for, with - values such as "i386" or "mips".""" - return self.arch - - def get_srcarch(self): - """Returns the value the environment variable SRCARCH had at the time - the Config instance was created, or None if SRCARCH was not set. For - the kernel, this corresponds to the arch/ subdirectory containing - architecture-specific source code.""" - return self.srcarch - - def get_srctree(self): - """Returns the value the environment variable srctree had at the time - the Config instance was created, or None if srctree was not defined. - This variable points to the source directory and is used when building - in a separate directory.""" - return self.srctree - - def get_config_filename(self): - """Returns the name of the most recently loaded configuration file, or - None if no configuration has been loaded.""" - return self.config_filename - - def get_mainmenu_text(self): - """Returns the text of the 'mainmenu' statement (with $-references to - symbols replaced by symbol values), or None if the configuration has no - 'mainmenu' statement.""" - return None if self.mainmenu_text is None else \ - self._expand_sym_refs(self.mainmenu_text) - - def get_defconfig_filename(self): - """Returns the name of the defconfig file, which is the first existing - file in the list given in a symbol having 'option defconfig_list' set. - $-references to symbols will be expanded ("$FOO bar" -> "foo bar" if - FOO has the value "foo"). Returns None in case of no defconfig file. - Setting 'option defconfig_list' on multiple symbols currently results - in undefined behavior. - - If the environment variable 'srctree' was set when the Config was - created, get_defconfig_filename() will first look relative to that - directory before looking in the current directory; see - Config.__init__().""" - - if self.defconfig_sym is None: - return None - - for (filename, cond_expr) in self.defconfig_sym.def_exprs: - if self._eval_expr(cond_expr) == "y": - filename = self._expand_sym_refs(filename) - - # We first look in $srctree. os.path.join() won't work here as - # an absolute path in filename would override $srctree. - srctree_filename = os.path.normpath(self.srctree + "/" + filename) - if os.path.exists(srctree_filename): - return srctree_filename - - if os.path.exists(filename): - return filename - - return None - - def get_symbol(self, name): - """Returns the symbol with name 'name', or None if no such symbol - appears in the configuration. An alternative shorthand is conf[name], - where conf is a Config instance, though that will instead raise - KeyError if the symbol does not exist.""" - return self.syms.get(name) - - def get_top_level_items(self): - """Returns a list containing the items (symbols, menus, choice - statements and comments) at the top level of the configuration -- that - is, all items that do not appear within a menu or choice. The items - appear in the same order as within the configuration.""" - return self.top_block.get_items() - - def get_symbols(self, all_symbols = True): - """Returns a list of symbols from the configuration. An alternative for - iterating over all defined symbols (in the order of definition) is - - for sym in config: - ... - - which relies on Config implementing __iter__() and is equivalent to - - for sym in config.get_symbols(False): - ... - - all_symbols (default: True) -- If True, all symbols - including special - and undefined symbols - will be included in the result, in - an undefined order. If False, only symbols actually defined - and not merely referred to in the configuration will be - included in the result, and will appear in the order that - they are defined within the Kconfig configuration files.""" - return self.syms.values() if all_symbols else self.kconfig_syms - - def get_choices(self): - """Returns a list containing all choice statements in the - configuration, in the order they appear in the Kconfig files.""" - return self.choices - - def get_menus(self): - """Returns a list containing all menus in the configuration, in the - order they appear in the Kconfig files.""" - return self.menus - - def get_comments(self): - """Returns a list containing all comments in the configuration, in the - order they appear in the Kconfig files.""" - return self.comments - - def eval(self, s): - """Returns the value of the expression 's' -- where 's' is represented - as a string -- in the context of the configuration. Raises - Kconfig_Syntax_Error if syntax errors are detected in 's'. - - For example, if FOO and BAR are tristate symbols at least one of which - has the value "y", then config.eval("y && (FOO || BAR)") => "y" - - This functions always yields a tristate value. To get the value of - non-bool, non-tristate symbols, use Symbol.get_value(). - - The result of this function is consistent with how evaluation works for - conditional expressions in the configuration as well as in the C - implementation. "m" and m are rewritten as '"m" && MODULES' and 'm && - MODULES', respectively, and a result of "m" will get promoted to "y" if - we're running without modules.""" - return self._eval_expr(self._parse_expr(self._tokenize(s, True), # Feed - None, # Current symbol or choice - s)) # line - - def get_config_header(self): - """Returns the (uncommented) textual header of the .config file most - recently loaded with load_config(). Returns None if no .config file has - been loaded or if the most recently loaded .config file has no header. - The header comprises all lines up to but not including the first line - that either - - 1. Does not start with "#" - 2. Has the form "# CONFIG_FOO is not set." - """ - return self.config_header - - def get_base_dir(self): - """Returns the base directory relative to which 'source' statements - will work, passed as an argument to Config.__init__().""" - return self.base_dir - - def set_print_warnings(self, print_warnings): - """Determines whether warnings related to this configuration (for - things like attempting to assign illegal values to symbols with - Symbol.set_user_value()) should be printed to stderr. - - print_warnings -- True if warnings should be - printed, otherwise False.""" - self.print_warnings = print_warnings - - def set_print_undef_assign(self, print_undef_assign): - """Determines whether informational messages related to assignments to - undefined symbols should be printed to stderr for this configuration. - - print_undef_assign -- If True, such messages will be printed.""" - self.print_undef_assign = print_undef_assign - - def __getitem__(self, key): - """Returns the symbol with name 'name'. Raises KeyError if the symbol - does not appear in the configuration.""" - return self.syms[key] - - def __iter__(self): - """Convenience function for iterating over the set of all defined - symbols in the configuration, used like - - for sym in conf: - ... - - The iteration happens in the order of definition within the Kconfig - configuration files. Symbols only referred to but not defined will not - be included, nor will the special symbols n, m, and y. If you want to - include such symbols as well, see config.get_symbols().""" - return iter(self.kconfig_syms) - - def unset_user_values(self): - """Resets the values of all symbols, as if Config.load_config() or - Symbol.set_user_value() had never been called.""" - for sym in self.syms.itervalues(): - sym._unset_user_value_no_recursive_invalidate() - - def __str__(self): - """Returns a string containing various information about the Config.""" - return _sep_lines("Configuration", - "File : " + self.filename, - "Base directory : " + self.base_dir, - "Value of $ARCH at creation time : " + - ("(not set)" if self.arch is None else self.arch), - "Value of $SRCARCH at creation time : " + - ("(not set)" if self.srcarch is None else self.srcarch), - "Source tree (derived from $srctree;", - "defaults to '.' if $srctree isn't set) : " + self.srctree, - "Most recently loaded .config : " + - ("(no .config loaded)" if self.config_filename is None else - self.config_filename), - "Print warnings : " + - bool_str[self.print_warnings], - "Print assignments to undefined symbols : " + - bool_str[self.print_undef_assign]) - - - # - # Private methods - # - - def _invalidate_all(self): - for sym in self.syms.itervalues(): - sym._invalidate() - - def _tokenize(self, - s, - for_eval = False, - filename = None, - linenr = None): - """Returns a _Feed instance containing tokens derived from the string - 's'. Registers any new symbols encountered (via _sym_lookup()). - - (I experimented with a pure regular expression implementation, but it - came out slower, less readable, and wouldn't have been as flexible.) - - for_eval -- True when parsing an expression for a call to - Config.eval(), in which case we should not treat the first - token specially nor register new symbols.""" - s = s.lstrip() - if s == "" or s[0] == "#": - return _Feed([]) - - if for_eval: - i = 0 # The current index in the string being tokenized - previous = None # The previous token seen - tokens = [] - else: - # The initial word on a line is parsed specially. Let - # command_chars = [A-Za-z0-9_]. Then - # - leading non-command_chars characters on the line are ignored, and - # - the first token consists the following one or more command_chars - # characters. - # This is why things like "----help--" are accepted. - - initial_token_match = initial_token_re.match(s) - if initial_token_match is None: - return _Feed([]) - # The current index in the string being tokenized - i = initial_token_match.end() - - keyword = keywords.get(initial_token_match.group(1)) - if keyword is None: - # We expect a keyword as the first token - _tokenization_error(s, len(s), filename, linenr) - if keyword == T_HELP: - # Avoid junk after "help", e.g. "---", being registered as a - # symbol - return _Feed([T_HELP]) - tokens = [keyword] - previous = keyword - - # _tokenize() is a hotspot during parsing, and this speeds things up a - # bit - strlen = len(s) - append = tokens.append - - # Main tokenization loop. (Handles tokens past the first one.) - while i < strlen: - # Test for an identifier/keyword preceded by whitespace first; this - # is the most common case. - id_keyword_match = id_keyword_re.match(s, i) - if id_keyword_match: - # We have an identifier or keyword. The above also stripped any - # whitespace for us. - name = id_keyword_match.group(1) - # Jump past it - i = id_keyword_match.end() - - # Keyword? - keyword = keywords.get(name) - if keyword is not None: - append(keyword) - # What would ordinarily be considered a name is treated as a - # string after certain tokens. - elif previous in string_lex: - append(name) - else: - # We're dealing with a symbol. _sym_lookup() will take care - # of allocating a new Symbol instance if it's the first - # time we see it. - sym = self._sym_lookup(name, not for_eval) - - if previous == T_CONFIG or previous == T_MENUCONFIG: - # If the previous token is T_(MENU)CONFIG - # ("(menu)config"), we're tokenizing the first line of - # a symbol definition, and should remember this as a - # location where the symbol is defined. - sym.def_locations.append((filename, linenr)) - else: - # Otherwise, it's a reference to the symbol - sym.ref_locations.append((filename, linenr)) - - append(sym) - - else: - # This restrips whitespace that could have been stripped in the - # regex above, but it's worth it since identifiers/keywords are - # more common - s = s[i:].lstrip() - if s == "": - break - strlen = len(s) - i = 0 - c = s[0] - - # String literal (constant symbol) - if c == '"' or c == "'": - i += 1 - - if "\\" in s: - # Slow path: This could probably be sped up, but it's a - # very unusual case anyway. - quote = c - value = "" - while 1: - if i >= strlen: - _tokenization_error(s, strlen, filename, - linenr) - c = s[i] - if c == quote: - break - if c == "\\": - if i + 1 >= strlen: - _tokenization_error(s, strlen, filename, - linenr) - value += s[i + 1] - i += 2 - else: - value += c - i += 1 - i += 1 - append(value) - else: - # Fast path: If the string contains no backslashes (almost - # always) we can simply look for the matching quote. - end = s.find(c, i) - if end == -1: - _tokenization_error(s, strlen, filename, linenr) - append(s[i:end]) - i = end + 1 - - elif c == "&": - if i + 1 >= strlen: - # Invalid characters are ignored - continue - if s[i + 1] != "&": - # Invalid characters are ignored - i += 1 - continue - append(T_AND) - i += 2 - - elif c == "|": - if i + 1 >= strlen: - # Invalid characters are ignored - continue - if s[i + 1] != "|": - # Invalid characters are ignored - i += 1 - continue - append(T_OR) - i += 2 - - elif c == "!": - if i + 1 >= strlen: - _tokenization_error(s, strlen, filename, linenr) - if s[i + 1] == "=": - append(T_UNEQUAL) - i += 2 - else: - append(T_NOT) - i += 1 - - elif c == "=": - append(T_EQUAL) - i += 1 - - elif c == "(": - append(T_OPEN_PAREN) - i += 1 - - elif c == ")": - append(T_CLOSE_PAREN) - i += 1 - - elif c == "#": - break - - else: - # Invalid characters are ignored - i += 1 - continue - - previous = tokens[-1] - - return _Feed(tokens) - - # - # Parsing - # - - # Expression grammar: - # - # -> - # '=' - # '!=' - # '(' ')' - # '!' - # '&&' - # '||' - - def _parse_expr(self, - feed, - cur_sym_or_choice, - line, - filename = None, - linenr = None, - transform_m = True): - """Parse an expression from the tokens in 'feed' using a simple - top-down approach. The result has the form (, ). - - feed -- _Feed instance containing the tokens for the expression. - - cur_sym_or_choice -- The symbol or choice currently being parsed, or - None if we're not parsing a symbol or choice. - Used for recording references to symbols. - - line -- The line containing the expression being parsed. - - filename (default: None) -- The file containing the expression. - - linenr (default: None) -- The line number containing the expression. - - transform_m (default: False) -- Determines if 'm' should be rewritten to - 'm && MODULES' -- see - parse_val_and_cond().""" - - # Use instance variables to avoid having to pass these as arguments - # through the top-down parser in _parse_expr_2(), which is tedious and - # obfuscates the code. A profiler run shows no noticeable performance - # difference. - self.parse_expr_cur_sym_or_choice = cur_sym_or_choice - self.parse_expr_line = line - self.parse_expr_filename = filename - self.parse_expr_linenr = linenr - self.parse_expr_transform_m = transform_m - - return self._parse_expr_2(feed) - - def _parse_expr_2(self, feed): - or_terms = [self._parse_or_term(feed)] - # Keep parsing additional terms while the lookahead is '||' - while feed.check(T_OR): - or_terms.append(self._parse_or_term(feed)) - - return or_terms[0] if len(or_terms) == 1 else (OR, or_terms) - - def _parse_or_term(self, feed): - and_terms = [self._parse_factor(feed)] - # Keep parsing additional terms while the lookahead is '&&' - while feed.check(T_AND): - and_terms.append(self._parse_factor(feed)) - - return and_terms[0] if len(and_terms) == 1 else (AND, and_terms) - - def _parse_factor(self, feed): - if feed.check(T_OPEN_PAREN): - expr_parse = self._parse_expr_2(feed) - - if not feed.check(T_CLOSE_PAREN): - _parse_error(self.parse_expr_line, - "missing end parenthesis.", - self.parse_expr_filename, - self.parse_expr_linenr) - - return expr_parse - - if feed.check(T_NOT): - return (NOT, self._parse_factor(feed)) - - sym_or_string = feed.get_next() - - if not isinstance(sym_or_string, (Symbol, str)): - _parse_error(self.parse_expr_line, - "malformed expression.", - self.parse_expr_filename, - self.parse_expr_linenr) - - if self.parse_expr_cur_sym_or_choice is not None and \ - isinstance(sym_or_string, Symbol): - self.parse_expr_cur_sym_or_choice.referenced_syms.add(sym_or_string) - - next_token = feed.peek_next() - - # For conditional expressions ('depends on ', '... if ', - # etc.), "m" and m are rewritten to "m" && MODULES. - if next_token != T_EQUAL and next_token != T_UNEQUAL: - if self.parse_expr_transform_m and (sym_or_string is self.m or - sym_or_string == "m"): - return (AND, ["m", self._sym_lookup("MODULES")]) - return sym_or_string - - relation = EQUAL if (feed.get_next() == T_EQUAL) else UNEQUAL - sym_or_string_2 = feed.get_next() - - if self.parse_expr_cur_sym_or_choice is not None and \ - isinstance(sym_or_string_2, Symbol): - self.parse_expr_cur_sym_or_choice.referenced_syms.add(sym_or_string_2) - - if sym_or_string is self.m: - sym_or_string = "m" - - if sym_or_string_2 is self.m: - sym_or_string_2 = "m" - - return (relation, sym_or_string, sym_or_string_2) - - def _parse_file(self, filename, parent, deps, visible_if_deps, res = None): - """Parse the Kconfig file 'filename'. The result is a _Block with all - items from the file. See _parse_block() for the meaning of the - parameters.""" - line_feeder = _FileFeed(_get_lines(filename), filename) - return self._parse_block(line_feeder, None, parent, deps, visible_if_deps, res) - - def _parse_block(self, line_feeder, end_marker, parent, deps, - visible_if_deps = None, res = None): - """Parses a block, which is the contents of either a file or an if, - menu, or choice statement. The result is a _Block with the items from - the block. - - end_marker -- The token that ends the block, e.g. T_ENDIF ("endif") for - if's. None for files. - - parent -- The enclosing menu, choice or if, or None if we're at the top - level. - - deps -- Dependencies from enclosing menus, choices and if's. - - visible_if_deps (default: None) -- 'visible if' dependencies from - enclosing menus. - - res (default: None) -- The _Block to add items to. If None, a new - _Block is created to hold the items.""" - - block = _Block() if res is None else res - - filename = line_feeder.get_filename() - - while 1: - - # Do we already have a tokenized line that we determined wasn't - # part of whatever we were parsing earlier? See comment in - # Config.__init__(). - if self.end_line is not None: - assert self.end_line_tokens is not None - tokens = self.end_line_tokens - tokens.go_to_start() - - line = self.end_line - linenr = line_feeder.get_linenr() - - self.end_line = None - self.end_line_tokens = None - - else: - line = line_feeder.get_next() - if line is None: - if end_marker is not None: - raise Kconfig_Syntax_Error, ( - "Unexpected end of file {0}." - .format(line_feeder.get_filename())) - return block - - linenr = line_feeder.get_linenr() - - tokens = self._tokenize(line, False, filename, linenr) - - if tokens.is_empty(): - continue - - t0 = tokens.get_next() - - # Have we reached the end of the block? - if t0 == end_marker: - return block - - if t0 == T_CONFIG or t0 == T_MENUCONFIG: - # The tokenizer will automatically allocate a new Symbol object - # for any new names it encounters, so we don't need to worry - # about that here. - sym = tokens.get_next() - - # Symbols defined in multiple places get the parent of their - # first definition. However, for symbols whose parents are choice - # statements, the choice statement takes precedence. - if not sym.is_defined_ or isinstance(parent, Choice): - sym.parent = parent - - sym.is_defined_ = True - - self.kconfig_syms.append(sym) - block.add_item(sym) - - self._parse_properties(line_feeder, sym, deps, visible_if_deps) - - elif t0 == T_MENU: - menu = Menu() - self.menus.append(menu) - menu.config = self - menu.parent = parent - menu.title = tokens.get_next() - - menu.filename = filename - menu.linenr = linenr - - # Parse properties and contents - self._parse_properties(line_feeder, menu, deps, visible_if_deps) - menu.block = self._parse_block(line_feeder, - T_ENDMENU, - menu, - menu.dep_expr, - _make_and(visible_if_deps, - menu.visible_if_expr)) - - block.add_item(menu) - - elif t0 == T_IF: - # If statements are treated as syntactic sugar for adding - # dependencies to enclosed items and do not have an explicit - # object representation. - - dep_expr = self._parse_expr(tokens, None, line, filename, linenr) - self._parse_block(line_feeder, - T_ENDIF, - parent, - _make_and(dep_expr, deps), - visible_if_deps, - block) # Add items to the same block - - elif t0 == T_CHOICE: - # We support named choices - already_defined = False - name = None - if len(tokens) > 1 and isinstance(tokens[1], str): - name = tokens[1] - already_defined = name in self.named_choices - - if already_defined: - choice = self.named_choices[name] - else: - choice = Choice() - self.choices.append(choice) - if name is not None: - choice.name = name - self.named_choices[name] = choice - - choice.config = self - choice.parent = parent - - choice.def_locations.append((filename, linenr)) - - # Parse properties and contents - self._parse_properties(line_feeder, choice, deps, visible_if_deps) - choice.block = self._parse_block(line_feeder, - T_ENDCHOICE, - choice, - None, - visible_if_deps) - - choice._determine_actual_symbols() - - # If no type is set for the choice, its type is that of the first - # choice item - if choice.type == UNKNOWN: - for item in choice.get_symbols(): - if item.type != UNKNOWN: - choice.type = item.type - break - - # Each choice item of UNKNOWN type gets the type of the choice - for item in choice.get_symbols(): - if item.type == UNKNOWN: - item.type = choice.type - - # For named choices defined in multiple locations, only record - # at the first definition - if not already_defined: - block.add_item(choice) - - elif t0 == T_COMMENT: - comment = Comment() - comment.config = self - comment.parent = parent - - comment.filename = filename - comment.linenr = linenr - - comment.text = tokens.get_next() - self._parse_properties(line_feeder, comment, deps, visible_if_deps) - - block.add_item(comment) - self.comments.append(comment) - - elif t0 == T_SOURCE: - kconfig_file = tokens.get_next() - exp_kconfig_file = self._expand_sym_refs(kconfig_file) - f = os.path.join(self.base_dir, exp_kconfig_file) - - if not os.path.exists(f): - raise IOError, ('{0}:{1}: sourced file "{2}" (expands to\n' - '"{3}") not found. Perhaps base_dir\n' - '(argument to Config.__init__(), currently\n' - '"{4}") is set to the wrong value.' - .format(filename, - linenr, - kconfig_file, - exp_kconfig_file, - self.base_dir)) - - # Add items to the same block - self._parse_file(f, parent, deps, visible_if_deps, block) - - elif t0 == T_MAINMENU: - text = tokens.get_next() - - if self.mainmenu_text is not None: - self._warn("overriding 'mainmenu' text. " - 'Old value: "{0}", new value: "{1}".' - .format(self.mainmenu_text, text), - filename, - linenr) - - self.mainmenu_text = text - - else: - _parse_error(line, "unrecognized construct.", filename, linenr) - - def _parse_properties(self, line_feeder, stmt, deps, visible_if_deps): - """Parsing of properties for symbols, menus, choices, and comments.""" - - def parse_val_and_cond(tokens, line, filename, linenr): - """Parses ' if ' constructs, where the 'if' part is - optional. Returns a tuple containing the parsed expressions, with - None as the second element if the 'if' part is missing.""" - val = self._parse_expr(tokens, stmt, line, filename, linenr, False) - - if tokens.check(T_IF): - return (val, self._parse_expr(tokens, stmt, line, filename, linenr)) - - return (val, None) - - # In case the symbol is defined in multiple locations, we need to - # remember what prompts, defaults, and selects are new for this - # definition, as "depends on" should only apply to the local - # definition. - new_prompt = None - new_def_exprs = [] - new_selects = [] - - # Dependencies from 'depends on' statements - depends_on_expr = None - - while 1: - line = line_feeder.get_next() - if line is None: - break - - filename = line_feeder.get_filename() - linenr = line_feeder.get_linenr() - - tokens = self._tokenize(line, False, filename, linenr) - - if tokens.is_empty(): - continue - - t0 = tokens.get_next() - - if t0 == T_HELP: - # Find first non-empty line and get its indentation - - line_feeder.remove_while(str.isspace) - line = line_feeder.get_next() - - if line is None: - stmt.help = "" - break - - indent = _indentation(line) - - # If the first non-empty lines has zero indent, there is no - # help text - if indent == 0: - stmt.help = "" - line_feeder.go_back() - break - - help_lines = [_deindent(line, indent)] - - # The help text goes on till the first non-empty line with less - # indent - while 1: - line = line_feeder.get_next() - if (line is None) or \ - (not line.isspace() and _indentation(line) < indent): - stmt.help = "".join(help_lines) - break - - help_lines.append(_deindent(line, indent)) - - if line is None: - break - - line_feeder.go_back() - - elif t0 == T_PROMPT: - # 'prompt' properties override each other within a single - # definition of a symbol, but additional prompts can be added - # by defining the symbol multiple times; hence 'new_prompt' - # instead of 'prompt'. - new_prompt = parse_val_and_cond(tokens, line, filename, linenr) - - elif t0 == T_DEFAULT: - new_def_exprs.append(parse_val_and_cond(tokens, line, filename, linenr)) - - elif t0 == T_DEPENDS: - if not tokens.check(T_ON): - _parse_error(line, 'expected "on" after "depends".', filename, linenr) - - parsed_deps = self._parse_expr(tokens, stmt, line, filename, linenr) - - if isinstance(stmt, (Menu, Comment)): - stmt.dep_expr = _make_and(stmt.dep_expr, parsed_deps) - else: - depends_on_expr = _make_and(depends_on_expr, parsed_deps) - - elif t0 == T_VISIBLE: - if not tokens.check(T_IF): - _parse_error(line, 'expected "if" after "visible".', filename, linenr) - if not isinstance(stmt, Menu): - _parse_error(line, - "'visible if' is only valid for menus.", - filename, - linenr) - - parsed_deps = self._parse_expr(tokens, stmt, line, filename, linenr) - stmt.visible_if_expr = _make_and(stmt.visible_if_expr, parsed_deps) - - elif t0 == T_SELECT: - target = tokens.get_next() - - stmt.referenced_syms.add(target) - stmt.selected_syms.add(target) - - if tokens.check(T_IF): - new_selects.append((target, - self._parse_expr(tokens, stmt, line, filename, linenr))) - else: - new_selects.append((target, None)) - - elif t0 in (T_BOOL, T_TRISTATE, T_INT, T_HEX, T_STRING): - stmt.type = token_to_type[t0] - - if len(tokens) > 1: - new_prompt = parse_val_and_cond(tokens, line, filename, linenr) - - elif t0 == T_RANGE: - lower = tokens.get_next() - upper = tokens.get_next() - stmt.referenced_syms.add(lower) - stmt.referenced_syms.add(upper) - - if tokens.check(T_IF): - stmt.ranges.append((lower, upper, - self._parse_expr(tokens, stmt, line, filename, linenr))) - else: - stmt.ranges.append((lower, upper, None)) - - elif t0 == T_DEF_BOOL: - stmt.type = BOOL - - if len(tokens) > 1: - new_def_exprs.append(parse_val_and_cond(tokens, line, filename, linenr)) - - elif t0 == T_DEF_TRISTATE: - stmt.type = TRISTATE - - if len(tokens) > 1: - new_def_exprs.append(parse_val_and_cond(tokens, line, filename, linenr)) - - elif t0 == T_OPTIONAL: - if not isinstance(stmt, Choice): - _parse_error(line, - '"optional" is only valid for choices.', - filename, - linenr) - stmt.optional = True - - elif t0 == T_OPTION: - if tokens.check(T_ENV) and tokens.check(T_EQUAL): - env_var = tokens.get_next() - - stmt.is_special_ = True - stmt.is_from_env = True - - if env_var not in os.environ: - self._warn(""" -The symbol {0} references the non-existent environment variable {1} and will -get the empty string as its value. - -If you're using kconfiglib via 'make (i)scriptconfig' it should have set up the -environment correctly for you. If you still got this message, that might be an -error, and you should e-mail kconfiglib@gmail.com. -.""" .format(stmt.name, env_var), - filename, - linenr) - - stmt.cached_value = "" - else: - stmt.cached_value = os.environ[env_var] - - elif tokens.check(T_DEFCONFIG_LIST): - self.defconfig_sym = stmt - - elif tokens.check(T_MODULES): - self._warn("the 'modules' option is not supported. " - "Let me know if this is a problem for you; " - "it shouldn't be that hard to implement.", - filename, - linenr) - - else: - _parse_error(line, "unrecognized option.", filename, linenr) - - else: - # See comment in Config.__init__() - self.end_line = line - self.end_line_tokens = tokens - break - - # Propagate dependencies from enclosing menus and if's. - - # For menus and comments.. - if isinstance(stmt, (Menu, Comment)): - stmt.orig_deps = stmt.dep_expr - stmt.deps_from_containing = deps - stmt.dep_expr = _make_and(stmt.dep_expr, deps) - - stmt.all_referenced_syms = \ - stmt.referenced_syms | _get_expr_syms(deps) - - # For symbols and choices.. - else: - - # See comment for 'menu_dep' - stmt.menu_dep = depends_on_expr - - # Propagate dependencies specified with 'depends on' to any new - # default expressions, prompts, and selections. ("New" since a - # symbol might be defined in multiple places and the dependencies - # should only apply to the local definition.) - - new_def_exprs = [(val_expr, _make_and(cond_expr, depends_on_expr)) - for (val_expr, cond_expr) in new_def_exprs] - - new_selects = [(target, _make_and(cond_expr, depends_on_expr)) - for (target, cond_expr) in new_selects] - - if new_prompt is not None: - prompt, cond_expr = new_prompt - - # 'visible if' dependencies from enclosing menus get propagated - # to prompts - if visible_if_deps is not None: - cond_expr = _make_and(cond_expr, visible_if_deps) - - new_prompt = (prompt, _make_and(cond_expr, depends_on_expr)) - - # We save the original expressions -- before any menu and if - # conditions have been propagated -- so these can be retrieved - # later. - - stmt.orig_def_exprs.extend(new_def_exprs) - if new_prompt is not None: - stmt.orig_prompts.append(new_prompt) - - # Only symbols can select - if isinstance(stmt, Symbol): - stmt.orig_selects.extend(new_selects) - - # Save dependencies from enclosing menus and if's - stmt.deps_from_containing = deps - - # The set of symbols referenced directly by the symbol/choice plus - # all symbols referenced by enclosing menus and if's. - stmt.all_referenced_syms = \ - stmt.referenced_syms | _get_expr_syms(deps) - - # Propagate dependencies from enclosing menus and if's - - stmt.def_exprs.extend([(val_expr, _make_and(cond_expr, deps)) - for (val_expr, cond_expr) in new_def_exprs]) - - for (target, cond) in new_selects: - target.rev_dep = _make_or(target.rev_dep, - _make_and(stmt, - _make_and(cond, deps))) - - if new_prompt is not None: - prompt, cond_expr = new_prompt - stmt.prompts.append((prompt, _make_and(cond_expr, deps))) - - # - # Symbol table manipulation - # - - def _sym_lookup(self, name, add_sym_if_not_exists = True): - """Fetches the symbol 'name' from the symbol table, optionally adding - it if it does not exist (this is usually what we want).""" - if name in self.syms: - return self.syms[name] - - new_sym = Symbol() - new_sym.config = self - new_sym.name = name - - if add_sym_if_not_exists: - self.syms[name] = new_sym - else: - # This warning is generated while evaluating an expression - # containing undefined symbols using Config.eval() - self._warn("no symbol {0} in configuration".format(name)) - - return new_sym - - # - # Evaluation of symbols and expressions - # - - def _eval_expr(self, expr): - """Evaluates an expression and returns one of the tristate values "n", - "m" or "y".""" - res = self._eval_expr_2(expr) - - # Promote "m" to "y" if we're running without modules. Internally, "m" - # is often rewritten to "m" && MODULES by both the C implementation and - # kconfiglib, which takes care of cases where "m" should be false if - # we're running without modules. - if res == "m" and not self._has_modules(): - return "y" - - return res - - def _eval_expr_2(self, expr): - if expr is None: - return "y" - - if isinstance(expr, Symbol): - # Non-bool/tristate symbols are always "n" in a tristate sense, - # regardless of their value - if expr.type != BOOL and expr.type != TRISTATE: - return "n" - return expr.get_value() - - if isinstance(expr, str): - return expr if (expr == "y" or expr == "m") else "n" - - first_expr = expr[0] - - if first_expr == OR: - res = "n" - - for subexpr in expr[1]: - ev = self._eval_expr_2(subexpr) - - # Return immediately upon discovering a "y" term - if ev == "y": - return "y" - - if ev == "m": - res = "m" - - # 'res' is either "n" or "m" here; we already handled the - # short-circuiting "y" case in the loop. - return res - - if first_expr == AND: - res = "y" - - for subexpr in expr[1]: - ev = self._eval_expr_2(subexpr) - - # Return immediately upon discovering an "n" term - if ev == "n": - return "n" - - if ev == "m": - res = "m" - - # 'res' is either "m" or "y" here; we already handled the - # short-circuiting "n" case in the loop. - return res - - if first_expr == NOT: - ev = self._eval_expr_2(expr[1]) - - if ev == "y": - return "n" - - return "y" if (ev == "n") else "m" - - if first_expr == EQUAL: - return "y" if (self._get_str_value(expr[1]) == - self._get_str_value(expr[2])) else "n" - - if first_expr == UNEQUAL: - return "y" if (self._get_str_value(expr[1]) != - self._get_str_value(expr[2])) else "n" - - _internal_error("Internal error while evaluating expression: " - "unknown operation {0}.".format(first_expr)) - - def _get_str_value(self, obj): - if isinstance(obj, str): - return obj - # obj is a Symbol - return obj.get_value() - - def _eval_min(self, e1, e2): - e1_eval = self._eval_expr(e1) - e2_eval = self._eval_expr(e2) - - return e1_eval if tri_less(e1_eval, e2_eval) else e2_eval - - def _eval_max(self, e1, e2): - e1_eval = self._eval_expr(e1) - e2_eval = self._eval_expr(e2) - - return e1_eval if tri_greater(e1_eval, e2_eval) else e2_eval - - # - # Methods related to the MODULES symbol - # - - def _has_modules(self): - modules_sym = self.syms.get("MODULES") - return (modules_sym is not None) and (modules_sym.get_value() == "y") - - # - # Dependency tracking - # - - def _build_dep(self): - """Populates the Symbol.dep sets, linking the symbol to the symbols - that immediately depend on it in the sense that changing the value of - the symbol might affect the values of those other symbols. This is used - for caching/invalidation purposes. The calculated sets might be larger - than necessary as we don't do any complicated analysis of the - expressions.""" - for sym in self.syms.itervalues(): - sym.dep = set() - - # Adds 'sym' as a directly dependent symbol to all symbols that appear - # in the expression 'e' - def add_expr_deps(e, sym): - for s in _get_expr_syms(e): - s.dep.add(sym) - - # The directly dependent symbols of a symbol are: - # - Any symbols whose prompts, default values, rev_dep (select - # condition), or ranges depend on the symbol - # - Any symbols that belong to the same choice statement as the symbol - # (these won't be included in 'dep' as that makes the dependency - # graph unwieldy, but Symbol._get_dependent() will include them) - # - Any symbols in a choice statement that depends on the symbol - for sym in self.syms.itervalues(): - for (_, e) in sym.prompts: - add_expr_deps(e, sym) - - for (v, e) in sym.def_exprs: - add_expr_deps(v, sym) - add_expr_deps(e, sym) - - add_expr_deps(sym.rev_dep, sym) - - for (l, u, e) in sym.ranges: - add_expr_deps(l, sym) - add_expr_deps(u, sym) - add_expr_deps(e, sym) - - if sym.is_choice_symbol_: - choice = sym.parent - - for (_, e) in choice.prompts: - add_expr_deps(e, sym) - - for (_, e) in choice.def_exprs: - add_expr_deps(e, sym) - - def _expr_val_str(self, expr, no_value_str = "(none)", get_val_instead_of_eval = False): - # Since values are valid expressions, _expr_to_str() will get a nice - # string representation for those as well. - - if expr is None: - return no_value_str - - if get_val_instead_of_eval: - if isinstance(expr, str): - return _expr_to_str(expr) - val = expr.get_value() - else: - val = self._eval_expr(expr) - - return "{0} (value: {1})".format(_expr_to_str(expr), _expr_to_str(val)) - - def _expand_sym_refs(self, s): - """Expands $-references to symbols in 's' to symbol values, or to the - empty string for undefined symbols.""" - - while 1: - sym_ref_re_match = sym_ref_re.search(s) - if sym_ref_re_match is None: - return s - - sym_name = sym_ref_re_match.group(0)[1:] - sym = self.syms.get(sym_name) - expansion = "" if sym is None else sym.get_value() - - s = s[:sym_ref_re_match.start()] + \ - expansion + \ - s[sym_ref_re_match.end():] - - def _get_sym_or_choice_str(self, sc): - """Symbols and choices have many properties in common, so we factor out - common __str__() stuff here. "sc" is short for "symbol or choice".""" - - # As we deal a lot with string representations here, use some - # convenient shorthand: - s = _expr_to_str - - # - # Common symbol/choice properties - # - - user_value_str = "(no user value)" if sc.user_val is None else s(sc.user_val) - - visibility_str = s(sc.get_visibility()) - - # Build prompts string - if sc.prompts == []: - prompts_str = " (no prompts)" - else: - prompts_str_rows = [] - - for (prompt, cond_expr) in sc.orig_prompts: - if cond_expr is None: - prompts_str_rows.append(' "{0}"'.format(prompt)) - else: - prompts_str_rows.append(' "{0}" if '.format(prompt) + - self._expr_val_str(cond_expr)) - - prompts_str = "\n".join(prompts_str_rows) - - # Build locations string - if sc.def_locations == []: - locations_str = "(no locations)" - else: - locations_str = " ".join(["{0}:{1}".format(filename, linenr) for - (filename, linenr) in sc.def_locations]) - - # Build additional-dependencies-from-menus-and-if's string - additional_deps_str = " " + self._expr_val_str(sc.deps_from_containing, - "(no additional dependencies)") - - # - # Symbol-specific stuff - # - - if isinstance(sc, Symbol): - - # Build value string - value_str = s(sc.get_value()) - - # Build ranges string - if isinstance(sc, Symbol): - if sc.ranges == []: - ranges_str = " (no ranges)" - else: - ranges_str_rows = [] - - for (l, u, cond_expr) in sc.ranges: - if cond_expr is None: - ranges_str_rows.append(" [{0}, {1}]".format(s(l), s(u))) - else: - ranges_str_rows.append(" [{0}, {1}] if {2}" - .format(s(l), s(u), self._expr_val_str(cond_expr))) - - ranges_str = "\n".join(ranges_str_rows) - - # Build default values string - if sc.def_exprs == []: - defaults_str = " (no default values)" - else: - defaults_str_rows = [] - - for (val_expr, cond_expr) in sc.orig_def_exprs: - row_str = " " + self._expr_val_str(val_expr, "(none)", sc.type == STRING) - defaults_str_rows.append(row_str) - defaults_str_rows.append(" Condition: " + self._expr_val_str(cond_expr)) - - defaults_str = "\n".join(defaults_str_rows) - - # Build selects string - if sc.orig_selects == []: - selects_str = " (no selects)" - else: - selects_str_rows = [] - - for (target, cond_expr) in sc.orig_selects: - if cond_expr is None: - selects_str_rows.append(" {0}".format(target.name)) - else: - selects_str_rows.append(" {0} if ".format(target.name) + - self._expr_val_str(cond_expr)) - - selects_str = "\n".join(selects_str_rows) - - # Build reverse dependencies string - if sc.rev_dep == "n": - rev_dep_str = " (no reverse dependencies)" - else: - rev_dep_str = " " + self._expr_val_str(sc.rev_dep) - - res = _sep_lines("Symbol " + (sc.name if sc.name is not None else "(no name)"), - "Type : " + typename[sc.type], - "Value : " + value_str, - "User value : " + user_value_str, - "Visibility : " + visibility_str, - "Is choice item : " + bool_str[sc.is_choice_symbol_], - "Is defined : " + bool_str[sc.is_defined_], - "Is from env. : " + bool_str[sc.is_from_env], - "Is special : " + bool_str[sc.is_special_] + "\n") - - if sc.ranges != []: - res += _sep_lines("Ranges:", - ranges_str + "\n") - - res += _sep_lines("Prompts:", - prompts_str, - "Default values:", - defaults_str, - "Selects:", - selects_str, - "Reverse dependencies:", - rev_dep_str, - "Additional dependencies from enclosing menus and if's:", - additional_deps_str, - "Locations: " + locations_str) - - return res - - # - # Choice-specific stuff - # - - # Build name string (for named choices) - if sc.name is None: - name_str = "(no name)" - else: - name_str = sc.name - - # Build selected symbol string - sel = sc.get_selection() - if sel is None: - sel_str = "(no selection)" - else: - sel_str = sel.name - - # Build mode string - mode_str = s(sc.get_mode()) - - # Build default values string - if sc.def_exprs == []: - defaults_str = " (no default values)" - else: - defaults_str_rows = [] - - for (sym, cond_expr) in sc.orig_def_exprs: - if cond_expr is None: - defaults_str_rows.append(" {0}".format(sym.name)) - else: - defaults_str_rows.append(" {0} if ".format(sym.name) + - self._expr_val_str(cond_expr)) - - defaults_str = "\n".join(defaults_str_rows) - - # Build contained symbols string - names = [sym.name for sym in sc.get_symbols()] - - if names == []: - syms_string = "(empty)" - else: - syms_string = " ".join(names) - - return _sep_lines("Choice", - "Name (for named choices): " + name_str, - "Type : " + typename[sc.type], - "Selected symbol : " + sel_str, - "User value : " + user_value_str, - "Mode : " + mode_str, - "Visibility : " + visibility_str, - "Optional : " + bool_str[sc.optional], - "Prompts:", - prompts_str, - "Defaults:", - defaults_str, - "Choice symbols:", - " " + syms_string, - "Additional dependencies from enclosing menus and if's:", - additional_deps_str, - "Locations: " + locations_str) - - def _expr_depends_on(self, expr, sym): - """Reimplementation of expr_depends_symbol() from mconf.c. Used to - determine if a submenu should be implicitly created, which influences what - items inside choice statements are considered choice items.""" - if expr is None: - return False - - def rec(expr): - if isinstance(expr, str): - return False - - if isinstance(expr, Symbol): - return expr is sym - - e0 = expr[0] - - if e0 == EQUAL or e0 == UNEQUAL: - return self._eq_to_sym(expr) is sym - - if e0 == AND: - for and_expr in expr[1]: - if rec(and_expr): - return True - - return False - - return rec(expr) - - def _eq_to_sym(self, eq): - """_expr_depends_on() helper. For (in)equalities of the form sym = y/m - or sym != n, returns sym. For other (in)equalities, returns None.""" - relation, left, right = eq - - left = self._transform_n_m_y(left) - right = self._transform_n_m_y(right) - - # Make sure the symbol (if any) appears to the left - if not isinstance(left, Symbol): - left, right = right, left - - if not isinstance(left, Symbol): - return None - - if (relation == EQUAL and (right == "m" or right == "y")) or \ - (relation == UNEQUAL and right == "n"): - return left - - return None - - def _transform_n_m_y(self, item): - """_eq_to_sym() helper. Translates the symbols n, m, and y to their - string equivalents.""" - if item is self.n: - return "n" - if item is self.m: - return "m" - if item is self.y: - return "y" - return item - - def _warn(self, msg, filename = None, linenr = None): - """For printing warnings to stderr.""" - if self.print_warnings: - self._warn_or_undef_assign(msg, WARNING, filename, linenr) - - def _undef_assign(self, msg, filename = None, linenr = None): - """For printing informational messages related to assignments - to undefined variables to stderr.""" - if self.print_undef_assign: - self._warn_or_undef_assign(msg, UNDEF_ASSIGN, filename, linenr) - - def _warn_or_undef_assign(self, msg, msg_type, filename, linenr): - if filename is not None: - sys.stderr.write("{0}:".format(_clean_up_path(filename))) - if linenr is not None: - sys.stderr.write("{0}:".format(linenr)) - - if msg_type == WARNING: - sys.stderr.write("warning: ") - elif msg_type == UNDEF_ASSIGN: - sys.stderr.write("info: ") - else: - _internal_error('Internal error while printing warning: unknown warning type "{0}".' - .format(msg_type)) - - sys.stderr.write(msg + "\n") - -def _get_expr_syms(expr): - """Returns the set() of symbols appearing in expr.""" - res = set() - if expr is None: - return res - - def rec(expr): - if isinstance(expr, Symbol): - res.add(expr) - return - - if isinstance(expr, str): - return - - e0 = expr[0] - - if e0 == OR or e0 == AND: - for term in expr[1]: - rec(term) - - elif e0 == NOT: - rec(expr[1]) - - elif e0 == EQUAL or e0 == UNEQUAL: - _, v1, v2 = expr - - if isinstance(v1, Symbol): - res.add(v1) - - if isinstance(v2, Symbol): - res.add(v2) - - else: - _internal_error("Internal error while fetching symbols from an " - "expression with token stream {0}.".format(expr)) - - rec(expr) - return res - - -# -# Construction of expressions -# - -# These functions as well as the _eval_min/max() functions above equate -# None with "y", which is usually what we want, but needs to be kept in -# mind. - -def _make_or(e1, e2): - # Perform trivial simplification and avoid None's (which - # correspond to y's) - if e1 is None or e2 is None or \ - e1 == "y" or e2 == "y": - return "y" - - if e1 == "n": - return e2 - - if e2 == "n": - return e1 - - # Prefer to merge/update argument list if possible instead of creating - # a new OR node - - if isinstance(e1, tuple) and e1[0] == OR: - if isinstance(e2, tuple) and e2[0] == OR: - return (OR, e1[1] + e2[1]) - return (OR, e1[1] + [e2]) - - if isinstance(e2, tuple) and e2[0] == OR: - return (OR, e2[1] + [e1]) - - return (OR, [e1, e2]) - -# Note: returns None if e1 == e2 == None - -def _make_and(e1, e2): - if e1 == "n" or e2 == "n": - return "n" - - if e1 is None or e1 == "y": - return e2 - - if e2 is None or e2 == "y": - return e1 - - # Prefer to merge/update argument list if possible instead of creating - # a new AND node - - if isinstance(e1, tuple) and e1[0] == AND: - if isinstance(e2, tuple) and e2[0] == AND: - return (AND, e1[1] + e2[1]) - return (AND, e1[1] + [e2]) - - if isinstance(e2, tuple) and e2[0] == AND: - return (AND, e2[1] + [e1]) - - return (AND, [e1, e2]) - -# -# Constants and functions related to types, parsing, evaluation and printing, -# put globally to unclutter the Config class a bit. -# - -# Tokens -(T_OR, T_AND, T_NOT, - T_OPEN_PAREN, T_CLOSE_PAREN, - T_EQUAL, T_UNEQUAL, - T_MAINMENU, T_MENU, T_ENDMENU, - T_SOURCE, T_CHOICE, T_ENDCHOICE, - T_COMMENT, T_CONFIG, T_MENUCONFIG, - T_HELP, T_IF, T_ENDIF, T_DEPENDS, T_ON, - T_OPTIONAL, T_PROMPT, T_DEFAULT, - T_BOOL, T_TRISTATE, T_HEX, T_INT, T_STRING, - T_DEF_BOOL, T_DEF_TRISTATE, - T_SELECT, T_RANGE, T_OPTION, T_ENV, - T_DEFCONFIG_LIST, T_MODULES, T_VISIBLE) = range(0, 38) - -# Keyword to token map -keywords = { - "mainmenu" : T_MAINMENU, - "menu" : T_MENU, - "endmenu" : T_ENDMENU, - "endif" : T_ENDIF, - "endchoice" : T_ENDCHOICE, - "source" : T_SOURCE, - "choice" : T_CHOICE, - "config" : T_CONFIG, - "comment" : T_COMMENT, - "menuconfig" : T_MENUCONFIG, - "help" : T_HELP, - "if" : T_IF, - "depends" : T_DEPENDS, - "on" : T_ON, - "optional" : T_OPTIONAL, - "prompt" : T_PROMPT, - "default" : T_DEFAULT, - "bool" : T_BOOL, - "boolean" : T_BOOL, - "tristate" : T_TRISTATE, - "int" : T_INT, - "hex" : T_HEX, - "def_bool" : T_DEF_BOOL, - "def_tristate" : T_DEF_TRISTATE, - "string" : T_STRING, - "select" : T_SELECT, - "range" : T_RANGE, - "option" : T_OPTION, - "env" : T_ENV, - "defconfig_list" : T_DEFCONFIG_LIST, - "modules" : T_MODULES, - "visible" : T_VISIBLE } - -# Strings to use for True and False -bool_str = { False : "false", True : "true" } - -# Tokens after which identifier-like lexemes are treated as strings. T_CHOICE -# is included to avoid symbols being registered for named choices. -string_lex = frozenset((T_BOOL, T_TRISTATE, T_INT, T_HEX, T_STRING, T_CHOICE, - T_PROMPT, T_MENU, T_COMMENT, T_SOURCE, T_MAINMENU)) - -# Matches the initial token on a line; see _tokenize(). -initial_token_re = re.compile(r"[^\w]*(\w+)") - -# Matches an identifier/keyword optionally preceded by whitespace -id_keyword_re = re.compile(r"\s*([\w./-]+)") - -# Regular expressions for parsing .config files -set_re = re.compile(r"CONFIG_(\w+)=(.*)") -unset_re = re.compile(r"# CONFIG_(\w+) is not set") - -# Regular expression for finding $-references to symbols in strings -sym_ref_re = re.compile(r"\$[A-Za-z_][0-9A-Za-z_]*") - -# Integers representing symbol types -UNKNOWN, BOOL, TRISTATE, STRING, HEX, INT = range(0, 6) - -# Strings to use for types -typename = { - UNKNOWN : "unknown", - BOOL : "bool", - TRISTATE : "tristate", - STRING : "string", - HEX : "hex", - INT : "int" } - -# Token to type mapping -token_to_type = { T_BOOL : BOOL, - T_TRISTATE : TRISTATE, - T_STRING : STRING, - T_INT : INT, - T_HEX : HEX } - -# Default values for symbols of different types (the value the symbol gets if -# it is not assigned a user value and none of its 'default' clauses kick in) -default_value = { BOOL : "n", - TRISTATE : "n", - STRING : "", - INT : "", - HEX : "" } - -# Indicates that no item is selected in a choice statement -NO_SELECTION = 0 - -# Integers representing expression types -OR, AND, NOT, EQUAL, UNEQUAL = range(0, 5) - -# Map from tristate values to integers -tri_to_int = { "n" : 0, "m" : 1, "y" : 2 } - -# Printing-related stuff - -op_to_str = { AND : " && ", - OR : " || ", - EQUAL : " = ", - UNEQUAL : " != " } - -precedence = { OR : 0, AND : 1, NOT : 2 } - -# Types of informational messages -WARNING = 0 -UNDEF_ASSIGN = 1 - -def _intersperse(lst, op): - """_expr_to_str() helper. Gets the string representation of each expression in lst - and produces a list where op has been inserted between the elements.""" - if lst == []: - return "" - - res = [] - - def handle_sub_expr(expr): - no_parens = isinstance(expr, (str, Symbol)) or \ - expr[0] in (EQUAL, UNEQUAL) or \ - precedence[op] <= precedence[expr[0]] - if not no_parens: - res.append("(") - res.extend(_expr_to_str_rec(expr)) - if not no_parens: - res.append(")") - - op_str = op_to_str[op] - - handle_sub_expr(lst[0]) - for expr in lst[1:]: - res.append(op_str) - handle_sub_expr(expr) - - return res - -def _expr_to_str(expr): - s = "".join(_expr_to_str_rec(expr)) - return s - -def _sym_str_string(sym_or_str): - if isinstance(sym_or_str, str): - return '"{0}"'.format(sym_or_str) - return sym_or_str.name - -def _expr_to_str_rec(expr): - if expr is None: - return [""] - - if isinstance(expr, (Symbol, str)): - return [_sym_str_string(expr)] - - e0 = expr[0] - - if e0 == OR or e0 == AND: - return _intersperse(expr[1], expr[0]) - - if e0 == NOT: - need_parens = not isinstance(expr[1], (str, Symbol)) - - res = ["!"] - if need_parens: - res.append("(") - res.extend(_expr_to_str_rec(expr[1])) - if need_parens: - res.append(")") - return res - - if e0 == EQUAL or e0 == UNEQUAL: - return [_sym_str_string(expr[1]), - op_to_str[expr[0]], - _sym_str_string(expr[2])] - -class _Block: - - """Represents a list of items (symbols, menus, choice statements and - comments) appearing at the top-level of a file or witin a menu, choice or - if statement.""" - - def __init__(self): - self.items = [] - - def get_items(self): - return self.items - - def add_item(self, item): - self.items.append(item) - - def _make_conf(self): - # Collect the substrings in a list and later use join() instead of += - # to build the final .config contents. With older Python versions, this - # yields linear instead of quadratic complexity. - strings = [] - for item in self.items: - strings.extend(item._make_conf()) - - return strings - - def add_depend_expr(self, expr): - for item in self.items: - item.add_depend_expr(expr) - -class Item(): - - """Base class for symbols and other Kconfig constructs. Subclasses are - Symbol, Choice, Menu, and Comment.""" - - def is_symbol(self): - """Returns True if the item is a symbol, otherwise False. Short for - isinstance(item, kconfiglib.Symbol).""" - return isinstance(self, Symbol) - - def is_choice(self): - """Returns True if the item is a choice, otherwise False. Short for - isinstance(item, kconfiglib.Choice).""" - return isinstance(self, Choice) - - def is_menu(self): - """Returns True if the item is a menu, otherwise False. Short for - isinstance(item, kconfiglib.Menu).""" - return isinstance(self, Menu) - - def is_comment(self): - """Returns True if the item is a comment, otherwise False. Short for - isinstance(item, kconfiglib.Comment).""" - return isinstance(self, Comment) - -class _HasVisibility(): - - """Base class for elements that have a "visibility" that acts as an upper - limit on the values a user can set for them. Subclasses are Symbol and - Choice (which supply some of the attributes).""" - - def __init__(self): - self.cached_visibility = None - self.prompts = [] - - def _invalidate(self): - self.cached_visibility = None - - def _get_visibility(self): - if self.cached_visibility is None: - vis = "n" - for (prompt, cond_expr) in self.prompts: - vis = self.config._eval_max(vis, cond_expr) - - if isinstance(self, Symbol) and self.is_choice_symbol_: - vis = self.config._eval_min(vis, self.parent._get_visibility()) - - # Promote "m" to "y" if we're dealing with a non-tristate - if vis == "m" and self.type != TRISTATE: - vis = "y" - - self.cached_visibility = vis - - return self.cached_visibility - -class Symbol(Item, _HasVisibility): - - """Represents a configuration symbol - e.g. FOO for - - config FOO - ...""" - - # - # Public interface - # - - def get_value(self): - """Calculate and return the value of the symbol. See also - Symbol.set_user_value().""" - - if self.cached_value is not None: - return self.cached_value - - self.write_to_conf = False - - # As a quirk of Kconfig, undefined symbols get their name as their - # value. This is why things like "FOO = bar" work for seeing if FOO has - # the value "bar". - if self.type == UNKNOWN: - self.cached_value = self.name - return self.name - - new_val = default_value[self.type] - - vis = self._get_visibility() - - if self.type == BOOL or self.type == TRISTATE: - # The visibility and mode (modules-only or single-selection) of - # choice items will be taken into account in self._get_visibility() - - if self.is_choice_symbol_: - if vis != "n": - choice = self.parent - mode = choice.get_mode() - - self.write_to_conf = (mode != "n") - - if mode == "y": - new_val = "y" if (choice.get_selection() is self) else "n" - elif mode == "m": - if self.user_val == "m" or self.user_val == "y": - new_val = "m" - - else: - use_defaults = True - - if vis != "n": - # If the symbol is visible and has a user value, use that. - # Otherwise, look at defaults. - self.write_to_conf = True - - if self.user_val is not None: - new_val = self.config._eval_min(self.user_val, vis) - use_defaults = False - - if use_defaults: - for (val_expr, cond_expr) in self.def_exprs: - cond_eval = self.config._eval_expr(cond_expr) - - if cond_eval != "n": - self.write_to_conf = True - new_val = self.config._eval_min(val_expr, cond_eval) - break - - # Reverse dependencies take precedence - rev_dep_val = self.config._eval_expr(self.rev_dep) - - if rev_dep_val != "n": - self.write_to_conf = True - new_val = self.config._eval_max(new_val, rev_dep_val) - - # Promote "m" to "y" for booleans - if new_val == "m" and self.type == BOOL: - new_val = "y" - - elif self.type == STRING: - use_defaults = True - - if vis != "n": - self.write_to_conf = True - if self.user_val is not None: - new_val = self.user_val - use_defaults = False - - if use_defaults: - for (val_expr, cond_expr) in self.def_exprs: - if self.config._eval_expr(cond_expr) != "n": - self.write_to_conf = True - new_val = self.config._get_str_value(val_expr) - break - - elif self.type == HEX or self.type == INT: - has_active_range = False - low = None - high = None - use_defaults = True - - base = 16 if self.type == HEX else 10 - - for(l, h, cond_expr) in self.ranges: - if self.config._eval_expr(cond_expr) != "n": - has_active_range = True - - low_str = self.config._get_str_value(l) - high_str = self.config._get_str_value(h) - - low = int(low_str, base) if \ - _is_base_n(low_str, base) else 0 - high = int(high_str, base) if \ - _is_base_n(high_str, base) else 0 - - break - - if vis != "n": - self.write_to_conf = True - - if self.user_val is not None and \ - _is_base_n(self.user_val, base) and \ - (not has_active_range or - low <= int(self.user_val, base) <= high): - - # If the user value is OK, it is stored in exactly the same - # form as specified in the assignment (with or without - # "0x", etc). - - use_defaults = False - new_val = self.user_val - - if use_defaults: - for (val_expr, cond_expr) in self.def_exprs: - if self.config._eval_expr(cond_expr) != "n": - self.write_to_conf = True - - # If the default value is OK, it is stored in exactly - # the same form as specified. Otherwise, it is clamped - # to the range, and the output has "0x" as appropriate - # for the type. - - new_val = self.config._get_str_value(val_expr) - - if _is_base_n(new_val, base): - new_val_num = int(new_val, base) - if has_active_range: - clamped_val = None - - if new_val_num < low: - clamped_val = low - elif new_val_num > high: - clamped_val = high - - if clamped_val is not None: - new_val = (hex(clamped_val) if \ - self.type == HEX else str(clamped_val)) - - break - else: # For the for loop - # If no user value or default kicks in but the hex/int has - # an active range, then the low end of the range is used, - # provided it's > 0, with "0x" prepended as appropriate. - - if has_active_range and low > 0: - new_val = (hex(low) if self.type == HEX else str(low)) - - self.cached_value = new_val - return new_val - - def set_user_value(self, v): - """Sets the user value of the symbol. - - Equal in effect to assigning the value to the symbol within a .config - file. Use get_lower/upper_bound() or get_assignable_values() to find - the range of currently assignable values for bool and tristate symbols; - setting values outside this range will cause the user value to differ - from the result of Symbol.get_value() (be truncated). Values that are - invalid for the type (such as a_bool.set_user_value("foo")) are - ignored, and a warning is emitted if an attempt is made to assign such - a value. - - For any type of symbol, is_modifiable() can be used to check if a user - value will currently have any effect on the symbol, as determined by - its visibility and range of assignable values. Any value that is valid - for the type (bool, tristate, etc.) will end up being reflected in - get_user_value() though, and might have an effect later if conditions - change. To get rid of the user value, use unset_user_value(). - - Any symbols dependent on the symbol are (recursively) invalidated, so - things will just work with regards to dependencies. - - v -- The user value to give to the symbol.""" - self._set_user_value_no_invalidate(v, False) - - # There might be something more efficient you could do here, but play - # it safe. - if self.name == "MODULES": - self.config._invalidate_all() - return - - self._invalidate() - self._invalidate_dependent() - - def unset_user_value(self): - """Resets the user value of the symbol, as if the symbol had never - gotten a user value via Config.load_config() or - Symbol.set_user_value().""" - self._unset_user_value_no_recursive_invalidate() - self._invalidate_dependent() - - def get_user_value(self): - """Returns the value assigned to the symbol in a .config or via - Symbol.set_user_value() (provided the value was valid for the type of the - symbol). Returns None in case of no user value.""" - return self.user_val - - def get_name(self): - """Returns the name of the symbol.""" - return self.name - - def get_upper_bound(self): - """For string/hex/int symbols and for bool and tristate symbols that - cannot be modified (see is_modifiable()), returns None. - - Otherwise, returns the highest value the symbol can be set to with - Symbol.set_user_value() (that will not be truncated): one of "m" or "y", - arranged from lowest to highest. This corresponds to the highest value - the symbol could be given in e.g. the 'make menuconfig' interface. - - See also the tri_less*() and tri_greater*() functions, which could come - in handy.""" - if self.type != BOOL and self.type != TRISTATE: - return None - rev_dep = self.config._eval_expr(self.rev_dep) - # A bool selected to "m" gets promoted to "y" - if self.type == BOOL and rev_dep == "m": - rev_dep = "y" - vis = self._get_visibility() - if (tri_to_int[vis] - tri_to_int[rev_dep]) > 0: - return vis - return None - - def get_lower_bound(self): - """For string/hex/int symbols and for bool and tristate symbols that - cannot be modified (see is_modifiable()), returns None. - - Otherwise, returns the lowest value the symbol can be set to with - Symbol.set_user_value() (that will not be truncated): one of "n" or "m", - arranged from lowest to highest. This corresponds to the lowest value - the symbol could be given in e.g. the 'make menuconfig' interface. - - See also the tri_less*() and tri_greater*() functions, which could come - in handy.""" - if self.type != BOOL and self.type != TRISTATE: - return None - rev_dep = self.config._eval_expr(self.rev_dep) - # A bool selected to "m" gets promoted to "y" - if self.type == BOOL and rev_dep == "m": - rev_dep = "y" - if (tri_to_int[self._get_visibility()] - tri_to_int[rev_dep]) > 0: - return rev_dep - return None - - def get_assignable_values(self): - """For string/hex/int symbols and for bool and tristate symbols that - cannot be modified (see is_modifiable()), returns the empty list. - - Otherwise, returns a list containing the user values that can be - assigned to the symbol (that won't be truncated). Usage example: - - if "m" in sym.get_assignable_values(): - sym.set_user_value("m") - - This is basically a more convenient interface to - get_lower/upper_bound() when wanting to test if a particular tristate - value can be assigned.""" - if self.type != BOOL and self.type != TRISTATE: - return [] - rev_dep = self.config._eval_expr(self.rev_dep) - # A bool selected to "m" gets promoted to "y" - if self.type == BOOL and rev_dep == "m": - rev_dep = "y" - res = ["n", "m", "y"][tri_to_int[rev_dep] : - tri_to_int[self._get_visibility()] + 1] - return res if len(res) > 1 else [] - - def get_type(self): - """Returns the type of the symbol: one of UNKNOWN, BOOL, TRISTATE, - STRING, HEX, or INT. These are defined at the top level of the module, - so you'd do something like - - if sym.get_type() == kconfiglib.STRING: - ...""" - return self.type - - def get_visibility(self): - """Returns the visibility of the symbol: one of "n", "m" or "y". For - bool and tristate symbols, this is an upper bound on the value users - can set for the symbol. For other types of symbols, a visibility of "n" - means the user value will be ignored. A visibility of "n" corresponds - to not being visible in the 'make *config' interfaces. - - Example (assuming we're running with modules enabled -- i.e., MODULES - set to 'y'): - - # Assume this has been assigned 'n' - config N_SYM - tristate "N_SYM" - - # Assume this has been assigned 'm' - config M_SYM - tristate "M_SYM" - - # Has visibility 'n' - config A - tristate "A" - depends on N_SYM - - # Has visibility 'm' - config B - tristate "B" - depends on M_SYM - - # Has visibility 'y' - config C - tristate "C" - - # Has no prompt, and hence visibility 'n' - config D - tristate - - Having visibility be tri-valued ensures that e.g. a symbol cannot be - set to "y" by the user if it depends on a symbol with value "m", which - wouldn't be safe. - - You should probably look at get_lower/upper_bound(), - get_assignable_values() and is_modifiable() before using this.""" - return self._get_visibility() - - def get_parent(self): - """Returns the menu or choice statement that contains the symbol, or - None if the symbol is at the top level. Note that if statements are - treated as syntactic and do not have an explicit class - representation.""" - return self.parent - - def get_referenced_symbols(self, refs_from_enclosing = False): - """Returns the set() of all symbols referenced by this symbol. For - example, the symbol defined by - - config FOO - bool - prompt "foo" if A && B - default C if D - depends on E - select F if G - - references the symbols A through G. - - refs_from_enclosing (default: False) -- If True, the symbols - referenced by enclosing menus and if's will be - included in the result.""" - return self.all_referenced_syms if refs_from_enclosing else self.referenced_syms - - def get_selected_symbols(self): - """Returns the set() of all symbols X for which this symbol has a - 'select X' or 'select X if Y' (regardless of whether Y is satisfied or - not). This is a subset of the symbols returned by - get_referenced_symbols().""" - return self.selected_syms - - def get_help(self): - """Returns the help text of the symbol, or None if the symbol has no - help text.""" - return self.help - - def get_config(self): - """Returns the Config instance this symbol is from.""" - return self.config - - def get_def_locations(self): - """Returns a list of (filename, linenr) tuples, where filename (string) - and linenr (int) represent a location where the symbol is defined. For - the vast majority of symbols this list will only contain one element. - For the following Kconfig, FOO would get two entries: the lines marked - with *. - - config FOO * - bool "foo prompt 1" - - config FOO * - bool "foo prompt 2" - """ - return self.def_locations - - def get_ref_locations(self): - """Returns a list of (filename, linenr) tuples, where filename (string) - and linenr (int) represent a location where the symbol is referenced in - the configuration. For example, the lines marked by * would be included - for FOO below: - - config A - bool - default BAR || FOO * - - config B - tristate - depends on FOO * - default m if FOO * - - if FOO * - config A - bool "A" - endif - - config FOO (definition not included) - bool - """ - return self.ref_locations - - def is_modifiable(self): - """Returns True if the value of the symbol could be modified by calling - Symbol.set_user_value() and False otherwise. - - For bools and tristates, this corresponds to the symbol being visible - in the 'make menuconfig' interface and not already being pinned to a - specific value (e.g. because it is selected by another symbol). - - For strings and numbers, this corresponds to just being visible. (See - Symbol.get_visibility().)""" - if self.is_special_: - return False - if self.type == BOOL or self.type == TRISTATE: - rev_dep = self.config._eval_expr(self.rev_dep) - # A bool selected to "m" gets promoted to "y" - if self.type == BOOL and rev_dep == "m": - rev_dep = "y" - return (tri_to_int[self._get_visibility()] - - tri_to_int[rev_dep]) > 0 - return self._get_visibility() != "n" - - def is_defined(self): - """Returns False if the symbol is referred to in the Kconfig but never - actually defined, otherwise True.""" - return self.is_defined_ - - def is_special(self): - """Returns True if the symbol is one of the special symbols n, m, y, or - UNAME_RELEASE, or gets its value from the environment. Otherwise, - returns False.""" - return self.is_special_ - - def is_from_environment(self): - """Returns True if the symbol gets its value from the environment. - Otherwise, returns False.""" - return self.is_from_env - - def has_ranges(self): - """Returns True if the symbol is of type INT or HEX and has ranges that - limits what values it can take on, otherwise False.""" - return self.ranges != [] - - def is_choice_symbol(self): - """Returns True if the symbol is in a choice statement and is an actual - choice symbol (see Choice.get_symbols()); otherwise, returns - False.""" - return self.is_choice_symbol_ - - def is_choice_selection(self): - """Returns True if the symbol is contained in a choice statement and is - the selected item, otherwise False. Equivalent to 'sym.is_choice_symbol() - and sym.get_parent().get_selection() is sym'.""" - return self.is_choice_symbol_ and self.parent.get_selection() is self - - def __str__(self): - """Returns a string containing various information about the symbol.""" - return self.config._get_sym_or_choice_str(self) - - # - # Private methods - # - - def __init__(self): - """Symbol constructor -- not intended to be called directly by - kconfiglib clients.""" - - # Set default values - _HasVisibility.__init__(self) - - self.config = None - - self.parent = None - self.name = None - self.type = UNKNOWN - - self.def_exprs = [] - self.ranges = [] - self.rev_dep = "n" - - # The prompt, default value and select conditions without any - # dependencies from menus or if's propagated to them - - self.orig_prompts = [] - self.orig_def_exprs = [] - self.orig_selects = [] - - # Dependencies inherited from containing menus and if's - self.deps_from_containing = None - - self.help = None - - # The set of symbols referenced by this symbol (see - # get_referenced_symbols()) - self.referenced_syms = set() - - # The set of symbols selected by this symbol (see - # get_selected_symbols()) - self.selected_syms = set() - - # Like 'referenced_syms', but includes symbols from - # dependencies inherited from enclosing menus and if's - self.all_referenced_syms = set() - - # This is set to True for "actual" choice symbols. See - # Choice._determine_actual_symbols(). The trailing underscore avoids a - # collision with is_choice_symbol(). - self.is_choice_symbol_ = False - - # This records only dependencies specified with 'depends on'. Needed - # when determining actual choice items (hrrrr...). See also - # Choice._determine_actual_symbols(). - self.menu_dep = None - - # See Symbol.get_ref/def_locations(). - self.def_locations = [] - self.ref_locations = [] - - self.user_val = None - - # Flags - - # Should the symbol get an entry in .config? - self.write_to_conf = False - - # Caches the calculated value - self.cached_value = None - - # Note: An instance variable 'self.dep' gets set on the Symbol in - # Config._build_dep(), linking the symbol to the symbols that - # immediately depend on it (in a caching/invalidation sense). The total - # set of dependent symbols for the symbol (the transitive closure) is - # calculated on an as-needed basis in _get_dependent(). - - # Caches the total list of dependent symbols. Calculated in - # _get_dependent(). - self.cached_deps = None - - # Does the symbol have an entry in the Kconfig file? The trailing - # underscore avoids a collision with is_defined(). - self.is_defined_ = False - - # Does the symbol get its value in some special way, e.g. from the - # environment or by being one of the special symbols n, m, and y? If - # so, the value is stored in self.cached_value, which is never - # invalidated. The trailing underscore avoids a collision with - # is_special(). - self.is_special_ = False - - # Does the symbol get its value from the environment? - self.is_from_env = False - - def _invalidate(self): - if self.is_special_: - return - - if self.is_choice_symbol_: - self.parent._invalidate() - - _HasVisibility._invalidate(self) - - self.write_to_conf = False - self.cached_value = None - - def _invalidate_dependent(self): - for sym in self._get_dependent(): - sym._invalidate() - - def _set_user_value_no_invalidate(self, v, suppress_load_warnings): - """Like set_user_value(), but does not invalidate any symbols. - - suppress_load_warnings -- - some warnings are annoying when loading a .config that can be helpful - when manually invoking set_user_value(). This flag is set to True to - suppress such warnings. - - Perhaps this could be made optional for load_config() instead.""" - - if self.is_special_: - if self.is_from_env: - self.config._warn('attempt to assign the value "{0}" to the ' - 'symbol {1}, which gets its value from the ' - 'environment. Assignment ignored.' - .format(v, self.name)) - else: - self.config._warn('attempt to assign the value "{0}" to the ' - 'special symbol {1}. Assignment ignored.' - .format(v, self.name)) - - return - - - if not self.is_defined_: - filename, linenr = self.ref_locations[0] - - self.config._undef_assign('attempt to assign the value "{0}" to {1}, ' - "which is referenced at {2}:{3} but never " - "defined. Assignment ignored." - .format(v, self.name, filename, linenr)) - return - - # Check if the value is valid for our type - - if not (( self.type == BOOL and (v == "n" or v == "y") ) or - ( self.type == TRISTATE and (v == "n" or v == "m" or - v == "y") ) or - ( self.type == STRING ) or - ( self.type == INT and _is_base_n(v, 10) ) or - ( self.type == HEX and _is_base_n(v, 16) )): - - self.config._warn('the value "{0}" is invalid for {1}, which has type {2}. ' - "Assignment ignored." - .format(v, self.name, typename[self.type])) - return - - if self.prompts == [] and not suppress_load_warnings: - self.config._warn('assigning "{0}" to the symbol {1} which ' - 'lacks prompts and thus has visibility "n". ' - 'The assignment will have no effect.' - .format(v, self.name)) - - self.user_val = v - - if self.is_choice_symbol_ and (self.type == BOOL or - self.type == TRISTATE): - choice = self.parent - if v == "y": - choice.user_val = self - choice.user_mode = "y" - elif v == "m": - choice.user_val = None - choice.user_mode = "m" - - def _unset_user_value_no_recursive_invalidate(self): - self._invalidate() - self.user_val = None - - if self.is_choice_symbol_: - self.parent._unset_user_value() - - def _make_conf(self): - if self.already_written: - return [] - - self.already_written = True - - # Note: write_to_conf is determined in get_value() - val = self.get_value() - if not self.write_to_conf: - return [] - - if self.type == BOOL or self.type == TRISTATE: - if val == "m" or val == "y": - return ["CONFIG_{0}={1}".format(self.name, val)] - return ["# CONFIG_{0} is not set".format(self.name)] - - elif self.type == STRING: - # Escape \ and " - return ['CONFIG_{0}="{1}"' - .format(self.name, - val.replace("\\", "\\\\").replace('"', '\\"'))] - - elif self.type == INT or self.type == HEX: - return ["CONFIG_{0}={1}".format(self.name, val)] - - else: - _internal_error('Internal error while creating .config: unknown type "{0}".' - .format(self.type)) - - def _get_dependent(self): - """Returns the set of symbols that should be invalidated if the value - of the symbol changes, because they might be affected by the change. - Note that this is an internal API -- it's probably of limited - usefulness to clients.""" - if self.cached_deps is not None: - return self.cached_deps - - res = set() - - self._add_dependent_ignore_siblings(res) - if self.is_choice_symbol_: - for s in self.parent.get_symbols(): - if s is not self: - res.add(s) - s._add_dependent_ignore_siblings(res) - - self.cached_deps = res - return res - - def _add_dependent_ignore_siblings(self, to): - """Calculating dependencies gets a bit tricky for choice items as they - all depend on each other, potentially leading to infinite recursion. - This helper function calculates dependencies ignoring the other symbols - in the choice. It also works fine for symbols that are not choice - items.""" - for s in self.dep: - to.add(s) - to |= s._get_dependent() - - def _has_auto_menu_dep_on(self, on): - """See Choice._determine_actual_symbols().""" - if not isinstance(self.parent, Choice): - _internal_error("Attempt to determine auto menu dependency for symbol ouside of choice.") - - if self.prompts == []: - # If we have no prompt, use the menu dependencies instead (what was - # specified with 'depends on') - return self.menu_dep is not None and \ - self.config._expr_depends_on(self.menu_dep, on) - - for (_, cond_expr) in self.prompts: - if self.config._expr_depends_on(cond_expr, on): - return True - - return False - -class Menu(Item): - - """Represents a menu statement.""" - - # - # Public interface - # - - def get_config(self): - """Return the Config instance this menu is from.""" - return self.config - - def get_visibility(self): - """Returns the visibility of the menu. This also affects the visibility - of subitems. See also Symbol.get_visibility().""" - return self.config._eval_expr(self.dep_expr) - - def get_visible_if_visibility(self): - """Returns the visibility the menu gets from its 'visible if' - condition. "y" if the menu has no 'visible if' condition.""" - return self.config._eval_expr(self.visible_if_expr) - - def get_items(self, recursive = False): - """Returns a list containing the items (symbols, menus, choice - statements and comments) in in the menu, in the same order that the - items appear within the menu. - - recursive (default: False) -- True if items contained in items within - the menu should be included - recursively (preorder).""" - - if not recursive: - return self.block.get_items() - - res = [] - for item in self.block.get_items(): - res.append(item) - if isinstance(item, Menu): - res.extend(item.get_items(True)) - elif isinstance(item, Choice): - res.extend(item.get_items()) - return res - - def get_symbols(self, recursive = False): - """Returns a list containing the symbols in the menu, in the same order - that they appear within the menu. - - recursive (default: False) -- True if symbols contained in items within - the menu should be included - recursively.""" - - return [item for item in self.get_items(recursive) if isinstance(item, Symbol)] - - def get_title(self): - """Returns the title text of the menu.""" - return self.title - - def get_parent(self): - """Returns the menu or choice statement that contains the menu, or - None if the menu is at the top level. Note that if statements are - treated as syntactic sugar and do not have an explicit class - representation.""" - return self.parent - - def get_referenced_symbols(self, refs_from_enclosing = False): - """See Symbol.get_referenced_symbols().""" - return self.all_referenced_syms if refs_from_enclosing else self.referenced_syms - - def get_location(self): - """Returns the location of the menu as a (filename, linenr) tuple, - where filename is a string and linenr an int.""" - return (self.filename, self.linenr) - - def __str__(self): - """Returns a string containing various information about the menu.""" - depends_on_str = self.config._expr_val_str(self.orig_deps, - "(no dependencies)") - visible_if_str = self.config._expr_val_str(self.visible_if_expr, - "(no dependencies)") - - additional_deps_str = " " + self.config._expr_val_str(self.deps_from_containing, - "(no additional dependencies)") - - return _sep_lines("Menu", - "Title : " + self.title, - "'depends on' dependencies : " + depends_on_str, - "'visible if' dependencies : " + visible_if_str, - "Additional dependencies from enclosing menus and if's:", - additional_deps_str, - "Location: {0}:{1}".format(self.filename, self.linenr)) - - # - # Private methods - # - - def __init__(self): - """Menu constructor -- not intended to be called directly by - kconfiglib clients.""" - - self.config = None - - self.parent = None - self.title = None - self.block = None - self.dep_expr = None - - # Dependency expression without dependencies from enclosing menus and - # if's propagated - self.orig_deps = None - - # Dependencies inherited from containing menus and if's - self.deps_from_containing = None - - # The 'visible if' expression - self.visible_if_expr = None - - # The set of symbols referenced by this menu (see - # get_referenced_symbols()) - self.referenced_syms = set() - - # Like 'referenced_syms', but includes symbols from - # dependencies inherited from enclosing menus and if's - self.all_referenced_syms = None - - self.filename = None - self.linenr = None - - def _make_conf(self): - item_conf = self.block._make_conf() - - if self.config._eval_expr(self.dep_expr) != "n" and \ - self.config._eval_expr(self.visible_if_expr) != "n": - return ["\n#\n# {0}\n#".format(self.title)] + item_conf - return item_conf - -class Choice(Item, _HasVisibility): - - """Represents a choice statement. A choice can be in one of three modes: - - "n" - The choice is not visible and no symbols can be selected. - - "m" - Any number of symbols can be set to "m". The rest will be "n". This - is safe since potentially conflicting options don't actually get - compiled into the kernel simultaneously with "m". - - "y" - One symbol will be "y" while the rest are "n". - - Only tristate choices can be in "m" mode, and the visibility of the choice - is an upper bound on the mode, so that e.g. a choice that depends on a - symbol with value "m" will be in "m" mode. - - The mode changes automatically when a value is assigned to a symbol within - the choice. - - See Symbol.get_visibility() too.""" - - # - # Public interface - # - - def get_selection(self): - """Returns the symbol selected (either by the user or through - defaults), or None if either no symbol is selected or the mode is not - "y".""" - if self.cached_selection is not None: - if self.cached_selection == NO_SELECTION: - return None - return self.cached_selection - - if self.get_mode() != "y": - return self._cache_ret(None) - - # User choice available? - if self.user_val is not None and \ - self.user_val._get_visibility() == "y": - return self._cache_ret(self.user_val) - - if self.optional: - return self._cache_ret(None) - - return self._cache_ret(self.get_selection_from_defaults()) - - def get_selection_from_defaults(self): - """Like Choice.get_selection(), but acts as if no symbol has been - selected by the user and no 'optional' flag is in effect.""" - - if self.actual_symbols == []: - return None - - for (symbol, cond_expr) in self.def_exprs: - if self.config._eval_expr(cond_expr) != "n": - chosen_symbol = symbol - break - else: - chosen_symbol = self.actual_symbols[0] - - # Is the chosen symbol visible? - if chosen_symbol._get_visibility() != "n": - return chosen_symbol - # Otherwise, pick the first visible symbol - for sym in self.actual_symbols: - if sym._get_visibility() != "n": - return sym - return None - - def get_user_selection(self): - """If the choice is in "y" mode and has a user-selected symbol, returns - that symbol. Otherwise, returns None.""" - return self.user_val - - def get_config(self): - """Returns the Config instance this choice is from.""" - return self.config - - def get_name(self): - """For named choices, returns the name. Returns None for unnamed - choices. No named choices appear anywhere in the kernel Kconfig files - as of Linux 3.7.0-rc8.""" - return self.name - - def get_help(self): - """Returns the help text of the choice, or None if the choice has no - help text.""" - return self.help - - def get_type(self): - """Returns the type of the choice. See Symbol.get_type().""" - return self.type - - def get_items(self): - """Gets all items contained in the choice in the same order as within - the configuration ("items" instead of "symbols" since choices and - comments might appear within choices. This only happens in one place as - of Linux 3.7.0-rc8, in drivers/usb/gadget/Kconfig).""" - return self.block.get_items() - - def get_symbols(self): - """Returns a list containing the choice's symbols. - - A quirk (perhaps a bug) of Kconfig is that you can put items within a - choice that will not be considered members of the choice insofar as - selection is concerned. This happens for example if one symbol within a - choice 'depends on' the symbol preceding it, or if you put non-symbol - items within choices. - - As of Linux 3.7.0-rc8, this seems to be used intentionally in one - place: drivers/usb/gadget/Kconfig. - - This function returns the "proper" symbols of the choice in the order - they appear in the choice, excluding such items. If you want all items - in the choice, use get_items().""" - return self.actual_symbols - - def get_parent(self): - """Returns the menu or choice statement that contains the choice, or - None if the choice is at the top level. Note that if statements are - treated as syntactic sugar and do not have an explicit class - representation.""" - return self.parent - - def get_referenced_symbols(self, refs_from_enclosing = False): - """See Symbol.get_referenced_symbols().""" - return self.all_referenced_syms if refs_from_enclosing else self.referenced_syms - - def get_def_locations(self): - """Returns a list of (filename, linenr) tuples, where filename (string) - and linenr (int) represent a location where the choice is defined. For - the vast majority of choices (all of them as of Linux 3.7.0-rc8) this - list will only contain one element, but its possible for named choices - to be defined in multiple locations.""" - return self.def_locations - - def get_visibility(self): - """Returns the visibility of the choice statement: one of "n", "m" or - "y". This acts as an upper limit on the mode of the choice (though bool - choices can only have the mode "y"). See the class documentation for an - explanation of modes.""" - return self._get_visibility() - - def get_mode(self): - """Returns the mode of the choice. See the class documentation for - an explanation of modes.""" - minimum_mode = "n" if self.optional else "m" - mode = self.user_mode if self.user_mode is not None else minimum_mode - mode = self.config._eval_min(mode, self._get_visibility()) - - # Promote "m" to "y" for boolean choices - if mode == "m" and self.type == BOOL: - return "y" - - return mode - - def is_optional(self): - """Returns True if the symbol has the optional flag set (and so will default - to "n" mode). Otherwise, returns False.""" - return self.optional - - def __str__(self): - """Returns a string containing various information about the choice - statement.""" - return self.config._get_sym_or_choice_str(self) - - # - # Private methods - # - - def __init__(self): - """Choice constructor -- not intended to be called directly by - kconfiglib clients.""" - - _HasVisibility.__init__(self) - - self.config = None - - self.parent = None - self.name = None # Yes, choices can be named - self.type = UNKNOWN - self.def_exprs = [] - self.help = None - self.optional = False - self.block = None - - # The prompts and default values without any dependencies from - # enclosing menus or if's propagated - - self.orig_prompts = [] - self.orig_def_exprs = [] - - # Dependencies inherited from containing menus and if's - self.deps_from_containing = None - - # We need to filter out symbols that appear within the choice block but - # are not considered choice items (see - # Choice._determine_actual_symbols()) This list holds the "actual" choice - # items. - self.actual_symbols = [] - - # The set of symbols referenced by this choice (see - # get_referenced_symbols()) - self.referenced_syms = set() - - # Like 'referenced_syms', but includes symbols from - # dependencies inherited from enclosing menus and if's - self.all_referenced_syms = set() - - # See Choice.get_def_locations() - self.def_locations = [] - - self.user_val = None - self.user_mode = None - - self.cached_selection = None - - def _determine_actual_symbols(self): - """If a symbol's visibility depends on the preceding symbol within a - choice, it is no longer viewed as a choice item (quite possibly a bug, - but some things consciously use it.. ugh. It stems from automatic - submenu creation). In addition, it's possible to have choices and - comments within choices, and those shouldn't be considered as choice - items either. Only drivers/usb/gadget/Kconfig seems to depend on any of - this. This method computes the "actual" items in the choice and sets - the is_choice_symbol_ flag on them (retrieved via is_choice_symbol()). - - Don't let this scare you: an earlier version simply checked for a - sequence of symbols where all symbols after the first appeared in the - 'depends on' expression of the first, and that worked fine. The added - complexity is to be future-proof in the event that - drivers/usb/gadget/Kconfig turns even more sinister. It might very well - be overkilling things (especially if that file is refactored ;).""" - - items = self.block.get_items() - - # Items might depend on each other in a tree structure, so we need a - # stack to keep track of the current tentative parent - stack = [] - - for item in items: - if not isinstance(item, Symbol): - stack = [] - continue - - while stack != []: - if item._has_auto_menu_dep_on(stack[-1]): - # The item should not be viewed as a choice item, so don't - # set item.is_choice_symbol_. - stack.append(item) - break - else: - stack.pop() - else: - item.is_choice_symbol_ = True - self.actual_symbols.append(item) - stack.append(item) - - def _cache_ret(self, selection): - # As None is used to indicate the lack of a cached value we can't use - # that to cache the fact that the choice has no selection. Instead, we - # use the symbolic constant NO_SELECTION. - if selection is None: - self.cached_selection = NO_SELECTION - else: - self.cached_selection = selection - - return selection - - def _invalidate(self): - _HasVisibility._invalidate(self) - self.cached_selection = None - - def _unset_user_value(self): - self._invalidate() - self.user_val = None - self.user_mode = None - - def _make_conf(self): - return self.block._make_conf() - -class Comment(Item): - - """Represents a comment statement.""" - - # - # Public interface - # - - def get_config(self): - """Returns the Config instance this comment is from.""" - return self.config - - def get_visibility(self): - """Returns the visibility of the comment. See also - Symbol.get_visibility().""" - return self.config._eval_expr(self.dep_expr) - - def get_text(self): - """Returns the text of the comment.""" - return self.text - - def get_parent(self): - """Returns the menu or choice statement that contains the comment, or - None if the comment is at the top level. Note that if statements are - treated as syntactic sugar and do not have an explicit class - representation.""" - return self.parent - - def get_referenced_symbols(self, refs_from_enclosing = False): - """See Symbol.get_referenced_symbols().""" - return self.all_referenced_syms if refs_from_enclosing else self.referenced_syms - - def get_location(self): - """Returns the location of the comment as a (filename, linenr) tuple, - where filename is a string and linenr an int.""" - return (self.filename, self.linenr) - - def __str__(self): - """Returns a string containing various information about the comment.""" - dep_str = self.config._expr_val_str(self.orig_deps, "(no dependencies)") - - additional_deps_str = " " + self.config._expr_val_str(self.deps_from_containing, - "(no additional dependencies)") - - return _sep_lines("Comment", - "Text: " + str(self.text), - "Dependencies: " + dep_str, - "Additional dependencies from enclosing menus and if's:", - additional_deps_str, - "Location: {0}:{1}".format(self.filename, self.linenr)) - - # - # Private methods - # - - def __init__(self): - """Comment constructor -- not intended to be called directly by - kconfiglib clients.""" - - self.config = None - - self.parent = None - self.text = None - self.dep_expr = None - - # Dependency expression without dependencies from enclosing menus and - # if's propagated - self.orig_deps = None - - # Dependencies inherited from containing menus and if's - self.deps_from_containing = None - - # The set of symbols referenced by this comment (see - # get_referenced_symbols()) - self.referenced_syms = set() - - # Like 'referenced_syms', but includes symbols from - # dependencies inherited from enclosing menus and if's - self.all_referenced_syms = None - - self.filename = None - self.linenr = None - - def _make_conf(self): - if self.config._eval_expr(self.dep_expr) != "n": - return ["\n#\n# {0}\n#".format(self.text)] - return [] - -class _Feed: - - """Class for working with sequences in a stream-like fashion; handy for tokens.""" - - def __init__(self, items): - self.items = items - self.length = len(self.items) - self.i = 0 - - def get_next(self): - if self.i >= self.length: - return None - - item = self.items[self.i] - self.i += 1 - return item - - def peek_next(self): - return None if self.i >= self.length else self.items[self.i] - - def go_to_start(self): - self.i = 0 - - def __getitem__(self, index): - return self.items[index] - - def __len__(self): - return len(self.items) - - def is_empty(self): - return self.items == [] - - def check(self, token): - """Check if the next token is 'token'. If so, remove it from the token - feed and return True. Otherwise, leave it in and return False.""" - if self.i >= self.length: - return None - - if self.items[self.i] == token: - self.i += 1 - return True - - return False - - def remove_while(self, pred): - while self.i < self.length and pred(self.items[self.i]): - self.i += 1 - - def go_back(self): - if self.i <= 0: - _internal_error("Attempt to move back in Feed while already at the beginning.") - self.i -= 1 - -class _FileFeed(_Feed): - - """Feed subclass that keeps track of the current filename and line - number.""" - - def __init__(self, lines, filename): - self.filename = _clean_up_path(filename) - _Feed.__init__(self, lines) - - def get_filename(self): - return self.filename - - def get_linenr(self): - return self.i - -# -# Misc. public global utility functions -# - -def tri_less(v1, v2): - """Returns True if the tristate v1 is less than the tristate v2, where "n", - "m" and "y" are ordered from lowest to highest. Otherwise, returns - False.""" - return tri_to_int[v1] < tri_to_int[v2] - -def tri_less_eq(v1, v2): - """Returns True if the tristate v1 is less than or equal to the tristate - v2, where "n", "m" and "y" are ordered from lowest to highest. Otherwise, - returns False.""" - return tri_to_int[v1] <= tri_to_int[v2] - -def tri_greater(v1, v2): - """Returns True if the tristate v1 is greater than the tristate v2, where - "n", "m" and "y" are ordered from lowest to highest. Otherwise, returns - False.""" - return tri_to_int[v1] > tri_to_int[v2] - -def tri_greater_eq(v1, v2): - """Returns True if the tristate v1 is greater than or equal to the tristate - v2, where "n", "m" and "y" are ordered from lowest to highest. Otherwise, - returns False.""" - return tri_to_int[v1] >= tri_to_int[v2] - -# -# Helper functions, mostly related to text processing -# - -def _strip_quotes(s, line, filename, linenr): - """Removes any quotes surrounding 's' if it has them; otherwise returns 's' - unmodified.""" - s = s.strip() - if not s: - return "" - if s[0] == '"' or s[0] == "'": - if len(s) < 2 or s[-1] != s[0]: - _parse_error(line, - "malformed string literal", - filename, - linenr) - return s[1:-1] - return s - -def _indentation(line): - """Returns the indentation of the line, treating tab stops as being spaced - 8 characters apart.""" - if line.isspace(): - _internal_error("Attempt to take indentation of blank line.") - indent = 0 - for c in line: - if c == " ": - indent += 1 - elif c == "\t": - # Go to the next tab stop - indent = (indent + 8) & ~7 - else: - return indent - -def _deindent(line, indent): - """Deindent 'line' by 'indent' spaces.""" - line = line.expandtabs() - if len(line) <= indent: - return line - return line[indent:] - -def _is_base_n(s, n): - try: - int(s, n) - return True - except ValueError: - return False - -def _sep_lines(*args): - """Returns a string comprised of all arguments, with newlines inserted - between them.""" - return "\n".join(args) - -def _comment(s): - """Returns a new string with "#" inserted before each line in 's'.""" - if not s: - return "#" - res = "".join(["#" + line for line in s.splitlines(True)]) - if s.endswith("\n"): - return res + "#" - return res - -def _get_lines(filename): - """Returns a list of lines from 'filename', joining any line ending in \\ - with the following line.""" - with open(filename, "r") as f: - lines = [] - accum = "" - while 1: - line = f.readline() - - if line == "": - return lines - - if line.endswith("\\\n"): - accum += line[:-2] - else: - accum += line - lines.append(accum) - accum = "" - -def _strip_trailing_slash(path): - """Removes any trailing slash from 'path'.""" - return path[:-1] if path.endswith("/") else path - -def _clean_up_path(path): - """Strips any initial "./" and trailing slash from 'path'.""" - if path.startswith("./"): - path = path[2:] - return _strip_trailing_slash(path) - -# -# Error handling -# - -class Kconfig_Syntax_Error(Exception): - """Exception raised for syntax errors.""" - pass - -class Internal_Error(Exception): - """Exception raised for internal errors.""" - pass - -def _tokenization_error(s, index, filename, linenr): - if filename is not None: - assert linenr is not None - sys.stderr.write("{0}:{1}:\n".format(filename, linenr)) - - if s.endswith("\n"): - s = s[:-1] - - # Calculate the visual offset corresponding to index 'index' in 's' - # assuming tabstops are spaced 8 characters apart - vis_index = 0 - for c in s[:index]: - if c == "\t": - vis_index = (vis_index + 8) & ~7 - else: - vis_index += 1 - - # Don't output actual tabs to be independent of how the terminal renders - # them - s = s.expandtabs() - - raise Kconfig_Syntax_Error, ( - _sep_lines("Error during tokenization at location indicated by caret.\n", - s, - " " * vis_index + "^\n")) - -def _parse_error(s, msg, filename, linenr): - error_str = "" - - if filename is not None: - assert linenr is not None - error_str += "{0}:{1}: ".format(filename, linenr) - - if s.endswith("\n"): - s = s[:-1] - - error_str += 'Error while parsing "{0}"'.format(s) + \ - ("." if msg is None else ": " + msg) - - raise Kconfig_Syntax_Error, error_str - -def _internal_error(msg): - msg += "\nSorry! You may want to send an email to kconfiglib@gmail.com " \ - "to tell me about this. Include the message above and the stack " \ - "trace and describe what you were doing." - - raise Internal_Error, msg - -if use_psyco: - import psyco - - Config._tokenize = psyco.proxy(Config._tokenize) - Config._eval_expr = psyco.proxy(Config._eval_expr) - - _indentation = psyco.proxy(_indentation) - _get_lines = psyco.proxy(_get_lines) diff --git a/support/scripts/mkmakefile b/support/scripts/mkmakefile index 27b15073f3..c7569f2ca5 100755 --- a/support/scripts/mkmakefile +++ b/support/scripts/mkmakefile @@ -15,13 +15,18 @@ if test -e $2/Makefile && ! grep -q Automatically $2/Makefile then exit 0 fi -if [ "${quiet}" != "silent_" ]; then - echo " GEN $2/Makefile" -fi +echo " GEN $2/Makefile" cat << EOF > $2/Makefile # Automatically generated by $0: don't edit +ifeq ("\$(origin V)", "command line") +VERBOSE := \$(V) +endif +ifneq (\$(VERBOSE),1) +Q := @ +endif + lastword = \$(word \$(words \$(1)),\$(1)) makedir := \$(dir \$(call lastword,\$(MAKEFILE_LIST))) @@ -30,12 +35,12 @@ MAKEARGS += O=\$(if \$(patsubst /%,,\$(makedir)),\$(CURDIR)/)\$(patsubst %/,%,\$ MAKEFLAGS += --no-print-directory -.PHONY: all \$(MAKECMDGOALS) +.PHONY: _all \$(MAKECMDGOALS) all := \$(filter-out Makefile,\$(MAKECMDGOALS)) _all: - \$(MAKE) \$(MAKEARGS) \$(all) + \$(Q)umask 0022 && \$(MAKE) \$(MAKEARGS) \$(all) Makefile:; diff --git a/support/scripts/mkusers b/support/scripts/mkusers index 8f2e76fb30..d00ba33823 100755 --- a/support/scripts/mkusers +++ b/support/scripts/mkusers @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e myname="${0##*/}" @@ -35,7 +35,7 @@ PASSWD="${TARGET_DIR}/etc/passwd" SHADOW="${TARGET_DIR}/etc/shadow" GROUP="${TARGET_DIR}/etc/group" # /etc/gshadow is not part of the standard skeleton, so not everybody -# will have it, but some may hav it, and its content must be in sync +# will have it, but some may have it, and its content must be in sync # with /etc/group, so any use of gshadow must be conditional. GSHADOW="${TARGET_DIR}/etc/gshadow" @@ -43,7 +43,7 @@ GSHADOW="${TARGET_DIR}/etc/gshadow" # such as: # BR2_DEFCONFIG="$(CONFIG_DIR)/defconfig" # which when sourced from a shell script will eventually try to execute -# a command name 'CONFIG_DIR', which is plain wrong for virtually every +# a command named 'CONFIG_DIR', which is plain wrong for virtually every # systems out there. # So, we have to scan that file instead. Sigh... :-( PASSWD_METHOD="$( sed -r -e '/^BR2_TARGET_GENERIC_PASSWD_METHOD="(.*)"$/!d;' \ @@ -75,6 +75,14 @@ get_gid() { '$1 == group { printf( "%d\n", $3 ); }' "${GROUP}" } +#---------------------------------------------------------------------------- +get_members() { + local group="${1}" + + awk -F: -v group="${group}" \ + '$1 == group { printf( "%s\n", $4 ); }' "${GROUP}" +} + #---------------------------------------------------------------------------- get_username() { local uid="${1}" @@ -129,47 +137,53 @@ check_user_validity() { fi if [ ${gid} -lt -1 -o ${gid} -eq 0 ]; then - fail "invalid gid '%d'\n" ${gid} + fail "invalid gid '%d' for '%s'\n" ${gid} "${username}" elif [ ${gid} -ne -1 ]; then # check the gid is not already used for another group if [ -n "${_group}" -a "${_group}" != "${group}" ]; then - fail "gid is already used by group '${_group}'\n" + fail "gid '%d' for '%s' is already used by group '%s'\n" \ + ${gid} "${username}" "${_group}" fi # check the group does not already exists with another gid # Need to split the check in two, otherwise '[' complains it # is missing arguments when _gid is empty if [ -n "${_gid}" ] && [ ${_gid} -ne ${gid} ]; then - fail "group already exists with gid '${_gid}'\n" + fail "group '%s' for '%s' already exists with gid '%d' (wants '%d')\n" \ + "${group}" "${username}" ${_gid} ${gid} fi # check the user does not already exists with another gid # Need to split the check in two, otherwise '[' complains it # is missing arguments when _ugid is empty if [ -n "${_ugid}" ] && [ ${_ugid} -ne ${gid} ]; then - fail "user already exists with gid '${_ugid}'\n" + fail "user '%s' already exists with gid '%d' (wants '%d')\n" \ + "${username}" ${_ugid} ${gid} fi fi if [ ${uid} -lt -1 -o ${uid} -eq 0 ]; then - fail "invalid uid '%d'\n" ${uid} + fail "invalid uid '%d' for '%s'\n" ${uid} "${username}" elif [ ${uid} -ne -1 ]; then # check the uid is not already used for another user if [ -n "${_username}" -a "${_username}" != "${username}" ]; then - fail "uid is already used by user '${_username}'\n" + fail "uid '%d' for '%s' already used by user '%s'\n" \ + ${uid} "${username}" "${_username}" fi # check the user does not already exists with another uid # Need to split the check in two, otherwise '[' complains it # is missing arguments when _uid is empty if [ -n "${_uid}" ] && [ ${_uid} -ne ${uid} ]; then - fail "user already exists with uid '${_uid}'\n" + fail "user '%s' already exists with uid '%d' (wants '%d')\n" \ + "${username}" ${_uid} ${uid} fi fi # check the user does not already exist in another group if [ -n "${_ugroup}" -a "${_ugroup}" != "${group}" ]; then - fail "user already exists with group '${_ugroup}'\n" + fail "user '%s' already exists with group '%s' (wants '%s')\n" \ + "${username}" "${_ugroup}" "${group}" fi return 0 @@ -205,20 +219,21 @@ generate_gid() { add_one_group() { local group="${1}" local gid="${2}" - local _f + local members # Generate a new GID if needed if [ ${gid} -eq -1 ]; then gid="$( generate_gid "${group}" )" fi + members=$(get_members "$group") # Remove any previous instance of this group, and re-add the new one - sed -i -e '/^'"${group}"':.*/d;' "${GROUP}" - printf "%s:x:%d:\n" "${group}" "${gid}" >>"${GROUP}" + sed -i --follow-symlinks -e '/^'"${group}"':.*/d;' "${GROUP}" + printf "%s:x:%d:%s\n" "${group}" "${gid}" "${members}" >>"${GROUP}" # Ditto for /etc/gshadow if it exists if [ -f "${GSHADOW}" ]; then - sed -i -e '/^'"${group}"':.*/d;' "${GSHADOW}" + sed -i --follow-symlinks -e '/^'"${group}"':.*/d;' "${GSHADOW}" printf "%s:*::\n" "${group}" >>"${GSHADOW}" fi } @@ -257,7 +272,8 @@ add_user_to_group() { for _f in "${GROUP}" "${GSHADOW}"; do [ -f "${_f}" ] || continue - sed -r -i -e 's/^('"${group}"':.*:)(([^:]+,)?)'"${username}"'(,[^:]+*)?$/\1\2\4/;' \ + sed -r -i --follow-symlinks \ + -e 's/^('"${group}"':.*:)(([^:]+,)?)'"${username}"'(,[^:]+*)?$/\1\2\4/;' \ -e 's/^('"${group}"':.*)$/\1,'"${username}"'/;' \ -e 's/,+/,/' \ -e 's/:,/:/' \ @@ -297,9 +313,7 @@ add_one_user() { # Remove any previous instance of this user for _f in "${PASSWD}" "${SHADOW}"; do - if test -f "${_f}"; then - sed -r -i -e '/^'"${username}"':.*/d;' "${_f}" - fi + sed -r -i --follow-symlinks -e '/^'"${username}"':.*/d;' "${_f}" done _gid="$( get_gid "${group}" )" @@ -314,6 +328,9 @@ add_one_user() { *) fail "home must be an absolute path\n";; esac case "${passwd}" in + -) + _passwd="" + ;; !=*) _passwd='!'"$( encode_password "${passwd#!=}" )" ;; @@ -325,17 +342,13 @@ add_one_user() { ;; esac - if test -f "${PASSWD}"; then - printf "%s:x:%d:%d:%s:%s:%s\n" \ - "${username}" "${uid}" "${_gid}" \ - "${comment}" "${_home}" "${_shell}" \ - >>"${PASSWD}" - fi - if test -f "${SHADOW}"; then - printf "%s:%s:::::::\n" \ - "${username}" "${_passwd}" \ - >>"${SHADOW}" - fi + printf "%s:x:%d:%d:%s:%s:%s\n" \ + "${username}" "${uid}" "${_gid}" \ + "${comment}" "${_home}" "${_shell}" \ + >>"${PASSWD}" + printf "%s:%s:::::::\n" \ + "${username}" "${_passwd}" \ + >>"${SHADOW}" # Add the user to its additional groups if [ "${groups}" != "-" ]; then @@ -355,6 +368,8 @@ add_one_user() { #---------------------------------------------------------------------------- main() { local username uid group gid passwd home shell groups comment + local line + local -a ENTRIES # Some sanity checks if [ ${MIN_UID} -le 0 ]; then @@ -364,36 +379,41 @@ main() { fail "MIN_GID must be >0 (currently %d)\n" ${MIN_GID} fi + # Read in all the file in memory, exclude empty lines and comments + while read line; do + ENTRIES+=( "${line}" ) + done < <( sed -r -e 's/#.*//; /^[[:space:]]*$/d;' "${USERS_TABLE}" ) + # We first create groups whose gid is not -1, and then we create groups # whose gid is -1 (automatic), so that, if a group is defined both with # a specified gid and an automatic gid, we ensure the specified gid is # used, rather than a different automatic gid is computed. # First, create all the main groups which gid is *not* automatic - while read username uid group gid passwd home shell groups comment; do - [ -n "${username}" ] || continue # Package with no user - [ ${gid} -ge 0 ] || continue # Automatic gid + for line in "${ENTRIES[@]}"; do + read username uid group gid passwd home shell groups comment <<<"${line}" + [ ${gid} -ge 0 ] || continue # Automatic gid add_one_group "${group}" "${gid}" - done <"${USERS_TABLE}" + done # Then, create all the main groups which gid *is* automatic - while read username uid group gid passwd home shell groups comment; do - [ -n "${username}" ] || continue # Package with no user - [ ${gid} -eq -1 ] || continue # Non-automatic gid + for line in "${ENTRIES[@]}"; do + read username uid group gid passwd home shell groups comment <<<"${line}" + [ ${gid} -eq -1 ] || continue # Non-automatic gid add_one_group "${group}" "${gid}" - done <"${USERS_TABLE}" + done # Then, create all the additional groups # If any additional group is already a main group, we should use # the gid of that main group; otherwise, we can use any gid - while read username uid group gid passwd home shell groups comment; do - [ -n "${username}" ] || continue # Package with no user + for line in "${ENTRIES[@]}"; do + read username uid group gid passwd home shell groups comment <<<"${line}" if [ "${groups}" != "-" ]; then for g in ${groups//,/ }; do add_one_group "${g}" -1 done fi - done <"${USERS_TABLE}" + done # When adding users, we do as for groups, in case two packages create # the same user, one with an automatic uid, the other with a specified @@ -401,20 +421,22 @@ main() { # uid be generated. # Now, add users whose uid is *not* automatic - while read username uid group gid passwd home shell groups comment; do - [ -n "${username}" ] || continue # Package with no user - [ ${uid} -ge 0 ] || continue # Automatic uid + for line in "${ENTRIES[@]}"; do + read username uid group gid passwd home shell groups comment <<<"${line}" + [ "${username}" != "-" ] || continue # Magic string to skip user creation + [ ${uid} -ge 0 ] || continue # Automatic uid add_one_user "${username}" "${uid}" "${group}" "${gid}" "${passwd}" \ "${home}" "${shell}" "${groups}" "${comment}" - done <"${USERS_TABLE}" + done # Finally, add users whose uid *is* automatic - while read username uid group gid passwd home shell groups comment; do - [ -n "${username}" ] || continue # Package with no user - [ ${uid} -eq -1 ] || continue # Non-automatic uid + for line in "${ENTRIES[@]}"; do + read username uid group gid passwd home shell groups comment <<<"${line}" + [ "${username}" != "-" ] || continue # Magic string to skip user creation + [ ${uid} -eq -1 ] || continue # Non-automatic uid add_one_user "${username}" "${uid}" "${group}" "${gid}" "${passwd}" \ "${home}" "${shell}" "${groups}" "${comment}" - done <"${USERS_TABLE}" + done } #---------------------------------------------------------------------------- diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index f5d6ec8df5..5adda6df08 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env python # Copyright (C) 2009 by Thomas Petazzoni # @@ -16,16 +16,649 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# This script generates an HTML file that contains a report about all -# Buildroot packages, their usage of the different package -# infrastructure and possible cleanup actions -# -# Run the script from the Buildroot toplevel directory: -# -# ./support/scripts/pkg-stats > /tmp/pkg.html -# +import argparse +import datetime +import fnmatch +import os +from collections import defaultdict +import re +import subprocess +import requests # URL checking +import json +import ijson +import certifi +import distutils.version +import time +import gzip +import sys +from urllib3 import HTTPSConnectionPool +from urllib3.exceptions import HTTPError +from multiprocessing import Pool + +sys.path.append('utils/') +from getdeveloperlib import parse_developers # noqa: E402 + +NVD_START_YEAR = 2002 +NVD_JSON_VERSION = "1.0" +NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION + +INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)") +URL_RE = re.compile(r"\s*https?://\S*\s*$") + +RM_API_STATUS_ERROR = 1 +RM_API_STATUS_FOUND_BY_DISTRO = 2 +RM_API_STATUS_FOUND_BY_PATTERN = 3 +RM_API_STATUS_NOT_FOUND = 4 + +# Used to make multiple requests to the same host. It is global +# because it's used by sub-processes. +http_pool = None + + +class Defconfig: + def __init__(self, name, path): + self.name = name + self.path = path + self.developers = None + + def set_developers(self, developers): + """ + Fills in the .developers field + """ + self.developers = [ + developer.name + for developer in developers + if developer.hasfile(self.path) + ] + + +def get_defconfig_list(): + """ + Builds the list of Buildroot defconfigs, returning a list of Defconfig + objects. + """ + return [ + Defconfig(name[:-len('_defconfig')], os.path.join('configs', name)) + for name in os.listdir('configs') + if name.endswith('_defconfig') + ] + + +class Package: + all_licenses = dict() + all_license_files = list() + all_versions = dict() + all_ignored_cves = dict() + # This is the list of all possible checks. Add new checks to this list so + # a tool that post-processeds the json output knows the checks before + # iterating over the packages. + status_checks = ['cve', 'developers', 'hash', 'license', + 'license-files', 'patches', 'pkg-check', 'url', 'version'] + + def __init__(self, name, path): + self.name = name + self.path = path + self.pkg_path = os.path.dirname(path) + self.infras = None + self.license = None + self.has_license = False + self.has_license_files = False + self.has_hash = False + self.patch_files = [] + self.warnings = 0 + self.current_version = None + self.url = None + self.url_worker = None + self.cves = list() + self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None} + self.status = {} + + def pkgvar(self): + return self.name.upper().replace("-", "_") + + def set_url(self): + """ + Fills in the .url field + """ + self.status['url'] = ("warning", "no Config.in") + for filename in os.listdir(os.path.dirname(self.path)): + if fnmatch.fnmatch(filename, 'Config.*'): + fp = open(os.path.join(os.path.dirname(self.path), filename), "r") + for config_line in fp: + if URL_RE.match(config_line): + self.url = config_line.strip() + self.status['url'] = ("ok", "found") + fp.close() + return + self.status['url'] = ("error", "missing") + fp.close() + + @property + def patch_count(self): + return len(self.patch_files) + + @property + def has_valid_infra(self): + try: + if self.infras[0][1] == 'virtual': + return False + except IndexError: + return False + return True + + def set_infra(self): + """ + Fills in the .infras field + """ + self.infras = list() + with open(self.path, 'r') as f: + lines = f.readlines() + for l in lines: + match = INFRA_RE.match(l) + if not match: + continue + infra = match.group(1) + if infra.startswith("host-"): + self.infras.append(("host", infra[5:])) + else: + self.infras.append(("target", infra)) + + def set_license(self): + """ + Fills in the .status['license'] and .status['license-files'] fields + """ + if not self.has_valid_infra: + self.status['license'] = ("na", "no valid package infra") + self.status['license-files'] = ("na", "no valid package infra") + return + + var = self.pkgvar() + self.status['license'] = ("error", "missing") + self.status['license-files'] = ("error", "missing") + if var in self.all_licenses: + self.license = self.all_licenses[var] + self.status['license'] = ("ok", "found") + if var in self.all_license_files: + self.status['license-files'] = ("ok", "found") + + def set_hash_info(self): + """ + Fills in the .status['hash'] field + """ + if not self.has_valid_infra: + self.status['hash'] = ("na", "no valid package infra") + self.status['hash-license'] = ("na", "no valid package infra") + return + + hashpath = self.path.replace(".mk", ".hash") + if os.path.exists(hashpath): + self.status['hash'] = ("ok", "found") + else: + self.status['hash'] = ("error", "missing") + + def set_patch_count(self): + """ + Fills in the .patch_count, .patch_files and .status['patches'] fields + """ + if not self.has_valid_infra: + self.status['patches'] = ("na", "no valid package infra") + return + + pkgdir = os.path.dirname(self.path) + for subdir, _, _ in os.walk(pkgdir): + self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch') + + if self.patch_count == 0: + self.status['patches'] = ("ok", "no patches") + elif self.patch_count < 5: + self.status['patches'] = ("warning", "some patches") + else: + self.status['patches'] = ("error", "lots of patches") + + def set_current_version(self): + """ + Fills in the .current_version field + """ + var = self.pkgvar() + if var in self.all_versions: + self.current_version = self.all_versions[var] + + def set_check_package_warnings(self): + """ + Fills in the .warnings and .status['pkg-check'] fields + """ + cmd = ["./utils/check-package"] + pkgdir = os.path.dirname(self.path) + self.status['pkg-check'] = ("error", "Missing") + for root, dirs, files in os.walk(pkgdir): + for f in files: + if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host": + cmd.append(os.path.join(root, f)) + o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] + lines = o.splitlines() + for line in lines: + m = re.match("^([0-9]*) warnings generated", line.decode()) + if m: + self.warnings = int(m.group(1)) + if self.warnings == 0: + self.status['pkg-check'] = ("ok", "no warnings") + else: + self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings)) + return + + def is_cve_ignored(self, cve): + """ + Tells if the CVE is ignored by the package + """ + return cve in self.all_ignored_cves.get(self.pkgvar(), []) + + def set_developers(self, developers): + """ + Fills in the .developers and .status['developers'] field + """ + self.developers = [ + dev.name + for dev in developers + if dev.hasfile(self.path) + ] + + if self.developers: + self.status['developers'] = ("ok", "{} developers".format(len(self.developers))) + else: + self.status['developers'] = ("warning", "no developers") + + def is_status_ok(self, name): + return self.status[name][0] == 'ok' + + def __eq__(self, other): + return self.path == other.path + + def __lt__(self, other): + return self.path < other.path + + def __str__(self): + return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \ + (self.name, self.path, self.is_status_ok('license'), + self.is_status_ok('license-files'), self.status['hash'], self.patch_count) + + +class CVE: + """An accessor class for CVE Items in NVD files""" + def __init__(self, nvd_cve): + """Initialize a CVE from its NVD JSON representation""" + self.nvd_cve = nvd_cve + + @staticmethod + def download_nvd_year(nvd_path, year): + metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year) + path_metaf = os.path.join(nvd_path, metaf) + jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year) + path_jsonf_gz = os.path.join(nvd_path, jsonf_gz) + + # If the database file is less than a day old, we assume the NVD data + # locally available is recent enough. + if os.path.exists(path_jsonf_gz) and os.stat(path_jsonf_gz).st_mtime >= time.time() - 86400: + return path_jsonf_gz + + # If not, we download the meta file + url = "%s/%s" % (NVD_BASE_URL, metaf) + print("Getting %s" % url) + page_meta = requests.get(url) + page_meta.raise_for_status() + + # If the meta file already existed, we compare the existing + # one with the data newly downloaded. If they are different, + # we need to re-download the database. + # If the database does not exist locally, we need to redownload it in + # any case. + if os.path.exists(path_metaf) and os.path.exists(path_jsonf_gz): + meta_known = open(path_metaf, "r").read() + if page_meta.text == meta_known: + return path_jsonf_gz + + # Grab the compressed JSON NVD, and write files to disk + url = "%s/%s" % (NVD_BASE_URL, jsonf_gz) + print("Getting %s" % url) + page_json = requests.get(url) + page_json.raise_for_status() + open(path_jsonf_gz, "wb").write(page_json.content) + open(path_metaf, "w").write(page_meta.text) + return path_jsonf_gz + + @classmethod + def read_nvd_dir(cls, nvd_dir): + """ + Iterate over all the CVEs contained in NIST Vulnerability Database + feeds since NVD_START_YEAR. If the files are missing or outdated in + nvd_dir, a fresh copy will be downloaded, and kept in .json.gz + """ + for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1): + filename = CVE.download_nvd_year(nvd_dir, year) + try: + content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item') + except: # noqa: E722 + print("ERROR: cannot read %s. Please remove the file then rerun this script" % filename) + raise + for cve in content: + yield cls(cve['cve']) + + def each_product(self): + """Iterate over each product section of this cve""" + for vendor in self.nvd_cve['affects']['vendor']['vendor_data']: + for product in vendor['product']['product_data']: + yield product + + @property + def identifier(self): + """The CVE unique identifier""" + return self.nvd_cve['CVE_data_meta']['ID'] + + @property + def pkg_names(self): + """The set of package names referred by this CVE definition""" + return set(p['product_name'] for p in self.each_product()) + + def affects(self, br_pkg): + """ + True if the Buildroot Package object passed as argument is affected + by this CVE. + """ + if br_pkg.is_cve_ignored(self.identifier): + return False + + for product in self.each_product(): + if product['product_name'] != br_pkg.name: + continue + + for v in product['version']['version_data']: + if v["version_affected"] == "=": + if br_pkg.current_version == v["version_value"]: + return True + elif v["version_affected"] == "<=": + pkg_version = distutils.version.LooseVersion(br_pkg.current_version) + if not hasattr(pkg_version, "version"): + print("Cannot parse package '%s' version '%s'" % (br_pkg.name, br_pkg.current_version)) + continue + cve_affected_version = distutils.version.LooseVersion(v["version_value"]) + if not hasattr(cve_affected_version, "version"): + print("Cannot parse CVE affected version '%s'" % v["version_value"]) + continue + return pkg_version <= cve_affected_version + else: + print("version_affected: %s" % v['version_affected']) + return False + + +def get_pkglist(npackages, package_list): + """ + Builds the list of Buildroot packages, returning a list of Package + objects. Only the .name and .path fields of the Package object are + initialized. + + npackages: limit to N packages + package_list: limit to those packages in this list + """ + WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"] + WALK_EXCLUDES = ["boot/common.mk", + "linux/linux-ext-.*.mk", + "package/freescale-imx/freescale-imx.mk", + "package/gcc/gcc.mk", + "package/gstreamer/gstreamer.mk", + "package/gstreamer1/gstreamer1.mk", + "package/gtk2-themes/gtk2-themes.mk", + "package/matchbox/matchbox.mk", + "package/opengl/opengl.mk", + "package/qt5/qt5.mk", + "package/x11r7/x11r7.mk", + "package/doc-asciidoc.mk", + "package/pkg-.*.mk", + "package/nvidia-tegra23/nvidia-tegra23.mk", + "toolchain/toolchain-external/pkg-toolchain-external.mk", + "toolchain/toolchain-external/toolchain-external.mk", + "toolchain/toolchain.mk", + "toolchain/helpers.mk", + "toolchain/toolchain-wrapper.mk"] + packages = list() + count = 0 + for root, dirs, files in os.walk("."): + rootdir = root.split("/") + if len(rootdir) < 2: + continue + if rootdir[1] not in WALK_USEFUL_SUBDIRS: + continue + for f in files: + if not f.endswith(".mk"): + continue + # Strip ending ".mk" + pkgname = f[:-3] + if package_list and pkgname not in package_list: + continue + pkgpath = os.path.join(root, f) + skip = False + for exclude in WALK_EXCLUDES: + # pkgpath[2:] strips the initial './' + if re.match(exclude, pkgpath[2:]): + skip = True + continue + if skip: + continue + p = Package(pkgname, pkgpath) + packages.append(p) + count += 1 + if npackages and count == npackages: + return packages + return packages + + +def package_init_make_info(): + # Fetch all variables at once + variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars", + "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"]) + variable_list = variables.decode().splitlines() + + # We process first the host package VERSION, and then the target + # package VERSION. This means that if a package exists in both + # target and host variants, with different values (eg. version + # numbers (unlikely)), we'll report the target one. + variable_list = [x[5:] for x in variable_list if x.startswith("HOST_")] + \ + [x for x in variable_list if not x.startswith("HOST_")] + + for l in variable_list: + # Get variable name and value + pkgvar, value = l.split("=") + + # Strip the suffix according to the variable + if pkgvar.endswith("_LICENSE"): + # If value is "unknown", no license details available + if value == "unknown": + continue + pkgvar = pkgvar[:-8] + Package.all_licenses[pkgvar] = value + + elif pkgvar.endswith("_LICENSE_FILES"): + if pkgvar.endswith("_MANIFEST_LICENSE_FILES"): + continue + pkgvar = pkgvar[:-14] + Package.all_license_files.append(pkgvar) + + elif pkgvar.endswith("_VERSION"): + if pkgvar.endswith("_DL_VERSION"): + continue + pkgvar = pkgvar[:-8] + Package.all_versions[pkgvar] = value + + elif pkgvar.endswith("_IGNORE_CVES"): + pkgvar = pkgvar[:-12] + Package.all_ignored_cves[pkgvar] = value.split() + + +def check_url_status_worker(url, url_status): + if url_status[0] == 'ok': + try: + url_status_code = requests.head(url, timeout=30).status_code + if url_status_code >= 400: + return ("error", "invalid {}".format(url_status_code)) + except requests.exceptions.RequestException: + return ("error", "invalid (err)") + return ("ok", "valid") + return url_status + + +def check_package_urls(packages): + pool = Pool(processes=64) + for pkg in packages: + pkg.url_worker = pool.apply_async(check_url_status_worker, (pkg.url, pkg.status['url'])) + for pkg in packages: + pkg.status['url'] = pkg.url_worker.get(timeout=3600) + del pkg.url_worker + pool.terminate() + + +def release_monitoring_get_latest_version_by_distro(pool, name): + try: + req = pool.request('GET', "/api/project/Buildroot/%s" % name) + except HTTPError: + return (RM_API_STATUS_ERROR, None, None) + + if req.status != 200: + return (RM_API_STATUS_NOT_FOUND, None, None) + + data = json.loads(req.data) + + if 'version' in data: + return (RM_API_STATUS_FOUND_BY_DISTRO, data['version'], data['id']) + else: + return (RM_API_STATUS_FOUND_BY_DISTRO, None, data['id']) + + +def release_monitoring_get_latest_version_by_guess(pool, name): + try: + req = pool.request('GET', "/api/projects/?pattern=%s" % name) + except HTTPError: + return (RM_API_STATUS_ERROR, None, None) + + if req.status != 200: + return (RM_API_STATUS_NOT_FOUND, None, None) + + data = json.loads(req.data) + + projects = data['projects'] + projects.sort(key=lambda x: x['id']) -echo " + for p in projects: + if p['name'] == name and 'version' in p: + return (RM_API_STATUS_FOUND_BY_PATTERN, p['version'], p['id']) + + return (RM_API_STATUS_NOT_FOUND, None, None) + + +def check_package_latest_version_worker(name): + """Wrapper to try both by name then by guess""" + print(name) + res = release_monitoring_get_latest_version_by_distro(http_pool, name) + if res[0] == RM_API_STATUS_NOT_FOUND: + res = release_monitoring_get_latest_version_by_guess(http_pool, name) + return res + + +def check_package_latest_version(packages): + """ + Fills in the .latest_version field of all Package objects + + This field is a dict and has the following keys: + + - status: one of RM_API_STATUS_ERROR, + RM_API_STATUS_FOUND_BY_DISTRO, RM_API_STATUS_FOUND_BY_PATTERN, + RM_API_STATUS_NOT_FOUND + - version: string containing the latest version known by + release-monitoring.org for this package + - id: string containing the id of the project corresponding to this + package, as known by release-monitoring.org + """ + global http_pool + http_pool = HTTPSConnectionPool('release-monitoring.org', port=443, + cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), + timeout=30) + worker_pool = Pool(processes=64) + results = worker_pool.map(check_package_latest_version_worker, (pkg.name for pkg in packages)) + for pkg, r in zip(packages, results): + pkg.latest_version = dict(zip(['status', 'version', 'id'], r)) + + if not pkg.has_valid_infra: + pkg.status['version'] = ("na", "no valid package infra") + continue + + if pkg.latest_version['status'] == RM_API_STATUS_ERROR: + pkg.status['version'] = ('warning', "Release Monitoring API error") + elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND: + pkg.status['version'] = ('warning', "Package not found on Release Monitoring") + + if pkg.latest_version['version'] is None: + pkg.status['version'] = ('warning', "No upstream version available on Release Monitoring") + elif pkg.latest_version['version'] != pkg.current_version: + pkg.status['version'] = ('error', "The newer version {} is available upstream".format(pkg.latest_version['version'])) + else: + pkg.status['version'] = ('ok', 'up-to-date') + + worker_pool.terminate() + del http_pool + + +def check_package_cves(nvd_path, packages): + if not os.path.isdir(nvd_path): + os.makedirs(nvd_path) + + for cve in CVE.read_nvd_dir(nvd_path): + for pkg_name in cve.pkg_names: + if pkg_name in packages and cve.affects(packages[pkg_name]): + packages[pkg_name].cves.append(cve.identifier) + + +def calculate_stats(packages): + stats = defaultdict(int) + stats['packages'] = len(packages) + for pkg in packages: + # If packages have multiple infra, take the first one. For the + # vast majority of packages, the target and host infra are the + # same. There are very few packages that use a different infra + # for the host and target variants. + if len(pkg.infras) > 0: + infra = pkg.infras[0][1] + stats["infra-%s" % infra] += 1 + else: + stats["infra-unknown"] += 1 + if pkg.is_status_ok('license'): + stats["license"] += 1 + else: + stats["no-license"] += 1 + if pkg.is_status_ok('license-files'): + stats["license-files"] += 1 + else: + stats["no-license-files"] += 1 + if pkg.is_status_ok('hash'): + stats["hash"] += 1 + else: + stats["no-hash"] += 1 + if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO: + stats["rmo-mapping"] += 1 + else: + stats["rmo-no-mapping"] += 1 + if not pkg.latest_version['version']: + stats["version-unknown"] += 1 + elif pkg.latest_version['version'] == pkg.current_version: + stats["version-uptodate"] += 1 + else: + stats["version-not-uptodate"] += 1 + stats["patches"] += pkg.patch_count + stats["total-cves"] += len(pkg.cves) + if len(pkg.cves) != 0: + stats["pkg-cves"] += 1 + return stats + + +html_header = """ + + Statistics of Buildroot packages Results
- +

+""" + + +html_footer = """ + + + +""" + + +def infra_str(infra_list): + if not infra_list: + return "Unknown" + elif len(infra_list) == 1: + return "%s
%s" % (infra_list[0][1], infra_list[0][0]) + elif infra_list[0][1] == infra_list[1][1]: + return "%s
%s + %s" % \ + (infra_list[0][1], infra_list[0][0], infra_list[1][0]) + else: + return "%s (%s)
%s (%s)" % \ + (infra_list[0][1], infra_list[0][0], + infra_list[1][1], infra_list[1][0]) + + +def boolean_str(b): + if b: + return "Yes" + else: + return "No" + + +def dump_html_pkg(f, pkg): + f.write(" \n") + f.write(" \n" % pkg.path[2:]) + + # Patch count + td_class = ["centered"] + if pkg.patch_count == 0: + td_class.append("nopatches") + elif pkg.patch_count < 5: + td_class.append("somepatches") + else: + td_class.append("lotsofpatches") + f.write(" \n" % + (" ".join(td_class), str(pkg.patch_count))) + + # Infrastructure + infra = infra_str(pkg.infras) + td_class = ["centered"] + if infra == "Unknown": + td_class.append("wrong") + else: + td_class.append("correct") + f.write(" \n" % + (" ".join(td_class), infra_str(pkg.infras))) + + # License + td_class = ["centered"] + if pkg.is_status_ok('license'): + td_class.append("correct") + else: + td_class.append("wrong") + f.write(" \n" % + (" ".join(td_class), boolean_str(pkg.is_status_ok('license')))) + + # License files + td_class = ["centered"] + if pkg.is_status_ok('license-files'): + td_class.append("correct") + else: + td_class.append("wrong") + f.write(" \n" % + (" ".join(td_class), boolean_str(pkg.is_status_ok('license-files')))) + + # Hash + td_class = ["centered"] + if pkg.is_status_ok('hash'): + td_class.append("correct") + else: + td_class.append("wrong") + f.write(" \n" % + (" ".join(td_class), boolean_str(pkg.is_status_ok('hash')))) + + # Current version + if len(pkg.current_version) > 20: + current_version = pkg.current_version[:20] + "..." + else: + current_version = pkg.current_version + f.write(" \n" % current_version) + + # Latest version + if pkg.latest_version['status'] == RM_API_STATUS_ERROR: + td_class.append("version-error") + if pkg.latest_version['version'] is None: + td_class.append("version-unknown") + elif pkg.latest_version['version'] != pkg.current_version: + td_class.append("version-needs-update") + else: + td_class.append("version-good") + + if pkg.latest_version['status'] == RM_API_STATUS_ERROR: + latest_version_text = "Error" + elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND: + latest_version_text = "Not found" + else: + if pkg.latest_version['version'] is None: + latest_version_text = "Found, but no version" + else: + latest_version_text = "%s" % \ + (pkg.latest_version['id'], str(pkg.latest_version['version'])) + + latest_version_text += "
" + + if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO: + latest_version_text += "found by distro" + else: + latest_version_text += "found by guess" + + f.write(" \n" % + (" ".join(td_class), latest_version_text)) + + # Warnings + td_class = ["centered"] + if pkg.warnings == 0: + td_class.append("correct") + else: + td_class.append("wrong") + f.write(" \n" % + (" ".join(td_class), pkg.warnings)) + + # URL status + td_class = ["centered"] + url_str = pkg.status['url'][1] + if pkg.status['url'][0] in ("error", "warning"): + td_class.append("missing_url") + if pkg.status['url'][0] == "error": + td_class.append("invalid_url") + url_str = "%s" % (pkg.url, pkg.status['url'][1]) + else: + td_class.append("good_url") + url_str = "Link" % pkg.url + f.write(" \n" % + (" ".join(td_class), url_str)) + + # CVEs + td_class = ["centered"] + if len(pkg.cves) == 0: + td_class.append("correct") + else: + td_class.append("wrong") + f.write(" \n") + + f.write(" \n") + + +def dump_html_all_pkgs(f, packages): + f.write(""" +
%s%s%s%s%s%s%s%s%d%s\n" % " ".join(td_class)) + for cve in pkg.cves: + f.write(" %s
\n" % (cve, cve)) + f.write("
- + + + + + + -" - -autotools_packages=0 -cmake_packages=0 -luarocks_package=0 -perl_packages=0 -python_packages=0 -virtual_packages=0 -generic_packages=0 -manual_packages=0 -packages_with_licence=0 -packages_without_licence=0 -packages_with_license_files=0 -packages_without_license_files=0 -total_patch_count=0 -cnt=0 - -for i in $(find boot/ linux/ package/ -name '*.mk' | sort) ; do - - if test \ - $i = "boot/common.mk" -o \ - $i = "linux/linux-ext-xenomai.mk" -o \ - $i = "linux/linux-ext-rtai.mk" -o \ - $i = "package/efl/efl.mk" -o \ - $i = "package/freescale-imx/freescale-imx.mk" -o \ - $i = "package/gcc/gcc.mk" -o \ - $i = "package/gstreamer/gstreamer.mk" -o \ - $i = "package/gstreamer1/gstreamer1.mk" -o \ - $i = "package/gtk2-themes/gtk2-themes.mk" -o \ - $i = "package/matchbox/matchbox.mk" -o \ - $i = "package/opengl/opengl.mk" -o \ - $i = "package/qt5/qt5.mk" -o \ - $i = "package/x11r7/x11r7.mk" -o \ - $i = "package/pkg-autotools.mk" -o \ - $i = "package/pkg-cmake.mk" -o \ - $i = "package/pkg-luarocks.mk" -o \ - $i = "package/pkg-perl.mk" -o \ - $i = "package/pkg-python.mk" -o \ - $i = "package/pkg-virtual.mk" -o \ - $i = "package/pkg-download.mk" -o \ - $i = "package/pkg-generic.mk" -o \ - $i = "package/pkg-utils.mk" ; then - echo "skipping $i" 1>&2 - continue - fi - - cnt=$((cnt+1)) - - hashost=0 - hastarget=0 - infratype="" - - # Determine package infrastructure - if grep -E "\(host-autotools-package\)" $i > /dev/null ; then - infratype="autotools" - hashost=1 - fi - - if grep -E "\(autotools-package\)" $i > /dev/null ; then - infratype="autotools" - hastarget=1 - fi - - if grep -E "\(host-luarocks-package\)" $i > /dev/null ; then - infratype="luarocks" - hashost=1 - fi - - if grep -E "\(luarocks-package\)" $i > /dev/null ; then - infratype="luarocks" - hastarget=1 - fi - - if grep -E "\(host-perl-package\)" $i > /dev/null ; then - infratype="perl" - hashost=1 - fi - - if grep -E "\(perl-package\)" $i > /dev/null ; then - infratype="perl" - hastarget=1 - fi - - if grep -E "\(host-python-package\)" $i > /dev/null ; then - infratype="python" - hashost=1 - fi - - if grep -E "\(python-package\)" $i > /dev/null ; then - infratype="python" - hastarget=1 - fi - if grep -E "\(host-virtual-package\)" $i > /dev/null ; then - infratype="virtual" - hashost=1 - fi - - if grep -E "\(virtual-package\)" $i > /dev/null ; then - infratype="virtual" - hastarget=1 - fi - - if grep -E "\(host-generic-package\)" $i > /dev/null ; then - infratype="generic" - hashost=1 - fi - - if grep -E "\(generic-package\)" $i > /dev/null ; then - infratype="generic" - hastarget=1 - fi - - if grep -E "\(host-cmake-package\)" $i > /dev/null ; then - infratype="cmake" - hashost=1 - fi - - if grep -E "\(cmake-package\)" $i > /dev/null ; then - infratype="cmake" - hastarget=1 - fi - - pkg=$(basename $i) - pkg=${pkg%.mk} - pkgvariable=$(echo ${pkg} | tr "a-z-" "A-Z_") - - - # Count packages per infrastructure - if [ -z ${infratype} ] ; then - infratype="manual" - manual_packages=$(($manual_packages+1)) - elif [ ${infratype} = "autotools" ]; then - autotools_packages=$(($autotools_packages+1)) - elif [ ${infratype} = "cmake" ]; then - cmake_packages=$(($cmake_packages+1)) - elif [ ${infratype} = "luarocks" ]; then - luarocks_packages=$(($luarocks_packages+1)) - elif [ ${infratype} = "perl" ]; then - perl_packages=$(($perl_packages+1)) - elif [ ${infratype} = "python" ]; then - python_packages=$(($python_packages+1)) - elif [ ${infratype} = "virtual" ]; then - virtual_packages=$(($virtual_packages+1)) - elif [ ${infratype} = "generic" ]; then - generic_packages=$(($generic_packages+1)) - fi - - if grep -qE "^${pkgvariable}_LICENSE[ ]*=" $i ; then - packages_with_license=$(($packages_with_license+1)) - license=1 - else - packages_without_license=$(($packages_without_license+1)) - license=0 - fi - - if grep -qE "^${pkgvariable}_LICENSE_FILES[ ]*=" $i ; then - packages_with_license_files=$(($packages_with_license_files+1)) - license_files=1 - else - packages_without_license_files=$(($packages_without_license_files+1)) - license_files=0 - fi - - echo "" - - echo "" - echo "" - - package_dir=$(dirname $i) - patch_count=$(find ${package_dir} -name '*.patch' | wc -l) - total_patch_count=$(($total_patch_count+$patch_count)) - - if test $patch_count -lt 1 ; then - patch_count_class="nopatches" - elif test $patch_count -lt 5 ; then - patch_count_class="somepatches" - else - patch_count_class="lotsofpatches" - fi - - echo "" - - if [ ${infratype} = "manual" ] ; then - echo "" - else - echo "" - fi - - if [ ${license} -eq 0 ] ; then - echo "" - else - echo "" - fi - - if [ ${license_files} -eq 0 ] ; then - echo "" - else - echo "" - fi - - echo "" - -done -echo "
Id Package Patch count Infrastructure License License filesHash fileCurrent versionLatest versionWarningsUpstream URLCVEs
$cnt$i" - echo "$patch_count" - echo "manual" - echo "${infratype}
" - if [ ${hashost} -eq 1 -a ${hastarget} -eq 1 ]; then - echo "target + host" - elif [ ${hashost} -eq 1 ]; then - echo "host" - else - echo "target" - fi - echo "		NoYesNoYes
" - -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "" -echo "
Packages using the generic infrastructure$generic_packages
Packages using the cmake infrastructure$cmake_packages
Packages using the autotools infrastructure$autotools_packages
Packages using the luarocks infrastructure$luarocks_packages
Packages using the perl infrastructure$perl_packages
Packages using the python infrastructure$python_packages
Packages using the virtual infrastructure$virtual_packages
Packages not using any infrastructure$manual_packages
Packages having license information$packages_with_license
Packages not having licence information$packages_without_license
Packages having license files information$packages_with_license_files
Packages not having licence files information$packages_without_license_files
Number of patches in all packages$total_patch_count
TOTAL$cnt
" - -echo "
" -echo "Updated on $(LANG=C date), Git commit $(git log master -n 1 --pretty=format:%H)" -echo "" -echo "" +""") + for pkg in sorted(packages): + dump_html_pkg(f, pkg) + f.write("") + + +def dump_html_stats(f, stats): + f.write("\n") + f.write("\n") + infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")] + for infra in infras: + f.write(" \n" % + (infra, stats["infra-%s" % infra])) + f.write(" \n" % + stats["license"]) + f.write(" \n" % + stats["no-license"]) + f.write(" \n" % + stats["license-files"]) + f.write(" \n" % + stats["no-license-files"]) + f.write(" \n" % + stats["hash"]) + f.write(" \n" % + stats["no-hash"]) + f.write(" \n" % + stats["patches"]) + f.write("\n" % + stats["rmo-mapping"]) + f.write("\n" % + stats["rmo-no-mapping"]) + f.write("\n" % + stats["version-uptodate"]) + f.write("\n" % + stats["version-not-uptodate"]) + f.write("\n" % + stats["version-unknown"]) + f.write("\n" % + stats["pkg-cves"]) + f.write("\n" % + stats["total-cves"]) + f.write("
Packages using the %s infrastructure%s
Packages having license information%s
Packages not having license information%s
Packages having license files information%s
Packages not having license files information%s
Packages having a hash file%s
Packages not having a hash file%s
Total number of patches%s
Packages having a mapping on release-monitoring.org%s
Packages lacking a mapping on release-monitoring.org%s
Packages that are up-to-date%s
Packages that are not up-to-date%s
Packages with no known upstream version%s
Packages affected by CVEs%s
Total number of CVEs affecting all packages%s
\n") + + +def dump_html_gen_info(f, date, commit): + # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032 + f.write("

Updated on %s, git commit %s

\n" % (str(date), commit)) + + +def dump_html(packages, stats, date, commit, output): + with open(output, 'w') as f: + f.write(html_header) + dump_html_all_pkgs(f, packages) + dump_html_stats(f, stats) + dump_html_gen_info(f, date, commit) + f.write(html_footer) + + +def dump_json(packages, defconfigs, stats, date, commit, output): + # Format packages as a dictionnary instead of a list + # Exclude local field that does not contains real date + excluded_fields = ['url_worker', 'name'] + pkgs = { + pkg.name: { + k: v + for k, v in pkg.__dict__.items() + if k not in excluded_fields + } for pkg in packages + } + defconfigs = { + d.name: { + k: v + for k, v in d.__dict__.items() + } for d in defconfigs + } + # Aggregate infrastructures into a single dict entry + statistics = { + k: v + for k, v in stats.items() + if not k.startswith('infra-') + } + statistics['infra'] = {k[6:]: v for k, v in stats.items() if k.startswith('infra-')} + # The actual structure to dump, add commit and date to it + final = {'packages': pkgs, + 'stats': statistics, + 'defconfigs': defconfigs, + 'package_status_checks': Package.status_checks, + 'commit': commit, + 'date': str(date)} + + with open(output, 'w') as f: + json.dump(final, f, indent=2, separators=(',', ': ')) + f.write('\n') + + +def parse_args(): + parser = argparse.ArgumentParser() + output = parser.add_argument_group('output', 'Output file(s)') + output.add_argument('--html', dest='html', action='store', + help='HTML output file') + output.add_argument('--json', dest='json', action='store', + help='JSON output file') + packages = parser.add_mutually_exclusive_group() + packages.add_argument('-n', dest='npackages', type=int, action='store', + help='Number of packages') + packages.add_argument('-p', dest='packages', action='store', + help='List of packages (comma separated)') + parser.add_argument('--nvd-path', dest='nvd_path', + help='Path to the local NVD database') + args = parser.parse_args() + if not args.html and not args.json: + parser.error('at least one of --html or --json (or both) is required') + return args + + +def __main__(): + args = parse_args() + if args.packages: + package_list = args.packages.split(",") + else: + package_list = None + date = datetime.datetime.utcnow() + commit = subprocess.check_output(['git', 'rev-parse', + 'HEAD']).splitlines()[0].decode() + print("Build package list ...") + packages = get_pkglist(args.npackages, package_list) + print("Getting developers ...") + developers = parse_developers() + print("Build defconfig list ...") + defconfigs = get_defconfig_list() + for d in defconfigs: + d.set_developers(developers) + print("Getting package make info ...") + package_init_make_info() + print("Getting package details ...") + for pkg in packages: + pkg.set_infra() + pkg.set_license() + pkg.set_hash_info() + pkg.set_patch_count() + pkg.set_check_package_warnings() + pkg.set_current_version() + pkg.set_url() + pkg.set_developers(developers) + print("Checking URL status") + check_package_urls(packages) + print("Getting latest versions ...") + check_package_latest_version(packages) + if args.nvd_path: + print("Checking packages CVEs") + check_package_cves(args.nvd_path, {p.name: p for p in packages}) + print("Calculate stats") + stats = calculate_stats(packages) + if args.html: + print("Write HTML") + dump_html(packages, stats, date, commit, args.html) + if args.json: + print("Write JSON") + dump_json(packages, defconfigs, stats, date, commit, args.json) + + +__main__() diff --git a/support/scripts/pycompile.py b/support/scripts/pycompile.py new file mode 100644 index 0000000000..9192a7016a --- /dev/null +++ b/support/scripts/pycompile.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +'''Wrapper for python2 and python3 around compileall to raise exception +when a python byte code generation failed. + +Inspired from: + http://stackoverflow.com/questions/615632/how-to-detect-errors-from-compileall-compile-dir +''' +from __future__ import print_function +import sys +import py_compile +import compileall +import argparse + + +def check_for_errors(comparison): + '''Wrap comparison operator with code checking for PyCompileError. + If PyCompileError was raised, re-raise it again to abort execution, + otherwise perform comparison as expected. + ''' + def operator(self, other): + exc_type, value, traceback = sys.exc_info() + if exc_type is not None and issubclass(exc_type, + py_compile.PyCompileError): + print("Cannot compile %s" % value.file) + raise value + + return comparison(self, other) + + return operator + + +class ReportProblem(int): + '''Class that pretends to be an int() object but implements all of its + comparison operators such that it'd detect being called in + PyCompileError handling context and abort execution + ''' + VALUE = 1 + + def __new__(cls, *args, **kwargs): + return int.__new__(cls, ReportProblem.VALUE, **kwargs) + + @check_for_errors + def __lt__(self, other): + return ReportProblem.VALUE < other + + @check_for_errors + def __eq__(self, other): + return ReportProblem.VALUE == other + + def __ge__(self, other): + return not self < other + + def __gt__(self, other): + return not self < other and not self == other + + def __ne__(self, other): + return not self == other + + +parser = argparse.ArgumentParser(description='Compile Python source files in a directory tree.') +parser.add_argument("target", metavar='DIRECTORY', + help='Directory to scan') +parser.add_argument("--force", action='store_true', + help="Force compilation even if alread compiled") + +args = parser.parse_args() + +compileall.compile_dir(args.target, force=args.force, quiet=ReportProblem()) diff --git a/support/scripts/readme.kconfiglib b/support/scripts/readme.kconfiglib deleted file mode 100644 index 5c82b6f71a..0000000000 --- a/support/scripts/readme.kconfiglib +++ /dev/null @@ -1,30 +0,0 @@ -Readme -====== - -Kconfiglib ----------- - -This python module, developped by Ulf Magnusson and released under the ISC -license, is fetched from: - -https://github.com/ulfalizer/Kconfiglib -commit: 02103fba9ae32a78291af53e50ee5d4bb3f69c1e - -Kconfiglib license -~~~~~~~~~~~~~~~~~~ - -License (ISC) - -Copyright (c) 2011-2013, Ulf Magnusson - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. diff --git a/support/scripts/scancpan b/support/scripts/scancpan deleted file mode 100755 index 9bc75f389e..0000000000 --- a/support/scripts/scancpan +++ /dev/null @@ -1,802 +0,0 @@ -#!/usr/bin/env perl - -# This chunk of stuff was generated by App::FatPacker. To find the original -# file's code, look for the end of this BEGIN block or the string 'FATPACK' -BEGIN { -my %fatpacked; - -$fatpacked{"MetaCPAN/API/Tiny.pm"} = <<'METACPAN_API_TINY'; - package MetaCPAN::API::Tiny; - { - $MetaCPAN::API::Tiny::VERSION = '1.131730'; - } - use strict; - use warnings; - # ABSTRACT: A Tiny API client for MetaCPAN - - use Carp; - use JSON::PP 'encode_json', 'decode_json'; - use HTTP::Tiny; - - - sub new { - my ($class, @args) = @_; - - $#_ % 2 == 0 - or croak 'Arguments must be provided as name/value pairs'; - - my %params = @args; - - die 'ua_args must be an array reference' - if $params{ua_args} && ref($params{ua_args}) ne 'ARRAY'; - - my $self = +{ - base_url => $params{base_url} || 'http://api.metacpan.org/v0', - ua => $params{ua} || HTTP::Tiny->new( - $params{ua_args} - ? @{$params{ua_args}} - : (agent => 'MetaCPAN::API::Tiny/' - . ($MetaCPAN::API::VERSION || 'xx'))), - }; - - return bless($self, $class); - } - - sub _build_extra_params { - my $self = shift; - - @_ % 2 == 0 - or croak 'Incorrect number of params, must be key/value'; - - my %extra = @_; - my $ua = $self->{ua}; - - foreach my $key (keys %extra) - { - # The implementation in HTTP::Tiny uses + instead of %20, fix that - $extra{$key} = $ua->_uri_escape($extra{$key}); - $extra{$key} =~ s/\+/%20/g; - } - - my $params = join '&', map { "$_=" . $extra{$_} } sort keys %extra; - - return $params; - } - - - # /source/{author}/{release}/{path} - sub source { - my $self = shift; - my %opts = @_ ? @_ : (); - my $url = ''; - my $error = "Provide 'author' and 'release' and 'path'"; - - %opts or croak $error; - - if ( - defined ( my $author = $opts{'author'} ) && - defined ( my $release = $opts{'release'} ) && - defined ( my $path = $opts{'path'} ) - ) { - $url = "source/$author/$release/$path"; - } else { - croak $error; - } - - $url = $self->{base_url} . "/$url"; - - my $result = $self->{ua}->get($url); - $result->{'success'} - or croak "Failed to fetch '$url': " . $result->{'reason'}; - - return $result->{'content'}; - } - - - # /release/{distribution} - # /release/{author}/{release} - sub release { - my $self = shift; - my %opts = @_ ? @_ : (); - my $url = ''; - my $error = "Either provide 'distribution', or 'author' and 'release', " . - "or 'search'"; - - %opts or croak $error; - - my %extra_opts = (); - - if ( defined ( my $dist = $opts{'distribution'} ) ) { - $url = "release/$dist"; - } elsif ( - defined ( my $author = $opts{'author'} ) && - defined ( my $release = $opts{'release'} ) - ) { - $url = "release/$author/$release"; - } elsif ( defined ( my $search_opts = $opts{'search'} ) ) { - ref $search_opts && ref $search_opts eq 'HASH' - or croak $error; - - %extra_opts = %{$search_opts}; - $url = 'release/_search'; - } else { - croak $error; - } - - return $self->fetch( $url, %extra_opts ); - } - - - # /pod/{module} - # /pod/{author}/{release}/{path} - sub pod { - my $self = shift; - my %opts = @_ ? @_ : (); - my $url = ''; - my $error = "Either provide 'module' or 'author and 'release' and 'path'"; - - %opts or croak $error; - - if ( defined ( my $module = $opts{'module'} ) ) { - $url = "pod/$module"; - } elsif ( - defined ( my $author = $opts{'author'} ) && - defined ( my $release = $opts{'release'} ) && - defined ( my $path = $opts{'path'} ) - ) { - $url = "pod/$author/$release/$path"; - } else { - croak $error; - } - - # check content-type - my %extra = (); - if ( defined ( my $type = $opts{'content-type'} ) ) { - $type =~ m{^ text/ (?: html|plain|x-pod|x-markdown ) $}x - or croak 'Incorrect content-type provided'; - - $extra{headers}{'content-type'} = $type; - } - - $url = $self->{base_url}. "/$url"; - - my $result = $self->{ua}->get( $url, \%extra ); - $result->{'success'} - or croak "Failed to fetch '$url': " . $result->{'reason'}; - - return $result->{'content'}; - } - - - # /module/{module} - sub module { - my $self = shift; - my $name = shift; - - $name or croak 'Please provide a module name'; - - return $self->fetch("module/$name"); - } - - - # file() is a synonym of module - sub file { goto &module } - - - # /author/{author} - sub author { - my $self = shift; - my ( $pause_id, $url, %extra_opts ); - - if ( @_ == 1 ) { - $url = 'author/' . shift; - } elsif ( @_ == 2 ) { - my %opts = @_; - - if ( defined $opts{'pauseid'} ) { - $url = "author/" . $opts{'pauseid'}; - } elsif ( defined $opts{'search'} ) { - my $search_opts = $opts{'search'}; - - ref $search_opts && ref $search_opts eq 'HASH' - or croak "'search' key must be hashref"; - - %extra_opts = %{$search_opts}; - $url = 'author/_search'; - } else { - croak 'Unknown option given'; - } - } else { - croak 'Please provide an author PAUSEID or a "search"'; - } - - return $self->fetch( $url, %extra_opts ); - } - - - - sub fetch { - my $self = shift; - my $url = shift; - my $extra = $self->_build_extra_params(@_); - my $base = $self->{base_url}; - my $req_url = $extra ? "$base/$url?$extra" : "$base/$url"; - - my $result = $self->{ua}->get($req_url); - return $self->_decode_result( $result, $req_url ); - } - - - sub post { - my $self = shift; - my $url = shift; - my $query = shift; - my $base = $self->{base_url}; - - defined $url - or croak 'First argument of URL must be provided'; - - ref $query and ref $query eq 'HASH' - or croak 'Second argument of query hashref must be provided'; - - my $query_json = encode_json( $query ); - my $result = $self->{ua}->request( - 'POST', - "$base/$url", - { - headers => { 'Content-Type' => 'application/json' }, - content => $query_json, - } - ); - - return $self->_decode_result( $result, $url, $query_json ); - } - - sub _decode_result { - my $self = shift; - my ( $result, $url, $original ) = @_; - my $decoded_result; - - ref $result and ref $result eq 'HASH' - or croak 'First argument must be hashref'; - - defined $url - or croak 'Second argument of a URL must be provided'; - - if ( defined ( my $success = $result->{'success'} ) ) { - my $reason = $result->{'reason'} || ''; - $reason .= ( defined $original ? " (request: $original)" : '' ); - - $success or croak "Failed to fetch '$url': $reason"; - } else { - croak 'Missing success in return value'; - } - - defined ( my $content = $result->{'content'} ) - or croak 'Missing content in return value'; - - eval { $decoded_result = decode_json $content; 1 } - or do { croak "Couldn't decode '$content': $@" }; - - return $decoded_result; - } - - 1; - - __END__ - - =pod - - =head1 NAME - - MetaCPAN::API::Tiny - A Tiny API client for MetaCPAN - - =head1 VERSION - - version 1.131730 - - =head1 DESCRIPTION - - This is the Tiny version of L. It implements a compatible API - with a few notable exceptions: - - =over 4 - - =item Attributes are direct hash access - - The attributes defined using Mo(o|u)se are now accessed via the blessed hash - directly. There are no accessors defined to access this elements. - - =item Exception handling - - Instead of using Try::Tiny, raw evals are used. This could potentially cause - issues, so just be aware. - - =item Testing - - Test::Fatal was replaced with an eval implementation of exception(). - Test::TinyMocker usage is retained, but may be absorbed since it is pure perl - - =back - - =head1 CLASS_METHODS - - =head2 new - - new is the constructor for MetaCPAN::API::Tiny. In the non-tiny version of this - module, this is provided via Any::Moose built from the attributes defined. In - the tiny version, we define our own constructor. It takes the same arguments - and provides similar checks to MetaCPAN::API with regards to arguments passed. - - =head1 PUBLIC_METHODS - - =head2 source - - my $source = $mcpan->source( - author => 'DOY', - release => 'Moose-2.0201', - path => 'lib/Moose.pm', - ); - - Searches MetaCPAN for a module or a specific release and returns the plain source. - - =head2 release - - my $result = $mcpan->release( distribution => 'Moose' ); - - # or - my $result = $mcpan->release( author => 'DOY', release => 'Moose-2.0001' ); - - Searches MetaCPAN for a dist. - - You can do complex searches using 'search' parameter: - - # example lifted from MetaCPAN docs - my $result = $mcpan->release( - search => { - author => "OALDERS AND ", - filter => "status:latest", - fields => "name", - size => 1, - }, - ); - - =head2 pod - - my $result = $mcpan->pod( module => 'Moose' ); - - # or - my $result = $mcpan->pod( - author => 'DOY', - release => 'Moose-2.0201', - path => 'lib/Moose.pm', - ); - - Searches MetaCPAN for a module or a specific release and returns the POD. - - =head2 module - - my $result = $mcpan->module('MetaCPAN::API'); - - Searches MetaCPAN and returns a module's ".pm" file. - - =head2 file - - A synonym of L - - =head2 author - - my $result1 = $mcpan->author('XSAWYERX'); - my $result2 = $mcpan->author( pauseid => 'XSAWYERX' ); - - Searches MetaCPAN for a specific author. - - You can do complex searches using 'search' parameter: - - # example lifted from MetaCPAN docs - my $result = $mcpan->author( - search => { - q => 'profile.name:twitter', - size => 1, - }, - ); - - =head2 fetch - - my $result = $mcpan->fetch('/release/distribution/Moose'); - - # with parameters - my $more = $mcpan->fetch( - '/release/distribution/Moose', - param => 'value', - ); - - This is a helper method for API implementations. It fetches a path from MetaCPAN, decodes the JSON from the content variable and returns it. - - You don't really need to use it, but you can in case you want to write your own extension implementation to MetaCPAN::API. - - It accepts an additional hash as "GET" parameters. - - =head2 post - - # /release&content={"query":{"match_all":{}},"filter":{"prefix":{"archive":"Cache-Cache-1.06"}}} - my $result = $mcpan->post( - 'release', - { - query => { match_all => {} }, - filter => { prefix => { archive => 'Cache-Cache-1.06' } }, - }, - ); - - The POST equivalent of the "fetch()" method. It gets the path and JSON request. - - =head1 THANKS - - Overall the tests and code were ripped directly from MetaCPAN::API and - tiny-fied. A big thanks to Sawyer X for writing the original module. - - =head1 AUTHOR - - Nicholas R. Perez - - =head1 COPYRIGHT AND LICENSE - - This software is copyright (c) 2013 by Nicholas R. Perez . - - This is free software; you can redistribute it and/or modify it under - the same terms as the Perl 5 programming language system itself. - - =cut -METACPAN_API_TINY - -s/^ //mg for values %fatpacked; - -unshift @INC, sub { - if (my $fat = $fatpacked{$_[1]}) { - if ($] < 5.008) { - return sub { - return 0 unless length $fat; - $fat =~ s/^([^\n]*\n?)//; - $_ = $1; - return 1; - }; - } - open my $fh, '<', \$fat - or die "FatPacker error loading $_[1] (could be a perl installation issue?)"; - return $fh; - } - return -}; - -} # END OF FATPACK CODE - - -use 5.018; # same major version as target perl -use strict; -use warnings; -use Fatal qw(open close); - -use Getopt::Long; -use Pod::Usage; -use File::Basename; -use Module::CoreList; -use HTTP::Tiny; -use MetaCPAN::API::Tiny; - -my ($help, $man, $quiet, $force, $recommend, $host); -my $target = 1; -GetOptions( 'help|?' => \$help, - 'man' => \$man, - 'quiet|q' => \$quiet, - 'force|f' => \$force, - 'host!' => \$host, - 'target!' => \$target, - 'recommend' => \$recommend -) or pod2usage(-exitval => 1); -pod2usage(-exitval => 0) if $help; -pod2usage(-exitval => 0, -verbose => 2) if $man; -pod2usage(-exitval => 1) if scalar @ARGV == 0; - -my %dist; # name -> metacpan data -my %need_target; # name -> 1 if target package is needed -my %need_host; # name -> 1 if host package is needed -my %need_dlopen; # name -> 1 if requires dynamic library -my %deps_build; # name -> list of host dependencies -my %deps_runtime; # name -> list of target dependencies -my %license_files; # name -> list of license files -my $mcpan = MetaCPAN::API::Tiny->new(); -my $ua = HTTP::Tiny->new(); - -sub get_manifest { - my ($author, $distname, $version) = @_; - my $url = qq{http://api.metacpan.org/source/${author}/${distname}-${version}/MANIFEST}; - my $response = $ua->get($url); - return $response->{content}; -} - -sub is_xs { - my ($manifest) = @_; - # This heuristic determines if a module is a native extension, by searching - # some file extension types in the MANIFEST of the distribution. - # It was inspired by http://deps.cpantesters.org/static/purity.html - return $manifest =~ m/\.(swg|xs|c|h|i)\n/; -} - -sub find_license_files { - my ($manifest) = @_; - my @license_files; - foreach (split /\n/, $manifest) { - next if m|/|; - push @license_files, $_ if m/(ARTISTIC|COPYING|COPYRIGHT|LICENSE)/i; - } - return \@license_files; -} - -sub fetch { - my ($name, $need_target, $need_host) = @_; - $need_target{$name} = $need_target if $need_target; - $need_host{$name} = $need_host if $need_host; - unless ($dist{$name}) { - say qq{fetch ${name}} unless $quiet; - my $result = $mcpan->release( distribution => $name ); - $dist{$name} = $result; - my $manifest = get_manifest( $result->{author}, $name, $result->{version} ); - $need_dlopen{$name} = is_xs( $manifest ); - $license_files{$name} = find_license_files( $manifest ); - my @deps_build = (); - my @deps_runtime = (); - my $mb; - foreach my $dep (@{$result->{dependency}}) { - my $modname = ${$dep}{module}; - $mb = 1 if $modname eq q{Module::Build}; - # Module::Build has a special treatment, because it is a core module, - # but some module require a very recent version of it - next if $modname eq q{perl}; - next if $modname =~ m|^Alien|; - next if $modname =~ m|^Win32|; - next if Module::CoreList::first_release( $modname ); - # we could use the host Module::CoreList data, because host perl and - # target perl have the same major version - next if ${$dep}{phase} eq q{develop}; - next if ${$dep}{phase} eq q{test}; - next if !$recommend && ${$dep}{relationship} ne q{requires}; - my $distname = $mcpan->module( $modname )->{distribution}; - if (${$dep}{phase} eq q{runtime}) { - push @deps_runtime, $distname; - } - else { # configure, build - push @deps_build, $distname; - } - } - unshift @deps_build, q{Module-Build} if $mb; - $deps_build{$name} = \@deps_build; - $deps_runtime{$name} = \@deps_runtime; - foreach my $distname (@deps_build) { - fetch( $distname, 0, 1 ); - } - foreach my $distname (@deps_runtime) { - fetch( $distname, $need_target, $need_host ); - $need_dlopen{$name} ||= $need_dlopen{$distname}; - } - } - return; -} - -foreach my $distname (@ARGV) { - # Command-line's distributions - fetch( $distname, !!$target, !!$host ); -} -say scalar keys %dist, q{ packages fetched.} unless $quiet; - -# Buildroot package name: lowercase -sub fsname { - my $name = shift; - return q{perl-} . lc $name; -} - -# Buildroot variable name: uppercase -sub brname { - my $name = shift; - $name =~ s|-|_|g; - return uc $name; -} - -while (my ($distname, $dist) = each %dist) { - my $fsname = fsname( $distname ); - my $dirname = q{package/} . $fsname; - my $cfgname = $dirname . q{/Config.in}; - my $mkname = $dirname . q{/} . $fsname . q{.mk}; - my $brname = brname( $fsname ); - mkdir $dirname unless -d $dirname; - if ($need_target{$distname} && ($force || !-f $cfgname)) { - my $abstract = $dist->{abstract}; - my $homepage = $dist->{resources}->{homepage} || qq{https://metacpan.org/release/${distname}}; - say qq{write ${cfgname}} unless $quiet; - open my $fh, q{>}, $cfgname; - say {$fh} qq{config BR2_PACKAGE_${brname}}; - say {$fh} qq{\tbool "${fsname}"}; - say {$fh} qq{\tdepends on !BR2_STATIC_LIBS} if $need_dlopen{$distname}; - foreach my $dep (sort @{$deps_runtime{$distname}}) { - my $brdep = brname( fsname( $dep ) ); - say {$fh} qq{\tselect BR2_PACKAGE_${brdep}}; - } - say {$fh} qq{\thelp}; - say {$fh} qq{\t ${abstract}\n} if $abstract; - say {$fh} qq{\t ${homepage}}; - if ($need_dlopen{$distname}) { - say {$fh} qq{\ncomment "${fsname} needs a toolchain w/ dynamic library"}; - say {$fh} qq{\tdepends on BR2_STATIC_LIBS}; - } - close $fh; - } - if ($force || !-f $mkname) { - my $version = $dist->{version}; - my($path) = $dist->{download_url} =~ m|^[^:/?#]+://[^/?#]*([^?#]*)|; - # this URL contains only the scheme, auth and path parts (but no query and fragment parts) - # the scheme is not used, because the job is done by the BR download infrastructure - # the auth part is not used, because we use $(BR2_CPAN_MIRROR) - my($filename, $directories, $suffix) = fileparse( $path, q{tar.gz}, q{tgz} ); - my $dependencies = join q{ }, qw( = perl ), - map( { q{host-} . fsname( $_ ); } sort @{$deps_build{$distname}} ), - map( { fsname( $_ ); } sort @{$deps_runtime{$distname}} ); - my $host_dependencies = join q{ }, qw( = ), - map { q{host-} . fsname( $_ ); } sort( @{$deps_build{$distname}}, - @{$deps_runtime{$distname}} ); - my $license = ref $dist->{license} eq 'ARRAY' - ? join q{ or }, @{$dist->{license}} - : $dist->{license}; - # BR requires license name as in http://spdx.org/licenses/ - $license =~ s|apache_2_0|Apache-2.0|; - $license =~ s|artistic_2|Artistic-2.0|; - $license =~ s|mit|MIT|; - $license =~ s|openssl|OpenSSL|; - $license =~ s|perl_5|Artistic or GPLv1+|; - my $license_files = join q{ }, @{$license_files{$distname}}; - say qq{write ${mkname}} unless $quiet; - open my $fh, q{>}, $mkname; - say {$fh} qq{################################################################################}; - say {$fh} qq{#}; - say {$fh} qq{# ${fsname}}; - say {$fh} qq{#}; - say {$fh} qq{################################################################################}; - say {$fh} qq{}; - say {$fh} qq{${brname}_VERSION = ${version}}; - say {$fh} qq{${brname}_SOURCE = ${distname}-\$(${brname}_VERSION).${suffix}}; - say {$fh} qq{${brname}_SITE = \$(BR2_CPAN_MIRROR)${directories}}; - say {$fh} qq{${brname}_DEPENDENCIES ${dependencies}} if $need_target{$distname}; - say {$fh} qq{HOST_${brname}_DEPENDENCIES ${host_dependencies}} if $need_host{$distname}; - say {$fh} qq{${brname}_LICENSE = ${license}} if $license && $license ne q{unknown}; - say {$fh} qq{${brname}_LICENSE_FILES = ${license_files}} if $license_files; - say {$fh} qq{}; - say {$fh} qq{\$(eval \$(perl-package))} if $need_target{$distname}; - say {$fh} qq{\$(eval \$(host-perl-package))} if $need_host{$distname}; - close $fh; - } -} - -my %pkg; -my $cfgname = q{package/Config.in}; -if (-f $cfgname) { - open my $fh, q{<}, $cfgname; - while (<$fh>) { - chomp; - $pkg{$_} = 1 if m|package/perl-|; - } - close $fh; -} - -foreach my $distname (keys %need_target) { - my $fsname = fsname( $distname ); - $pkg{qq{\tsource "package/${fsname}/Config.in"}} = 1; -} - -say qq{${cfgname} must contain the following lines:}; -say join qq{\n}, sort keys %pkg; - -__END__ - -=head1 NAME - -support/scripts/scancpan Try-Tiny Moo - -=head1 SYNOPSIS - -curl -kL http://install.perlbrew.pl | bash - -perlbrew install perl-5.18.2 - -supports/scripts/scancpan [options] [distname ...] - - Options: - -help - -man - -quiet - -force - -target/-notarget - -host/-nohost - -recommend - -=head1 OPTIONS - -=over 8 - -=item B<-help> - -Prints a brief help message and exits. - -=item B<-man> - -Prints the manual page and exits. - -=item B<-quiet> - -Executes without output - -=item B<-force> - -Forces the overwriting of existing files. - -=item B<-target/-notarget> - -Switches package generation for the target variant (the default is C<-target>). - -=item B<-host/-nohost> - -Switches package generation for the host variant (the default is C<-nohost>). - -=item B<-recommend> - -Adds I dependencies. - -=back - -=head1 DESCRIPTION - -This script creates templates of the Buildroot package files for all the -Perl/CPAN distributions required by the specified distnames. The -dependencies and metadata are fetched from https://metacpan.org/. - -After running this script, it is necessary to check the generated files. -You have to manually add the license files (PERL_FOO_LICENSE_FILES variable). -For distributions that link against a target library, you have to add the -buildroot package name for that library to the DEPENDENCIES variable. - -See the Buildroot documentation for details on the usage of the Perl -infrastructure. - -The major version of the host perl must be aligned on the target one, -in order to work with the right CoreList data. - -=head1 LICENSE - -Copyright (C) 2013-2014 by Francois Perrad - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -This script is a part of Buildroot. - -This script requires the module C (version 1.131730) -which was included at the beginning of this file by the tool C. - -See L. - -See L. - -These both libraries are free software and may be distributed under the same -terms as perl itself. - -And perl may be distributed under the terms of Artistic v1 or GPL v1 license. - -=cut diff --git a/support/scripts/setlocalversion b/support/scripts/setlocalversion index adeeb781e7..b39b751f03 100755 --- a/support/scripts/setlocalversion +++ b/support/scripts/setlocalversion @@ -53,8 +53,8 @@ if head=`git rev-parse --verify --short HEAD 2>/dev/null`; then fi # Check for mercurial and a mercurial repo. -if hgid=`hg id 2>/dev/null`; then - tag=`printf '%s' "$hgid" | cut -d' ' -f2` +if hgid=`HGRCPATH= hg id --id --tags 2>/dev/null`; then + tag=`printf '%s' "$hgid" | cut -d' ' -f2 --only-delimited` # Do we have an untagged version? if [ -z "$tag" -o "$tag" = tip ]; then diff --git a/support/scripts/size-stats b/support/scripts/size-stats new file mode 100755 index 0000000000..dea3a6007c --- /dev/null +++ b/support/scripts/size-stats @@ -0,0 +1,308 @@ +#!/usr/bin/env python + +# Copyright (C) 2014 by Thomas Petazzoni + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import sys +import os +import os.path +import argparse +import csv +import collections +import math + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.font_manager as fm + import matplotlib.pyplot as plt +except ImportError: + sys.stderr.write("You need python-matplotlib to generate the size graph\n") + exit(1) + + +class Config: + biggest_first = False + iec = False + size_limit = 0.01 + colors = ['#e60004', '#f28e00', '#ffed00', '#940084', + '#2e1d86', '#0068b5', '#009836', '#97c000'] + + +# +# This function adds a new file to 'filesdict', after checking its +# size. The 'filesdict' contain the relative path of the file as the +# key, and as the value a tuple containing the name of the package to +# which the file belongs and the size of the file. +# +# filesdict: the dict to which the file is added +# relpath: relative path of the file +# fullpath: absolute path to the file +# pkg: package to which the file belongs +# +def add_file(filesdict, relpath, abspath, pkg): + if not os.path.exists(abspath): + return + if os.path.islink(abspath): + return + sz = os.stat(abspath).st_size + filesdict[relpath] = (pkg, sz) + + +# +# This function returns a dict where each key is the path of a file in +# the root filesystem, and the value is a tuple containing two +# elements: the name of the package to which this file belongs and the +# size of the file. +# +# builddir: path to the Buildroot output directory +# +def build_package_dict(builddir): + filesdict = {} + with open(os.path.join(builddir, "build", "packages-file-list.txt")) as f: + for l in f.readlines(): + pkg, fpath = l.split(",", 1) + # remove the initial './' in each file path + fpath = fpath.strip()[2:] + fullpath = os.path.join(builddir, "target", fpath) + add_file(filesdict, fpath, fullpath, pkg) + return filesdict + + +# +# This function builds a dictionary that contains the name of a +# package as key, and the size of the files installed by this package +# as the value. +# +# filesdict: dictionary with the name of the files as key, and as +# value a tuple containing the name of the package to which the files +# belongs, and the size of the file. As returned by +# build_package_dict. +# +# builddir: path to the Buildroot output directory +# +def build_package_size(filesdict, builddir): + pkgsize = collections.defaultdict(int) + + seeninodes = set() + for root, _, files in os.walk(os.path.join(builddir, "target")): + for f in files: + fpath = os.path.join(root, f) + if os.path.islink(fpath): + continue + + st = os.stat(fpath) + if st.st_ino in seeninodes: + # hard link + continue + else: + seeninodes.add(st.st_ino) + + frelpath = os.path.relpath(fpath, os.path.join(builddir, "target")) + if frelpath not in filesdict: + print("WARNING: %s is not part of any package" % frelpath) + pkg = "unknown" + else: + pkg = filesdict[frelpath][0] + + pkgsize[pkg] += st.st_size + + return pkgsize + + +# +# Given a dict returned by build_package_size(), this function +# generates a pie chart of the size installed by each package. +# +# pkgsize: dictionary with the name of the package as a key, and the +# size as the value, as returned by build_package_size. +# +# outputf: output file for the graph +# +def draw_graph(pkgsize, outputf): + def size2string(sz): + if Config.iec: + divider = 1024.0 + prefixes = ['', 'Ki', 'Mi', 'Gi', 'Ti'] + else: + divider = 1000.0 + prefixes = ['', 'k', 'M', 'G', 'T'] + while sz > divider and len(prefixes) > 1: + prefixes = prefixes[1:] + sz = sz/divider + # precision is made so that there are always at least three meaningful + # digits displayed (e.g. '3.14' and '10.4', not just '3' and '10') + precision = int(2-math.floor(math.log10(sz))) if sz < 1000 else 0 + return '{:.{prec}f} {}B'.format(sz, prefixes[0], prec=precision) + + total = sum(pkgsize.values()) + labels = [] + values = [] + other_value = 0 + unknown_value = 0 + for (p, sz) in sorted(pkgsize.items(), key=lambda x: x[1], + reverse=Config.biggest_first): + if sz < (total * Config.size_limit): + other_value += sz + elif p == "unknown": + unknown_value = sz + else: + labels.append("%s (%s)" % (p, size2string(sz))) + values.append(sz) + if unknown_value != 0: + labels.append("Unknown (%s)" % (size2string(unknown_value))) + values.append(unknown_value) + if other_value != 0: + labels.append("Other (%s)" % (size2string(other_value))) + values.append(other_value) + + plt.figure() + patches, texts, autotexts = plt.pie(values, labels=labels, + autopct='%1.1f%%', shadow=True, + colors=Config.colors) + # Reduce text size + proptease = fm.FontProperties() + proptease.set_size('xx-small') + plt.setp(autotexts, fontproperties=proptease) + plt.setp(texts, fontproperties=proptease) + + plt.suptitle("Filesystem size per package", fontsize=18, y=.97) + plt.title("Total filesystem size: %s" % (size2string(total)), fontsize=10, + y=.96) + plt.savefig(outputf) + + +# +# Generate a CSV file with statistics about the size of each file, its +# size contribution to the package and to the overall system. +# +# filesdict: dictionary with the name of the files as key, and as +# value a tuple containing the name of the package to which the files +# belongs, and the size of the file. As returned by +# build_package_dict. +# +# pkgsize: dictionary with the name of the package as a key, and the +# size as the value, as returned by build_package_size. +# +# outputf: output CSV file +# +def gen_files_csv(filesdict, pkgsizes, outputf): + total = 0 + for (p, sz) in pkgsizes.items(): + total += sz + with open(outputf, 'w') as csvfile: + wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL) + wr.writerow(["File name", + "Package name", + "File size", + "Package size", + "File size in package (%)", + "File size in system (%)"]) + for f, (pkgname, filesize) in filesdict.items(): + pkgsize = pkgsizes[pkgname] + + if pkgsize == 0: + percent_pkg = 0 + else: + percent_pkg = float(filesize) / pkgsize * 100 + + percent_total = float(filesize) / total * 100 + + wr.writerow([f, pkgname, filesize, pkgsize, + "%.1f" % percent_pkg, + "%.1f" % percent_total]) + + +# +# Generate a CSV file with statistics about the size of each package, +# and their size contribution to the overall system. +# +# pkgsize: dictionary with the name of the package as a key, and the +# size as the value, as returned by build_package_size. +# +# outputf: output CSV file +# +def gen_packages_csv(pkgsizes, outputf): + total = sum(pkgsizes.values()) + with open(outputf, 'w') as csvfile: + wr = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL) + wr.writerow(["Package name", "Package size", + "Package size in system (%)"]) + for (pkg, size) in pkgsizes.items(): + wr.writerow([pkg, size, "%.1f" % (float(size) / total * 100)]) + + +# +# Our special action for --iec, --binary, --si, --decimal +# +class PrefixAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + for key in ["type", "nargs"]: + if key in kwargs: + raise ValueError('"{}" not allowed'.format(key)) + super(PrefixAction, self).__init__(option_strings, dest, nargs=0, + type=bool, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, option_string in ["--iec", "--binary"]) + + +def main(): + parser = argparse.ArgumentParser(description='Draw size statistics graphs') + + parser.add_argument("--builddir", '-i', metavar="BUILDDIR", required=True, + help="Buildroot output directory") + parser.add_argument("--graph", '-g', metavar="GRAPH", + help="Graph output file (.pdf or .png extension)") + parser.add_argument("--file-size-csv", '-f', metavar="FILE_SIZE_CSV", + help="CSV output file with file size statistics") + parser.add_argument("--package-size-csv", '-p', metavar="PKG_SIZE_CSV", + help="CSV output file with package size statistics") + parser.add_argument("--biggest-first", action='store_true', + help="Sort packages in decreasing size order, " + + "rather than in increasing size order") + parser.add_argument("--iec", "--binary", "--si", "--decimal", + action=PrefixAction, + help="Use IEC (binary, powers of 1024) or SI (decimal, " + "powers of 1000, the default) prefixes") + parser.add_argument("--size-limit", "-l", type=float, + help='Under this size ratio, files are accounted to ' + + 'the generic "Other" package. Default: 0.01 (1%%)') + args = parser.parse_args() + + Config.biggest_first = args.biggest_first + Config.iec = args.iec + if args.size_limit is not None: + if args.size_limit < 0.0 or args.size_limit > 1.0: + parser.error("--size-limit must be in [0.0..1.0]") + Config.size_limit = args.size_limit + + # Find out which package installed what files + pkgdict = build_package_dict(args.builddir) + + # Collect the size installed by each package + pkgsize = build_package_size(pkgdict, args.builddir) + + if args.graph: + draw_graph(pkgsize, args.graph) + if args.file_size_csv: + gen_files_csv(pkgdict, pkgsize, args.file_size_csv) + if args.package_size_csv: + gen_packages_csv(pkgsize, args.package_size_csv) + + +if __name__ == "__main__": + main() diff --git a/support/scripts/xorg-release b/support/scripts/xorg-release deleted file mode 100755 index 66fc100b5e..0000000000 --- a/support/scripts/xorg-release +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/python - -# This script generates a report on the packaging status of X.org -# releases in Buildroot. It does so by downloading the list of -# tarballs that are part of a given X.org release, and compare that -# with the packages that are available in Buildroot. - -import BeautifulSoup -import re -import os -import urllib -from distutils.version import LooseVersion - -# This can be customized -XORG_VERSION = "X11R7.7" - -# Key names in dictionaries -XORG_VERSION_KEY = "xorg-version" -BR_VERSION_KEY = "br-version" -BR_NAME_KEY = "br-name" - -# Packages part of X.org releases that we do not want to package in -# Buildroot (old drivers for hardware unlikely to be used in embedded -# contexts). -XORG_EXCEPTIONS = [ - 'xf86-video-suncg6', - 'xf86-video-sunffb', -] - -# Get the list of tarballs of a X.org release, parse it, and return a -# dictionary of dictionaries, of the form: -# -# { : { XORG_VERSION_KEY: }, -# : { XORG_VERSION_KEY: }} -# -def get_xorg_release_pkgs(): - u = urllib.URLopener().open("http://www.x.org/releases/%s/src/everything/" % XORG_VERSION) - b = BeautifulSoup.BeautifulSoup() - b.feed(u.read()) - links = b.findAll("a") - packages = {} - r = re.compile("(.*)-([0-9\.]*).tar.bz2") - # We now have a list of all links. - for link in links: - href = link.get("href") - # Skip everything but tarballs - if not href.endswith(".tar.bz2"): - continue - # Separate the name and the version - groups = r.match(href) - if not groups: - continue - name = groups.group(1) - version = groups.group(2) - # Skip packages we don't want to hear about - if name in XORG_EXCEPTIONS: - continue - packages[name] = { XORG_VERSION_KEY : version } - return packages - -# Files and directories in package/x11r7/ that should be ignored in -# our processing. -BUILDROOT_EXCEPTIONS = [ - "mcookie", # Code is directly in package directory - "x11r7.mk", - "Config.in", - "xdriver_xf86-input-tslib", # From Pengutronix, not part of X.org releases -] - -# Prefixes of directories in package/x11r7/ that must be stripped -# before trying to match Buildroot package names with X.org tarball -# names. -BUILDROOT_PREFIXES = [ - "xapp", - "xdriver", - "xfont", - "xlib", - "xserver", - "xutil", - "xproto", -] - -# From a Buildroot package name, try to see if a prefix should be -# stripped from it. For example, passing "xapp_xlsfonts" as argument -# to this function will return "xlsfonts". -def buildroot_strip_prefix(dirname): - for prefix in BUILDROOT_PREFIXES: - if dirname.startswith(prefix + "_"): - return dirname[len(prefix) + 1:] - return dirname - -# From a Buildroot package name, parse its .mk file to find the -# Buildroot version of the package by looking at the _VERSION -# line. -def buildroot_get_version(dirname): - f = open(os.path.join("package", "x11r7", dirname, dirname + ".mk")) - r = re.compile("^([A-Z0-9_]*)_VERSION = ([0-9\.]*)$") - for l in f.readlines(): - m = r.match(l) - if m: - return m.group(2) - return None - -# Augment the information of the X.org list of packages (given as -# argument) by details about their packaging in Buildroot. Those -# details are found by looking at the contents of package/x11r7/. -def get_buildroot_pkgs(packages): - dirs = os.listdir(os.path.join(os.getcwd(), "package", "x11r7")) - for d in dirs: - # Skip exceptions - if d in BUILDROOT_EXCEPTIONS: - continue - pkgname = buildroot_strip_prefix(d) - version = buildroot_get_version(d) - if packages.has_key(pkgname): - # There is a X.org package of the same name, so we just - # add information to the existing dict entry. - packages[pkgname]['br-version'] = version - packages[pkgname]['br-name'] = d - else: - # There is no X.org package with this name, so we add a - # new dict entry. - packages[pkgname] = { BR_VERSION_KEY: version, - BR_NAME_KEY : d } - return packages - -def show_summary(packages): - FORMAT_STRING = "%40s | %15s | %15s | %-30s" - print FORMAT_STRING % ("Package name", "Vers in BR", "Vers in X.org", "Action") - print FORMAT_STRING % ("-" * 40, "-" * 15, "-" * 15, "-" * 30) - pkgs = packages.keys() - pkgs.sort() - total_pkgs = 0 - upgrade_pkgs = 0 - add_pkgs = 0 - remove_pkgs = 0 - nothing_todo_pkgs = 0 - for pkgname in pkgs: - pkg = packages[pkgname] - total_pkgs += 1 - if pkg.has_key(XORG_VERSION_KEY) and not pkg.has_key(BR_VERSION_KEY): - xorg_version = pkg[XORG_VERSION_KEY] - br_version = "N/A" - action = "Add to Buildroot" - add_pkgs += 1 - elif not pkg.has_key(XORG_VERSION_KEY) and pkg.has_key(BR_VERSION_KEY): - br_version = pkg[BR_VERSION_KEY] - xorg_version = "N/A" - action = "Remove from Buildroot" - remove_pkgs += 1 - elif LooseVersion(pkg[XORG_VERSION_KEY]) > LooseVersion(pkg[BR_VERSION_KEY]): - br_version = pkg[BR_VERSION_KEY] - xorg_version = pkg[XORG_VERSION_KEY] - action = "Upgrade" - upgrade_pkgs += 1 - elif LooseVersion(pkg[XORG_VERSION_KEY]) < LooseVersion(pkg[BR_VERSION_KEY]): - br_version = pkg[BR_VERSION_KEY] - xorg_version = pkg[XORG_VERSION_KEY] - action = "More recent" - nothing_todo_pkgs += 1 - else: - br_version = pkg[BR_VERSION_KEY] - xorg_version = pkg[XORG_VERSION_KEY] - action = "" - nothing_todo_pkgs += 1 - - print FORMAT_STRING % (pkgname, br_version.center(15), xorg_version.center(15), action) - print FORMAT_STRING % ("-" * 40, "-" * 15, "-" * 15, "-" * 30) - STAT_FORMAT_STRING = "%40s : %3d" - print STAT_FORMAT_STRING % ("Total number of packages", total_pkgs) - print STAT_FORMAT_STRING % ("Packages to upgrade", upgrade_pkgs) - print STAT_FORMAT_STRING % ("Packages to add", add_pkgs) - print STAT_FORMAT_STRING % ("Packages to remove", remove_pkgs) - print STAT_FORMAT_STRING % ("Packages with nothing to do", nothing_todo_pkgs) - -packages = get_xorg_release_pkgs() -packages = get_buildroot_pkgs(packages) -# print packages -show_summary(packages) - diff --git a/support/testing/.gitkeep b/support/testing/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/toolchain/helpers.mk b/toolchain/helpers.mk index 1452ec6917..796ba73087 100644 --- a/toolchain/helpers.mk +++ b/toolchain/helpers.mk @@ -164,11 +164,16 @@ copy_toolchain_sysroot = \ # Check the specified kernel headers version actually matches the # version in the toolchain. # -# $1: sysroot directory -# $2: kernel version string, in the form: X.Y +# $1: build directory +# $2: sysroot directory +# $3: kernel version string, in the form: X.Y +# $4: test to do for the latest kernel version, 'strict' or 'loose' +# always 'strict' if this is not the latest version. # check_kernel_headers_version = \ - if ! support/scripts/check-kernel-headers.sh $(1) $(2); then \ + if ! support/scripts/check-kernel-headers.sh $(1) $(2) $(3) \ + $(if $(BR2_TOOLCHAIN_HEADERS_LATEST),$(4),strict); \ + then \ exit 1; \ fi From 1a0b5c9044cd435cadc929b36575baedf440c70a Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 00:18:47 +0100 Subject: [PATCH 09/12] Copy DEVELOPERS from upstream Needed for support/scripts/pkg-stats --- DEVELOPERS | 2714 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2714 insertions(+) create mode 100644 DEVELOPERS diff --git a/DEVELOPERS b/DEVELOPERS new file mode 100644 index 0000000000..f1bf5c263c --- /dev/null +++ b/DEVELOPERS @@ -0,0 +1,2714 @@ +# Syntax: +# +# N: Firstname Lastname +# F: file pattern or directory +# F: file pattern or directory +# +# The "F" entries can be: +# +# - A directory, in which case all patches touching any file in this +# directory or its subdirectories will be CC'ed to the developer. +# - A pattern, in which case the pattern will be expanded, and then +# all files/directories (and their subdirectories) will be +# considered when matching against a patch +# +# Notes: +# +# - When a developer adds an "arch/Config.in." file to its list +# of files, he is considered a developer of this architecture. He +# will receive e-mail notifications about build failures occuring on +# this architecture. Not more than one e-mail per day is sent. +# - When a developer adds a directory that contains one or several +# packages, this developer will be notified when build failures +# occur. Not more than one e-mail per day is sent. +# - When a developer adds an "package/pkg-.mk" file to its list +# of files, he is considered interested by this package +# infrastructure, and will be CC'ed on all patches that add or +# modify packages that use this infrastructure. + +N: Adam Duskett +F: package/audit/ +F: package/busybox/ +F: package/checkpolicy/ +F: package/cppdb/ +F: package/gobject-introspection/ +F: package/gstreamer1/gstreamer1/ +F: package/gstreamer1/gstreamer1-mm/ +F: package/gstreamer1/gst1-plugins-bad/ +F: package/gstreamer1/gst1-plugins-base/ +F: package/gstreamer1/gst1-plugins-good/ +F: package/gstreamer1/gst1-plugins-ugly/ +F: package/gstreamer1/gst1-python/ +F: package/gstreamer1/gst1-vaapi/ +F: package/imx-usb-loader/ +F: package/janus-gateway/ +F: package/json-for-modern-cpp/ +F: package/libcpprestsdk/ +F: package/libressl/ +F: package/libselinux/ +F: package/libsemanage/ +F: package/libsepol/ +F: package/libwebsockets/ +F: package/mender-grubenv/ +F: package/nginx-naxsi/ +F: package/openjdk/ +F: package/openjdk-bin/ +F: package/php/ +F: package/policycoreutils/ +F: package/prelink-cross/ +F: package/polkit/ +F: package/python3/ +F: package/python-aioredis/ +F: package/python-asgiref/ +F: package/python-channels/ +F: package/python-channels-redis/ +F: package/python-daphne/ +F: package/python-django-enumfields/ +F: package/python-flask-sqlalchemy/ +F: package/python-gitdb2/ +F: package/python-gobject/ +F: package/python-lockfile/ +F: package/python-mutagen/ +F: package/python-nested-dict/ +F: package/python-pbr/ +F: package/python-pip/ +F: package/python-psycopg2/ +F: package/python-smmap2/ +F: package/python-sqlalchemy/ +F: package/python-sqlparse/ +F: package/python-visitor/ +F: package/restorecond/ +F: package/refpolicy/ +F: package/selinux-python/ +F: package/semodule-utils/ +F: package/setools/ +F: package/sngrep/ +F: package/spidermonkey/ +F: package/systemd/ +F: support/testing/tests/package/test_gst1_python.py +F: support/testing/tests/package/test_python_gobject.py + +N: Adam Heinrich +F: package/jack1/ + +N: Adrian Perez de Castro +F: package/brotli/ +F: package/bubblewrap/ +F: package/cog/ +F: package/libepoxy/ +F: package/libwpe/ +F: package/webkitgtk/ +F: package/woff2/ +F: package/wpebackend-fdo/ +F: package/wpewebkit/ +F: package/xdg-dbus-proxy/ + +N: Adrien Gallouët +F: package/bird/ +F: package/glorytun/ + +N: Aleksander Morgado +F: package/libmbim/ +F: package/libqmi/ +F: package/modem-manager/ + +N: Alex Michel +F: package/network-manager-openvpn/ + +N: Alex Suykov +F: board/chromebook/snow/ +F: configs/chromebook_snow_defconfig +F: package/vboot-utils/ + +N: Alexander Clouter +F: package/odhcp6c/ + +N: Alexander Dahl +F: package/fastd/ +F: package/libuecc/ +F: package/putty/ + +N: Alexander Kurz +F: package/minimodem/ + +N: Alexander Lukichev +F: package/openpgm/ + +N: Alexander Mukhin +F: package/tinyproxy/ + +N: Alexander Sverdlin +F: package/mini-snmpd/ + +N: Alexander Varnin +F: package/liblog4c-localtime/ + +N: Alexandre Belloni +F: package/tz/ + +N: Alexandre Esse +F: package/kvazaar/ +F: package/v4l2loopback/ + +N: Alexey Brodkin +F: board/cubietech/cubieboard2/ +F: configs/cubieboard2_defconfig + +N: Alistair Francis +F: board/sifive/ +F: boot/opensbi/ +F: configs/hifive_unleashed_defconfig +F: package/xen/ + +N: Alvaro G. M +F: package/dcron/ +F: package/libxmlrpc/ +F: package/python-docopt/ + +N: Anders Darander +F: package/ktap/ + +N: André Hentschel +F: board/freescale/imx8qxpmek/ +F: configs/freescale_imx8qxpmek_defconfig +F: package/freescale-imx/imx-sc-firmware/ +F: package/libkrb5/ +F: package/openal/ +F: package/p7zip/ +F: package/wine/ + +N: Andrey Smirnov +F: package/python-backports-shutil-get-terminal-size/ +F: package/python-decorator/ +F: package/python-ipython-genutils/ +F: package/python-pathlib2/ +F: package/python-pickleshare/ +F: package/python-scandir/ +F: package/python-simplegeneric/ +F: package/python-systemd/ +F: package/python-traitlets/ +F: package/zstd/ + +N: Andrey Yurovsky +F: package/rauc/ + +N: Angelo Compagnucci +F: package/corkscrew/ +F: package/fail2ban/ +F: package/i2c-tools/ +F: package/mender/ +F: package/mender-artifact/ +F: package/mono/ +F: package/mono-gtksharp3/ +F: package/monolite/ +F: package/python-can/ +F: package/python-pillow/ +F: package/python-pydal/ +F: package/python-spidev/ +F: package/python-web2py/ +F: package/sshguard/ +F: package/sunwait/ +F: package/sysdig/ + +N: Anisse Astier +F: package/go/ +F: package/nghttp2/ +F: package/pkg-golang.mk + +N: Anthony Viallard +F: package/gnuplot/ + +N: Antoine Ténart +F: package/wf111/ + +N: Antony Pavlov +F: package/lsscsi/ + +N: ARC Maintainers +F: arch/Config.in.arc +F: board/synopsys/ +F: configs/snps_arc700_axs101_defconfig +F: configs/snps_archs38_axs103_defconfig +F: configs/snps_archs38_haps_defconfig +F: configs/snps_archs38_hsdk_defconfig +F: configs/snps_archs38_vdk_defconfig + +N: Ariel D'Alessandro +F: package/axfsutils/ +F: package/mali-t76x/ + +N: Arnaud Aujon +F: package/espeak/ + +N: Arnout Vandecappelle +F: package/arp-scan/ +F: package/dehydrated/ +F: package/freescale-imx/firmware-imx/ +F: package/freescale-imx/imx-lib/ +F: package/libpagekite/ +F: package/lua-bit32/ +F: package/owfs/ +F: package/python-bottle/ +F: package/sqlcipher/ +F: package/stress/ + +N: Arthur Courtel +F: board/raspberrypi/genimage-raspberrypi4-64.cfg +F: configs/raspberrypi4_64_defconfig + +N: Asaf Kahlon +F: package/collectd/ +F: package/libuv/ +F: package/python* +F: package/snmpclitools/ +F: package/spdlog/ +F: package/uftp/ +F: package/uvw/ +F: package/zeromq/ + +N: Ash Charles +F: package/pru-software-support/ +F: package/ti-cgt-pru/ + +N: Assaf Inbal +F: package/lbase64/ +F: package/luabitop/ +F: package/luaexpatutils/ +F: package/luaposix/ +F: package/luasec/ +F: package/lua-ev/ +F: package/orbit/ + +N: Bartosz Bilas +F: board/stmicroelectronics/stm32mp157a-dk1/ +F: configs/stm32mp157a_dk1_defconfig +F: package/python-esptool/ +F: package/python-pyaes/ +F: package/qt5/qt5scxml/ +F: package/qt5/qt5webview/ + +N: Bartosz Golaszewski +F: package/autoconf-archive/ +F: package/doxygen/ +F: package/libgpiod/ +F: package/libserialport/ +F: package/libsigrok/ +F: package/libsigrokdecode/ +F: package/libzip/ +F: package/pulseview/ +F: package/sigrok-cli/ + +N: Baruch Siach +F: board/solidrun/clearfog_gt_8k/ +F: configs/solidrun_clearfog_gt_8k_defconfig +F: package/18xx-ti-utils/ +F: package/cpuburn-arm/ +F: package/daemon/ +F: package/dropbear/ +F: package/ebtables/ +F: package/i2c-tools/ +F: package/libcurl/ +F: package/libpcap/ +F: package/openipmi/ +F: package/socat/ +F: package/strace/ +F: package/tcpdump/ +F: package/ti-uim/ +F: package/uhubctl/ + +N: Ben Boeckel +F: package/taskd/ + +N: Benjamin Kamath +F: package/lapack/ + +N: Bernd Kuhls +F: package/alsa-lib/ +F: package/alsa-utils/ +F: package/apache/ +F: package/apr/ +F: package/apr-util/ +F: package/bcg729/ +F: package/bluez-tools/ +F: package/boinc/ +F: package/clamav/ +F: package/dav1d/ +F: package/dovecot/ +F: package/dovecot-pigeonhole/ +F: package/dtv-scan-tables/ +F: package/eudev/ +F: package/exim/ +F: package/fetchmail/ +F: package/ffmpeg/ +F: package/flac/ +F: package/freeswitch/ +F: package/freeswitch-mod-bcg729/ +F: package/freetype/ +F: package/fstrcmp/ +F: package/ghostscript/ +F: package/giflib/ +F: package/gli/ +F: package/glmark2/ +F: package/gpsd/ +F: package/hdparm/ +F: package/jsoncpp/ +F: package/kodi* +F: package/lame/ +F: package/leafnode2/ +F: package/libaacs/ +F: package/libasplib/ +F: package/libass/ +F: package/libbdplus/ +F: package/libbluray/ +F: package/libbroadvoice/ +F: package/libcdio/ +F: package/libcec/ +F: package/libcodec2/ +F: package/libcrossguid/ +F: package/libdcadec/ +F: package/libdrm/ +F: package/libdvbcsa/ +F: package/libdvdcss/ +F: package/libdvdnav/ +F: package/libdvdread/ +F: package/libebur128/ +F: package/libfreeglut/ +F: package/libg7221/ +F: package/libglew/ +F: package/libglfw/ +F: package/libglu/ +F: package/libhdhomerun/ +F: package/libilbc/ +F: package/libldns/ +F: package/libmicrohttpd/ +F: package/libminiupnpc/ +F: package/libmspack/ +F: package/libnatpmp/ +F: package/libnpth/ +F: package/libogg/ +F: package/libopenh264/ +F: package/libpciaccess/ +F: package/libplatform/ +F: package/libpng/ +F: package/libsidplay2/ +F: package/libsilk/ +F: package/libsndfile/ +F: package/libsoil/ +F: package/libsoundtouch/ +F: package/libsquish/ +F: package/liburiparser/ +F: package/libva/ +F: package/libva-intel-driver/ +F: package/libva-utils/ +F: package/libvorbis/ +F: package/libvpx/ +F: package/libyuv/ +F: package/mesa3d/ +F: package/minidlna/ +F: package/mjpg-streamer/ +F: package/perl-crypt-openssl-guess/ +F: package/perl-crypt-openssl-random/ +F: package/perl-crypt-openssl-rsa/ +F: package/perl-digest-sha1/ +F: package/perl-encode-detect/ +F: package/perl-encode-locale/ +F: package/perl-file-listing/ +F: package/perl-html-parser/ +F: package/perl-html-tagset/ +F: package/perl-http-cookies/ +F: package/perl-http-daemon/ +F: package/perl-http-date/ +F: package/perl-http-message/ +F: package/perl-http-negotiate/ +F: package/perl-io-html/ +F: package/perl-lwp-mediatypes/ +F: package/perl-mail-dkim/ +F: package/perl-mailtools/ +F: package/perl-net-dns/ +F: package/perl-net-http/ +F: package/perl-netaddr-ip/ +F: package/perl-timedate/ +F: package/perl-uri/ +F: package/perl-www-robotrules/ +F: package/pixman/ +F: package/pngquant/ +F: package/pound/ +F: package/pulseaudio/ +F: package/pure-ftpd/ +F: package/python-couchdb/ +F: package/python-cssutils/ +F: package/python-futures/ +F: package/python-mwclient/ +F: package/python-mwscrape/ +F: package/python-mwscrape2slob/ +F: package/python-mako/ +F: package/python-oauthlib/ +F: package/python-pyicu/ +F: package/python-pylru/ +F: package/python-requests-oauthlib/ +F: package/python-slob/ +F: package/rtmpdump/ +F: package/samba4/ +F: package/softether/ +F: package/spandsp/ +F: package/sqlite/ +F: package/stellarium/ +F: package/taglib/ +F: package/tinyxml2/ +F: package/tor/ +F: package/transmission/ +F: package/tvheadend/ +F: package/unixodbc/ +F: package/utf8proc/ +F: package/vdr/ +F: package/vdr-plugin-vnsiserver/ +F: package/vlc/ +F: package/vnstat/ +F: package/waylandpp/ +F: package/x11r7/ +F: package/x264/ +F: package/x265/ +F: package/ytree/ +F: package/znc/ +F: support/testing/tests/package/test_perl_html_parser.py + +N: Biagio Montaruli +F: board/acmesystems/ +F: configs/acmesystems_* + +N: Bimal Jacob +F: package/nginx-upload/ + +N: Bogdan Radulescu +F: package/iftop/ +F: package/ncdu/ + +N: Brandon Maier +F: package/vmtouch/ + +N: Brock Williams +F: package/pdmenu/ + +N: Carlo Caione +F: package/jailhouse/ +F: package/sunxi-boards/ + +N: Carlos Santos +F: package/busybox/ +F: package/cups/ +F: package/cups-filters/ +F: package/gtest/ +F: package/initscripts/ +F: package/intel-microcode/ +F: package/libpam-radius-auth/ +F: package/libpam-tacplus/ +F: package/liburiparser/ +F: package/modem-manager/ +F: package/pamtester/ +F: package/pcm-tools/ +F: package/perl-file-util/ +F: package/skeleton-custom/ +F: package/skeleton-init-common/ +F: package/skeleton-init-none/ +F: package/skeleton-init-systemd/ +F: package/skeleton-init-sysv/ +F: package/skeleton/ +F: package/sysvinit/ +F: package/udev-gentoo-scripts/ +F: package/util-linux/ +F: package/tpm2-abrmd/ +F: package/tpm2-tools/ +F: package/tpm2-totp/ +F: package/tpm2-tss/ + +N: Carsten Schoenert +F: package/dvbsnoop/ +F: package/libdvbsi/ +F: package/libsvg/ +F: package/libsvg-cairo/ + +N: Cédric Chépied +F: package/znc/ + +N: Chakra Divi +F: board/friendlyarm/nanopi-m1 +F: board/friendlyarm/nanopi-m1-plus +F: board/olimex/a13_olinuxino +F: board/orangepi/orangepi-plus +F: configs/nanopi_m1_defconfig +F: configs/nanopi_m1_plus_defconfig +F: configs/olimex_a13_olinuxino_defconfig +F: configs/orangepi_plus_defconfig + +N: Chris Packham +F: package/gstreamer1/gst1-shark/ +F: package/micropython/ +F: package/micropython-lib/ +F: package/syslog-ng/ + +N: Christian Kellermann +F: package/python-pylibftdi/ + +N: Christian Stewart +F: linux/linux-ext-aufs.mk +F: package/aufs/ +F: package/aufs-util/ +F: package/batman-adv/ +F: package/docker-cli/ +F: package/docker-containerd/ +F: package/docker-engine/ +F: package/docker-proxy/ +F: package/go/ +F: package/mosh/ +F: package/pkg-golang.mk +F: package/rtl8821au/ +F: package/runc/ +F: package/tini/ + +N: Christophe Priouzeau +F: board/stmicroelectronics/stm32f429-disco/ +F: board/stmicroelectronics/stm32f469-disco/ +F: configs/stm32f429_disco_defconfig +F: configs/stm32f469_disco_defconfig + +N: Christophe Vu-Brugier +F: package/drbd-utils/ +F: package/iotop/ +F: package/python-configshell-fb/ +F: package/python-rtslib-fb/ +F: package/python-urwid/ +F: package/targetcli-fb/ + +N: Christopher McCrory +F: package/perl-appconfig/ +F: package/perl-astro-suntime/ +F: package/perl-class-load/ +F: package/perl-class-std/ +F: package/perl-class-std-fast/ +F: package/perl-data-dump/ +F: package/perl-data-optlist/ +F: package/perl-data-uuid/ +F: package/perl-date-manip/ +F: package/perl-dbd-mysql/ +F: package/perl-dbi/ +F: package/perl-device-serialport/ +F: package/perl-dist-checkconflicts/ +F: package/perl-file-slurp/ +F: package/perl-io-interface/ +F: package/perl-io-socket-multicast/ +F: package/perl-json-maybexs/ +F: package/perl-mime-tools/ +F: package/perl-module-implementation/ +F: package/perl-module-runtime/ +F: package/perl-number-bytes-human/ +F: package/perl-package-stash/ +F: package/perl-params-util/ +F: package/perl-sub-install/ +F: package/perl-sys-cpu/ +F: package/perl-sys-meminfo/ +F: package/perl-sys-mmap/ +F: package/perl-time-parsedate/ +F: package/perl-x10/ + +N: Clayton Shotwell +F: package/audit/ +F: package/checkpolicy/ +F: package/cpio/ +F: package/libcgroup/ +F: package/libee/ +F: package/libestr/ +F: package/liblogging/ +F: package/libselinux/ +F: package/libsemanage/ +F: package/libsepol/ +F: package/policycoreutils/ + +N: Clément Péron +F: board/beelink/gs1/ +F: configs/beelink_gs1_defconfig + +N: Corentin Guillevic +F: package/libloki/ + +N: Cyril Bur +F: arch/Config.in.powerpc +F: package/kvm-unit-tests + +N: Daniel J. Leach +F: package/dacapo/ + +N: Damien Lanson +F: package/libvdpau/ +F: package/log4cpp/ + +N: Daniel Nicoletti +F: package/cutelyst/ + +N: Daniel Price +F: package/nodejs/ +F: package/redis/ + +N: Daniel Sangue +F: package/libftdi1/ + +N: Danomi Manchego +F: package/cjson/ +F: package/jq/ +F: package/libwebsockets/ +F: package/ljsyscall/ +F: package/lua-cjson/ +F: package/luaexpat/ +F: package/xinetd/ + +N: David Bachelart +F: package/ccrypt/ +F: package/dos2unix/ +F: package/ipmiutil/ +F: package/jsmn/ +F: package/python-daemon/ +F: package/sslh/ +F: package/udpxy/ + +N: David Bender +F: package/benejson/ +F: package/cgic/ +F: package/freeradius-client/ +F: package/openldap/ + +N: David du Colombier <0intro@gmail.com> +F: package/x264/ + +N: David Graziano +F: package/libcsv/ + +N: David Lechner +F: board/lego/ev3/ +F: configs/lego_ev3_defconfig +F: linux/linux-ext-ev3dev-linux-drivers.mk +F: package/brickd/ +F: package/ev3dev-linux-drivers/ + +N: Davide Viti +F: package/flann/ +F: package/python-paho-mqtt/ +F: package/qhull/ +F: package/tcllib/ + +N: Denis Bodor +F: package/libstrophe/ + +N: Dimitrios Siganos +F: package/wireless-regdb/ + +N: Dominik Faessler +F: package/logsurfer/ +F: package/python-id3/ + +N: Doug Kehn +F: package/nss-pam-ldapd/ +F: package/sp-oops-extract/ +F: package/unscd/ + +N: Dushara Jayasinghe +F: package/prosody/ + +N: Eloi Bail +F: package/bayer2rgb-neon/ +F: package/gstreamer1/gst1-plugins-bayer2rgb-neon/ + +N: Eric Le Bihan +F: docs/manual/adding-packages-meson.txt +F: package/adwaita-icon-theme/ +F: package/cargo-bin/ +F: package/cargo/ +F: package/darkhttpd/ +F: package/eudev/ +F: package/execline/ +F: package/hicolor-icon-theme/ +F: package/jemalloc/ +F: package/mdevd/ +F: package/meson/ +F: package/ninja/ +F: package/pkg-meson.mk +F: package/rust-bin/ +F: package/rust/ +F: package/s6/ +F: package/s6-dns/ +F: package/s6-linux-init/ +F: package/s6-linux-utils/ +F: package/s6-networking/ +F: package/s6-portable-utils/ +F: package/s6-rc/ +F: package/skalibs/ +F: package/smack/ +F: package/xvisor/ + +N: Eric Limpens +F: package/pifmrds/ +F: package/ympd/ + +N: Erico Nunes +F: board/aarch64-efi/ +F: configs/aarch64_efi_defconfig +F: package/acpica/ +F: package/acpitool/ +F: package/efibootmgr/ +F: package/efivar/ +F: package/fwts/ +F: package/spi-tools/ +F: package/xdotool/ +F: configs/pc_x86_64_* + +N: Erik Larsson +F: package/imx-mkimage/ + +N: Erik Stromdahl +F: package/mxsldr/ + +N: Ernesto L. Williams Jr +F: package/szip/ + +N: Esben Haabendal +F: boot/gummiboot/ +F: package/python-kiwisolver/ + +N: Etienne Carriere +F: boot/optee-os/ +F: package/optee-benchmark/ +F: package/optee-client/ +F: package/optee-examples/ +F: package/optee-test/ + +N: Eugene Tarassov +F: package/tcf-agent/ + +N: Evan Zelkowitz +F: package/sdl_gfx/ + +N: Ezequiel Garcia +F: board/ci20/ +F: configs/ci20_defconfig +F: arch/Config.in.nios2 +F: package/fio/ +F: package/iptraf-ng/ +F: package/jimtcl/ +F: package/mimic/ +F: package/nodm/ +F: package/openbox/ +F: package/rtl8723bs/ +F: package/supertuxkart/ + +N: Fabio Estevam +F: board/freescale/warpboard/ +F: board/warp7/ +F: configs/freescale_imx* +F: configs/imx23evk_defconfig +F: configs/imx6-sabre* +F: configs/imx6slevk_defconfig +F: configs/imx6sx-sdb_defconfig +F: configs/imx6ulevk_defconfig +F: configs/imx6ulpico_defconfig +F: configs/imx7d-sdb_defconfig +F: configs/imx7dpico_defconfig +F: configs/mx25pdk_defconfig +F: configs/mx51evk_defconfig +F: configs/mx53loco_defconfig +F: configs/mx6cubox_defconfig +F: configs/mx6sx_udoo_neo_defconfig +F: configs/mx6udoo_defconfig +F: configs/wandboard_defconfig +F: configs/warp7_defconfig +F: configs/warpboard_defconfig +F: package/atest/ +F: package/kmscube/ + +N: Fabio Porcedda +F: package/netsurf-buildsystem/ + +N: Fabio Urquiza +F: package/bitcoin/ + +N: Fabrice Fontaine +F: package/domoticz/ +F: package/libmediaart/ +F: package/libmaxminddb/ +F: package/openzwave/ + +N: Fabrice Fontaine +F: package/bearssl/ +F: package/belle-sip/ +F: package/belr/ +F: package/boinc/ +F: package/cairo/ +F: package/duktape/ +F: package/expat/ +F: package/flatbuffers/ +F: package/gerbera/ +F: package/gtksourceview/ +F: package/gssdp/ +F: package/gupnp/ +F: package/gupnp-dlna/ +F: package/gupnp-tools/ +F: package/haproxy/ +F: package/hiredis/ +F: package/i2pd/ +F: package/igd2-for-linux/ +F: package/json-c/ +F: package/lcms2/ +F: package/lftp/ +F: package/libcap-ng/ +F: package/libcdio-paranoia/ +F: package/libcgicc/ +F: package/libconfig/ +F: package/libcue/ +F: package/libebml/ +F: package/libgee/ +F: package/libglib2/ +F: package/libgtk2/ +F: package/libgtk3/ +F: package/libhtp/ +F: package/libidn/ +F: package/libidn2/ +F: package/libjpeg/ +F: package/liblockfile/ +F: package/libmatroska/ +F: package/libmpdclient/ +F: package/libnetfilter_conntrack/ +F: package/libnetfilter_queue/ +F: package/liboping/ +F: package/libpfm4/ +F: package/libraw/ +F: package/libraw1394/ +F: package/libroxml/ +F: package/librsvg/ +F: package/librsync/ +F: package/libsoup/ +F: package/libsoxr/ +F: package/libupnp/ +F: package/libupnp18/ +F: package/libv4l/ +F: package/libxslt/ +F: package/mbedtls/ +F: package/minissdpd/ +F: package/minizip/ +F: package/mongodb/ +F: package/motion/ +F: package/mutt/ +F: package/ncmpc/ +F: package/oniguruma/ +F: package/oprofile/ +F: package/pcmanfm/ +F: package/python-backcall/ +F: package/python-jedi/ +F: package/python-parso/ +F: package/rocksdb/ +F: package/rygel/ +F: package/safeclib/ +F: package/suricata/ +F: package/tinycbor/ +F: package/tinydtls/ +F: package/tinymembench/ +F: package/whois/ + +N: Fabrice Goucem +F: board/freescale/imx6ullevk/ +F: configs/freescale_imx6ullevk_defconfig + +N: Falco Hyfing +F: package/python-pymodbus/ + +N: Floris Bos +F: package/ipmitool/ +F: package/odhcploc/ + +N: Francisco Gonzalez +F: package/ser2net/ + +N: Francois Perrad +F: board/olimex/a20_olinuxino +F: board/olimex/imx233_olinuxino/ +F: configs/olimex_a20_olinuxino_* +F: configs/olimex_imx233_olinuxino_defconfig +F: package/4th/ +F: package/cgilua/ +F: package/chipmunk/ +F: package/cog/ +F: package/collectl/ +F: package/copas/ +F: package/coxpcall/ +F: package/dado/ +F: package/ficl/ +F: package/libtomcrypt/ +F: package/libtommath/ +F: package/libwpe/ +F: package/linenoise/ +F: package/ljlinenoise/ +F: package/lpeg/ +F: package/lpty/ +F: package/lrandom/ +F: package/lsqlite3/ +F: package/lua* +F: package/lzlib/ +F: package/moarvm/ +F: package/netsurf/ +F: package/perl* +F: package/pkg-perl.mk +F: package/pkg-luarocks.mk +F: package/rings/ +F: package/tekui/ +F: package/wpebackend-fdo/ +F: package/wpewebkit/ +F: package/wsapi/ +F: package/wsapi-fcgi/ +F: package/wsapi-xavante/ +F: package/xavante/ +F: utils/scancpan + +N: Frank Hunleth +F: package/am335x-pru-package/ +F: package/libconfuse/ +F: package/libdmtx/ +F: package/libsodium/ +F: package/php-amqp/ +F: package/python-cherrypy/ +F: package/rabbitmq-server/ +F: package/sane-backends/ +F: package/ucl/ +F: package/upx/ +F: package/zxing-cpp/ + +N: Frank Vanbever +F: package/elixir/ +F: package/libmodsecurity/ +F: package/nginx-modsecurity/ + +N: Gaël Portay +F: package/qt5/qt5virtualkeyboard/ +F: package/qt5/qt5webengine/ +F: package/qt5/qt5webkit/ +F: package/qt5/qt5webkit-examples/ + +N: Gao Xiang +F: package/erofs-utils/ + +N: Gary Bisson +F: board/boundarydevices/ +F: configs/nitrogen* +F: package/freescale-imx/ +F: package/gstreamer1/gst1-imx/ +F: package/libimxvpuapi/ +F: package/mfgtools/ +F: package/sshpass/ +F: package/x11r7/xdriver_xf86-video-imx-viv/ + +N: Geoff Levand +F: package/flannel/ + +N: Geoffrey Ragot +F: package/python-pycli/ +F: package/python-pyyaml/ + +N: Gerome Burlats +F: board/qemu/ +F: configs/qemu_* + +N: Gilles Talis +F: board/freescale/imx8mmevk/ +F: configs/freescale_imx8mmevk_defconfig +F: package/cctz/ +F: package/fdk-aac/ +F: package/httping/ +F: package/iozone/ +F: package/leptonica/ +F: package/libeXosip2/ +F: package/libolm/ +F: package/libosip2/ +F: package/ocrad/ +F: package/restclient-cpp/ +F: package/tesseract-ocr/ +F: package/webp/ +F: package/xapian/ + +N: Giulio Benetti +F: package/at/ +F: package/libnspr/ +F: package/libnss/ +F: package/minicom/ +F: package/nfs-utils/ +F: package/sunxi-mali-mainline/ +F: package/sunxi-mali-mainline-driver/ + +N: Gregory Dymarek +F: package/ding-libs/ +F: package/gengetopt/ +F: package/janus-gateway/ +F: package/libnice/ +F: package/libsrtp/ +F: package/libwebsock/ +F: package/sofia-sip/ + +N: Grzegorz Blach +F: fs/f2fs/ +F: package/bluez5_utils-headers/ +F: package/f2fs-tools/ +F: package/pigpio/ +F: package/python-aioblescan/ +F: package/python-bluezero/ +F: package/python-crontab/ +F: package/python-falcon/ +F: package/python-ifaddr/ +F: package/python-hiredis/ +F: package/python-mimeparse/ +F: package/python-pigpio/ +F: package/python-pyjwt/ +F: package/python-redis/ +F: package/python-rpi-ws281x/ +F: package/python-wtforms/ + +N: Guillaume Gardet +F: package/c-icap/ +F: package/c-icap-modules/ +F: package/sdl2/ + +N: Guillaume William Brs +F: package/liquid-dsp/ +F: package/pixiewps/ +F: package/reaver/ + +N: Guo Ren +F: arch/Config.in.csky +F: board/csky/ +F: board/qemu/csky +F: configs/csky_* +F: configs/qemu_csky* + +N: Gustavo Pimentel +F: configs/arm_juno_defconfig +F: board/arm/juno/ + +N: Gwenhael Goavec-Merou +F: package/gnuradio/ +F: package/gqrx/ +F: package/gr-osmosdr/ +F: package/libusbgx/ +F: package/python-cheetah/ +F: package/python-markdown/ +F: package/python-remi/ +F: package/python-sip/ + +N: Heiko Thiery +F: package/libnetconf2/ +F: package/libyang/ +F: package/sysrepo/ + +N: Henrique Camargo +F: package/json-glib/ + +N: Hiroshi Kawashima +F: package/gauche/ +F: package/gmrender-resurrect/ +F: package/squeezelite/ + +N: Ian Haylock +F: package/python-rpi-gpio/ + +N: Ignacy Gawędzki +F: package/angularjs/ + +N: Ilias Apalodimas +F: package/keepalived/ + +N: Ilya Averyanov +F: package/exempi/ + +N: Ismael Luceno +F: package/axel/ + +N: Jagan Teki +F: board/amarula/ +F: board/asus/ +F: board/bananapi/ +F: board/engicam/ +F: board/friendlyarm/nanopi-a64/ +F: board/friendlyarm/nanopi-neo2/ +F: board/olimex/a33_olinuxino/ +F: board/olimex/a64-olinuxino/ +F: board/orangepi/orangepi-lite2/ +F: board/orangepi/orangepi-one-plus +F: board/orangepi/orangepi-pc2/ +F: board/orangepi/orangepi-prime/ +F: board/orangepi/orangepi-win/ +F: board/orangepi/orangepi-zero-plus2/ +F: board/pine64/ +F: configs/amarula_a64_relic_defconfig +F: configs/amarula_vyasa_rk3288_defconfig +F: configs/asus_tinker_rk3288_defconfig +F: configs/bananapi_m1_defconfig +F: configs/bananapi_m64_defconfig +F: configs/engicam_imx6qdl_icore_defconfig +F: configs/engicam_imx6qdl_icore_qt5_defconfig +F: configs/engicam_imx6qdl_icore_rqs_defconfig +F: configs/engicam_imx6ul_geam_defconfig +F: configs/engicam_imx6ul_isiot_defconfig +F: configs/friendlyarm_nanopi_a64_defconfig +F: configs/friendlyarm_nanopi_neo2_defconfig +F: configs/olimex_a33_olinuxino_defconfig +F: configs/olimex_a64_olinuxino_defconfig +F: configs/orangepi_lite2_defconfig +F: configs/orangepi_one_plus_defconfig +F: configs/orangepi_pc2_defconfig +F: configs/orangepi_prime_defconfig +F: configs/orangepi_win_defconfig +F: configs/orangepi_zero_plus2_defconfig +F: configs/pine64_defconfig +F: configs/pine64_sopine_defconfig + +N: James Hilliard +F: package/gensio/ +F: package/lua-std-debug/ +F: package/lua-std-normalize/ +F: package/pipewire/ +F: package/python-aioconsole/ +F: package/python-aiodns/ +F: package/python-aiohttp/ +F: package/python-aiohttp-cors/ +F: package/python-aiohttp-debugtoolbar/ +F: package/python-aiohttp-jinja2/ +F: package/python-aiohttp-mako/ +F: package/python-aiohttp-remotes/ +F: package/python-aiohttp-security/ +F: package/python-aiohttp-session/ +F: package/python-aiohttp-sse/ +F: package/python-aiologstash/ +F: package/python-aiomonitor/ +F: package/python-aiojobs/ +F: package/python-aiorwlock/ +F: package/python-aiosignal/ +F: package/python-aiozipkin/ +F: package/python-async-lru/ +F: package/python-async-timeout/ +F: package/python-brotli/ +F: package/python-cchardet/ +F: package/python-flatbuffers/ +F: package/python-frozenlist/ +F: package/python-janus/ +F: package/python-logstash/ +F: package/python-multidict/ +F: package/python-pycares/ +F: package/python-sockjs/ +F: package/python-terminaltables/ +F: package/python-yarl/ + +N: James Knight +F: package/atkmm/ +F: package/cairomm/ +F: package/google-material-design-icons/ +F: package/glibmm/ +F: package/gtkmm3/ +F: package/libpqxx/ +F: package/pangomm/ +F: package/rpm/ +F: package/yad/ + +N: Jan Heylen +F: package/opentracing-cpp/ + +N: Jan Kraval +F: board/orangepi/orangepi-lite +F: configs/orangepi_lite_defconfig + +N: Jan Kundrát +F: configs/solidrun_clearfog_defconfig +F: board/solidrun/clearfog/ +F: package/libnetconf2/ +F: package/libyang/ +F: package/sysrepo/ + +N: Jan Pedersen +F: package/zip/ + +N: Jan Viktorin +F: package/python-pexpect/ +F: package/python-ptyprocess/ +F: package/zynq-boot-bin/ + +N: Jared Bents +F: package/davici/ +F: package/python-filelock/ +F: package/python-pysftp/ + +N: Jarkko Sakkinen +F: package/quota/ + +N: Jason Pruitt +F: package/librtlsdr/ + +N: Jean Burgat +F: package/openfpgaloader/ + +N: Jens Kleintje +F: package/gcnano-binaries/ + +N: Jens Rosenboom +F: package/sl/ + +N: Jens Zettelmeyer +F: package/batctl/ + +N: Jeremy Rosen +F: package/fxload/ + +N: Jérôme Oufella +F: package/libdri2/ +F: package/qt-webkit-kiosk/ + +N: Jérôme Pouiller +F: package/apitrace/ +F: package/freescale-imx/gpu-amd-bin-mx51/ +F: package/freescale-imx/libz160/ +F: package/lxc/ +F: package/strongswan/ +F: package/wmctrl/ +F: package/x11r7/xdriver_xf86-video-imx/ +F: package/x11r7/xdriver_xf86-video-imx-viv/ + +N: Jianhui Zhao +F: package/libuwsc/ +F: package/rtty/ + +N: Joao Pinto +F: board/synopsys/vdk/ +F: configs/snps_aarch64_vdk_defconfig + +N: Joel Carlson +F: package/c-capnproto/ +F: package/capnproto/ +F: package/cmocka/ +F: package/flatcc/ +F: package/libcorrect/ + +N: Joel Stanley +F: package/pdbg/ +F: board/qemu/ppc64le-pseries/ +F: configs/qemu_ppc64le_pseries_defconfig +F: board/qemu/ppc-mac99/ +F: configs/qemu_ppc_mac99_defconfig + +N: Johan Derycke +F: package/python-libconfig/ + +N: Johan Oudinet +F: package/ejabberd/ +F: package/erlang-base64url/ +F: package/erlang-eimp/ +F: package/erlang-goldrush/ +F: package/erlang-idna/ +F: package/erlang-jiffy/ +F: package/erlang-jose/ +F: package/erlang-lager/ +F: package/erlang-p1-acme/ +F: package/erlang-p1-cache-tab/ +F: package/erlang-p1-mqtree/ +F: package/erlang-p1-oauth2/ +F: package/erlang-p1-pkix/ +F: package/erlang-p1-sip/ +F: package/erlang-p1-stringprep/ +F: package/erlang-p1-stun/ +F: package/erlang-p1-tls/ +F: package/erlang-p1-utils/ +F: package/erlang-p1-xml/ +F: package/erlang-p1-xmpp/ +F: package/erlang-p1-yaml/ +F: package/erlang-p1-yconf/ +F: package/erlang-p1-zlib/ +F: package/nginx-dav-ext/ +F: package/vuejs/ + +N: John Stile +F: package/dhcpcd/ + +N: John Faith +F: package/python-inflection/ +F: package/sdbusplus/ + +N: Jonathan Ben Avraham +F: arch/Config.in.xtensa +F: package/autofs/ +F: package/dawgdic/ +F: package/libphidget/ +F: package/phidgetwebservice/ +F: package/rapidxml/ +F: package/sphinxbase/ + +N: Joris Offouga +F: package/python-colorlog/ +F: package/python-simplelogging/ + +N: Jörg Krause +F: board/lemaker/bananapro/ +F: configs/bananapro_defconfig +F: package/augeas/ +F: package/bluez-alsa/ +F: package/caps/ +F: package/freescale-imx/imx-alsa-plugins/ +F: package/libopusenc/ +F: package/libupnpp/ +F: package/luv/ +F: package/luvi/ +F: package/mpd/ +F: package/shairport-sync/ +F: package/swupdate/ +F: package/upmpdcli/ +F: package/wavemon/ + +N: Joris Lijssens +F: package/emlog/ +F: package/libcoap/ +F: package/libnet/ +F: package/libuio/ +F: package/netsniff-ng/ +F: package/rabbitmq-c/ + +N: Joseph Kogut +F: package/at-spi2-atk/ +F: package/at-spi2-core/ +F: package/clang/ +F: package/gconf/ +F: package/libnss/ +F: package/lld/ +F: package/llvm/ +F: package/python-cython/ +F: package/python-raven/ +F: package/python-schedule/ +F: package/python-sentry-sdk/ +F: package/python-websockets/ +F: package/python-xlib/ + +N: Joshua Henderson +F: package/qt5/qt5wayland/ + +N: Jugurtha BELKALEM +F: package/python-cycler/ +F: package/python-matplotlib/ + +N: Juha Rantanen +F: package/acsccid/ + +N: Julian Scheel +F: package/bitstream/ +F: package/cbootimage/ +F: package/cryptopp/ +F: package/dvblast/ +F: package/tegrarcm/ + +N: Julien Boibessot +F: board/armadeus/ +F: configs/armadeus* +F: package/abootimg/ +F: package/gpm/ +F: package/lbreakout2/ +F: package/libcddb/ +F: package/libmodbus/ +F: package/ltris/ +F: package/opentyrian/ +F: package/python-pygame/ + +N: Julien Corjon +F: package/qt5/ + +N: Julien Grossholtz +F: board/technologic/ts7680/ +F: configs/ts7680_defconfig +F: package/paho-mqtt-c + +N: Julien Olivain +F: board/technexion/imx8mmpico/ +F: board/technexion/imx8mpico/ +F: configs/imx8mmpico_defconfig +F: configs/imx8mpico_defconfig +F: package/fluid-soundfont/ +F: package/fluidsynth/ +F: package/glslsandbox-player/ +F: package/ptm2human/ + +N: Julien Viard de Galbert +F: package/dieharder/ +F: package/easy-rsa/ + +N: Justin Maggard +F: package/dtach/ + +N: Karoly Kasza +F: package/irqbalance/ +F: package/openvmtools/ + +N: Kelvin Cheung +F: package/cpuload/ +F: package/bwm-ng/ +F: package/ramsmp/ + +N: Kieran Bingham +F: package/libcamera/ + +N: Koen Martens +F: package/capnproto/ +F: package/linuxconsoletools/ + +N: Kurt Van Dijck +F: package/bcusdk/ +F: package/libpthsem/ +F: package/nilfs-utils/ + +N: Laurent Cans +F: package/aircrack-ng/ + +N: Laurent Charpentier +F: package/open-lldp/ + +N: Lee Jones +F: boot/afboot-stm32/ + +N: Leon Anavi +F: board/olimex/a10_olinuxino +F: configs/olimex_a10_olinuxino_lime_defconfig + +N: Lionel Flandrin +F: package/python-babel/ +F: package/python-daemonize/ +F: package/python-flask/ +F: package/python-flask-babel/ +F: package/python-gunicorn/ + +N: Lionel Orry +F: package/mongrel2/ + +N: Lothar Felten +F: board/bananapi/bananapi-m2-ultra/ +F: board/beaglebone/ +F: configs/bananapi_m2_ultra_defconfig +F: configs/beaglebone_defconfig +F: configs/beaglebone_qt5_defconfig +F: package/ti-sgx-demos/ +F: package/ti-sgx-libgbm/ +F: package/ti-sgx-km/ +F: package/ti-sgx-um/ + +N: Louis Aussedat +F: board/friendlyarm/nanopi-neo-plus2/ +F: configs/friendlyarm_nanopi_neo_plus2_defconfig +F: package/mfoc +F: package/libpam-nfc +F: package/python-dnspython/ +F: package/python-future/ +F: package/python-huepy/ +F: package/python-tqdm/ + +N: Louis-Paul Cordier +F: package/intel-gmmlib/ +F: package/intel-mediadriver/ +F: package/intel-mediasdk/ + +N: Luca Ceresoli +F: board/olimex/a20_olinuxino/ +F: board/zynq/ +F: board/zynqmp/ +F: configs/olimex_a20_olinuxino_* +F: configs/zynq_microzed_defconfig +F: configs/zynq_zed_defconfig +F: configs/zynq_zc706_defconfig +F: configs/zynqmp_zcu106_defconfig +F: package/agentpp/ +F: package/exim/ +F: package/libpjsip/ +F: package/qpid-proton/ +F: package/rtl8188eu/ +F: package/snmppp/ +F: package/stm32flash/ +F: package/unzip/ +F: support/legal-info/ + +N: Lucas De Marchi +F: package/fswebcam/ + +N: Ludovic Desroches +F: board/atmel/ +F: configs/at91* +F: configs/atmel_* +F: package/fb-test-app/ +F: package/python-json-schema-validator/ +F: package/python-keyring/ +F: package/python-simplejson/ +F: package/python-versiontools/ +F: package/wilc1000-firmware/ + +N: Maeva Manuel +F: board/freescale/imx8qmmek/ +F: configs/freescale_imx8qmmek_defconfig +F: package/freescale-imx/imx-seco/ + +N: Mahyar Koshkouei +F: package/ffmpeg/ +F: package/mpv/ +F: package/rpi-firmware/ +F: package/rpi-userland/ + +N: Mamatha Inamdar +F: package/nvme/ + +N: Manuel Vögele +F: package/python-pyqt5/ +F: package/python-requests-toolbelt/ + +N: Marcin Bis +F: package/bluez5_utils/ +F: package/cc-tool/ +F: package/ecryptfs-utils/ + +N: Marcin Niestroj +F: board/grinn/ +F: configs/grinn_* +F: package/argparse/ +F: package/dt-utils/ +F: package/easydbus/ +F: package/lua-flu/ +F: package/lua-stdlib/ +F: package/luaossl/ +F: package/murata-cyw-fw/ +F: package/netdata/ +F: package/rs485conf/ +F: package/turbolua/ +F: support/testing/tests/package/test_netdata.py + +N: Marcus Folkesson +F: package/libostree/ +F: package/libselinux/ +F: package/libsemanage/ +F: package/libsepol/ +F: package/selinux-python/ +F: utils/config +F: utils/diffconfig + +N: Marek Belisko +F: package/libatasmart/ +F: package/polkit/ +F: package/sg3_utils/ +F: package/udisks/ + +N: Mario Lang +F: package/brltty/ +F: package/lynx/ + +N: Mario Rugiero +F: package/ratpoison/ + +N: Mark Corbin +F: arch/arch.mk.riscv +F: arch/Config.in.riscv +F: board/qemu/riscv32-virt/ +F: board/qemu/riscv64-virt/ +F: configs/qemu_riscv32_virt_defconfig +F: configs/qemu_riscv64_virt_defconfig + +N: Martin Bark +F: board/raspberrypi/ +F: configs/raspberrypi3_defconfig +F: package/ca-certificates/ +F: package/connman/ +F: package/nodejs/ +F: package/rpi-bt-firmware/ +F: package/rpi-firmware/ +F: package/rpi-wifi-firmware/ +F: package/tzdata/ +F: package/zic/ + +N: Martin Hicks +F: package/cryptsetup/ + +N: Martin Kepplinger +F: package/tslib/ +F: package/x11r7/xdriver_xf86-input-tslib/ +F: package/x11vnc/ + +N: Masahiro Yamada +F: board/arm/foundation-v8/ +F: configs/arm_foundationv8_defconfig + +N: Mathieu Audat +F: board/technologic/ts4900/ +F: configs/ts4900_defconfig +F: package/ts4900-fpga/ + +N: Matt Weber +F: board/freescale/p* +F: board/freescale/t* +F: board/qemu/ppc64-e5500/ +F: configs/freescale_p* +F: configs/freescale_t* +F: configs/qemu_ppc64_e5500_defconfig +F: package/argp-standalone/ +F: package/aufs/ +F: package/aufs-util/ +F: package/bc/ +F: package/bridge-utils/ +F: package/checkpolicy/ +F: package/checksec/ +F: package/cgroupfs-mount/ +F: package/crda/ +F: package/cunit/ +F: package/dacapo/ +F: package/dnsmasq/ +F: package/dosfstools/ +F: package/eigen/ +F: package/ethtool/ +F: package/flashbench/ +F: package/fmc/ +F: package/fmlib/ +F: package/git/ +F: package/gnutls/ +F: package/hostapd/ +F: package/i2c-tools/ +F: package/ifplugd/ +F: package/igmpproxy/ +F: package/iperf/ +F: package/iperf3/ +F: package/iputils/ +F: package/iw/ +F: package/jitterentropy-library/ +F: package/kvm-unit-tests/ +F: package/kvmtool/ +F: package/libcsv/ +F: package/libcurl/ +F: package/libeastl/ +F: package/libfcgi/ +F: package/libopenssl/ +F: package/libselinux/ +F: package/libsemanage/ +F: package/libsepol/ +F: package/libssh2/ +F: package/libqmi/ +F: package/lighttpd/ +F: package/logrotate/ +F: package/makedevs/ +F: package/memtester/ +F: package/mii-diag/ +F: package/mrouted/ +F: package/mtd/ +F: package/mtools/ +F: package/nginx-upload/ +F: package/omniorb/ +F: package/openresolv/ +F: package/paxtest/ +F: package/picocom/ +F: package/policycoreutils/ +F: package/proftpd/ +F: package/protobuf-c/ +F: package/protobuf/ +F: package/python-bunch/ +F: package/python-colorama/ +F: package/python-flask-cors/ +F: package/python-iptables/ +F: package/python-ipy/ +F: package/python-posix-ipc/ +F: package/python-pycairo/ +F: package/python-pypcap/ +F: package/python-pyrex/ +F: package/python-tinyrpc/ +F: package/python-txdbus/ +F: package/raptor/ +F: package/rcw/ +F: package/rng-tools/ +F: package/rsyslog/ +F: package/setools/ +F: package/smcroute/ +F: package/tclap/ +F: package/tini/ +F: package/uboot-tools/ +F: package/unionfs/ +F: package/valijson/ +F: package/wpa_supplicant/ +F: package/wireless_tools/ +F: package/xen/ +F: support/testing/tests/package/br2-external/openjdk/ +F: support/testing/tests/package/test_openjdk.py +F: support/testing/tests/package/test_opkg/ +F: support/testing/tests/package/test_opkg.py + +N: Mauro Condarelli +F: package/mc/ +F: package/python-autobahn/ +F: package/python-cbor/ +F: package/python-characteristic/ +F: package/python-click/ +F: package/python-crossbar/ +F: package/python-lmdb/ +F: package/python-mistune/ +F: package/python-netaddr/ +F: package/python-pyasn-modules/ +F: package/python-pygments/ +F: package/python-pynacl/ +F: package/python-pytrie/ +F: package/python-service-identity/ +F: package/python-setproctitle/ +F: package/python-shutilwhich/ +F: package/python-treq/ +F: package/python-txaio/ +F: package/python-ujson/ +F: package/python-wsaccel/ + +N: Max Filippov +F: arch/Config.in.xtensa + +N: Maxime Hadjinlian +F: package/babeld/ +F: package/dante/ +F: package/faifa/ +F: package/initscripts/ +F: package/intel-microcode/ +F: package/iucode-tool/ +F: package/jasper/ +F: package/kodi/ +F: package/libass/ +F: package/libbluray/ +F: package/libcdio/ +F: package/libcofi/ +F: package/libenca/ +F: package/libmodplug/ +F: package/libnfs/ +F: package/libplist/ +F: package/libshairplay/ +F: package/linux-zigbee/ +F: package/netcat-openbsd/ +F: package/open-plc-utils/ +F: package/rpi-firmware/ +F: package/rpi-userland/ +F: package/rtmpdump/ +F: package/skeleton/ +F: package/systemd/ +F: package/systemd-bootchart/ +F: package/tinyalsa/ +F: package/tinyxml/ + +N: Maxime Ripard +F: package/kmsxx/ + +N: Michael Durrant +F: board/arcturus/ +F: configs/arcturus_ucp1020_defconfig +F: configs/arcturus_ucls1012a_defconfig + +N: Michael Fischer +F: package/gnuplot/ +F: package/sdl2/ + +N: Michael Rommel +F: package/knock/ +F: package/python-crc16/ +F: package/python-pyzmq/ + +N: Michael Trimarchi +F: package/python-spidev/ + +N: Michael Vetter +F: package/jasper/ +F: package/libstrophe/ + +N: Michael Walle +F: package/libavl/ + +N: Michał Łyszczek +F: board/altera/socrates_cyclone5/ +F: board/pine64/rock64 +F: configs/rock64_defconfig +F: configs/socrates_cyclone5_defconfig +F: package/netifrc/ +F: package/openrc/ +F: package/skeleton-init-openrc/ + +N: Michel Stempin +F: board/licheepi/ +F: configs/licheepi_zero_defconfig + +N: Mike Harmony +F: board/sinovoip/m2-plus/ +F: configs/bananapi_m2_plus_defconfig + +N: Mikhail Boiko +F: package/libfribidi/ + +N: Min Xu +F: package/shadowsocks-libev/ + +N: Mirza Krak +F: package/mender/ +F: package/mender-artifact/ + +N: Murat Demirten +F: package/jpeg-turbo/ +F: package/libgeotiff/ + +N: Mylène Josserand +F: package/rtl8723bu/ + +N: Nathaniel Roach +F: package/bandwidthd/ +F: package/libgudev/ + +N: Naumann Andreas +F: package/evemu/ +F: package/libevdev/ +F: package/pkg-qmake.mk + +N: Nicola Di Lieto +F: package/uacme/ + +N: Nicholas Sielicki +F: board/intel/galileo/ +F: configs/galileo_defconfig + +N: Nicolas Cavallari +F: package/libgit2/ + +N: Nicolas Serafini +F: package/exiv2/ +F: package/nvidia-tegra23/nvidia-tegra23-binaries/ +F: package/nvidia-tegra23/nvidia-tegra23-codecs/ +F: package/ofono/ + +N: Nikolay Dimitrov +F: board/embest/riotboard/ +F: configs/riotboard_defconfig + +N: Nimai Mahajan +F: package/libucl/ + +N: Noé Rubinstein +F: package/tpm-tools/ +F: package/trousers/ + +N: Norbert Lange +F: package/tcf-agent/ + +N: Nylon Chen +F: arch/Config.in.nds32 +F: board/andes +F: configs/andes_ae3xx_defconfig +F: toolchain/toolchain-external/toolchain-external-andes-nds32/ + +N: Olaf Rempel +F: package/ctorrent/ + +N: Oleksandr Zhadan +F: board/arcturus/ +F: configs/arcturus_ucp1020_defconfig +F: configs/arcturus_ucls1012a_defconfig + +N: Oli Vogt +F: package/python-django/ +F: package/python-flup/ + +N: Olivier Matz +F: package/python-pyelftools/ + +N: Olivier Schonken +F: package/cups/ +F: package/cups-filters/ +F: package/ijs/ +F: package/poppler/ +F: package/qpdf/ +F: package/openjpeg/ + +N: Olivier Singla +F: package/shellinabox/ + +N: Paresh Chaudhary +F: package/checksec/ + +N: Parnell Springmeyer +F: package/scrypt/ + +N: Pascal de Bruijn +F: package/libargon2/ +F: package/linux-tools/S10hyperv +F: package/linux-tools/hyperv*.service +F: package/linux-tools/linux-tool-hv.mk.in + +N: Pascal Huerst +F: package/google-breakpad/ + +N: Patrick Gerber +F: package/yavta/ + +N: Patrick Havelange +F: support/testing/tests/package/test_lxc.py +F: support/testing/tests/package/test_lxc/ + +N: Paul Cercueil +F: package/libiio/ +F: package/lightning/ +F: package/umtprd/ + +N: Pedro Aguilar +F: package/libunistring/ + +N: Peter Korsgaard +F: board/beagleboneai/ +F: board/minnowboard/ +F: board/librecomputer/lafrite/ +F: board/nexbox/a95x/ +F: board/openblocks/a6/ +F: board/orangepi/ +F: board/pandaboard/ +F: board/roseapplepi/ +F: boot/shim/ +F: configs/beagleboneai_defconfig +F: configs/lafrite_defconfig +F: configs/minnowboard_max-graphical_defconfig +F: configs/minnowboard_max_defconfig +F: configs/nexbox_a95x_defconfig +F: configs/openblocks_a6_defconfig +F: configs/orangepi_pc_defconfig +F: configs/orangepi_r1_defconfig +F: configs/pandaboard_defconfig +F: configs/roseapplepi_defconfig +F: configs/sheevaplug_defconfig +F: package/bats-core/ +F: package/docker-compose/ +F: package/dump1090/ +F: package/fatcat/ +F: package/flickcurl/ +F: package/fscryptctl/ +F: package/ifmetric/ +F: package/jo/ +F: package/jose/ +F: package/libfastjson/ +F: package/luksmeta/ +F: package/lzop/ +F: package/memtool/ +F: package/mosquitto/ +F: package/python-alsaaudio/ +F: package/python-backports-ssl-match-hostname/ +F: package/python-cached-property/ +F: package/python-docker/ +F: package/python-dockerpty/ +F: package/python-docker-pycreds/ +F: package/python-enum/ +F: package/python-enum34/ +F: package/python-functools32/ +F: package/python-ipaddr/ +F: package/python-pam/ +F: package/python-psutil/ +F: package/python-request-id/ +F: package/python-semver/ +F: package/python-texttable/ +F: package/python-validators/ +F: package/python-webob/ +F: package/python-websocket-client/ +F: package/sedutil/ +F: package/tpm2-totp/ +F: package/triggerhappy/ +F: package/wireguard-linux-compat/ +F: package/wireguard-tools/ +F: support/testing/tests/package/test_docker_compose.py + +N: Peter Seiderer +F: board/raspberrypi/ +F: configs/raspberrypi*_defconfig +F: package/assimp/ +F: package/bcm2835/ +F: package/ddrescue/ +F: package/dejavu/ +F: package/dillo/ +F: package/edid-decode/ +F: package/ell/ +F: package/ghostscript-fonts/ +F: package/gstreamer1/gst1-interpipe/ +F: package/gstreamer1/gst1-validate/ +F: package/gstreamer1/gstreamer1-editing-services/ +F: package/iwd/ +F: package/libevdev/ +F: package/log4cplus/ +F: package/postgresql/ +F: package/qt5/ +F: package/quotatool/ +F: package/racehound/ +F: package/wiringpi/ + +N: Peter Thompson +F: package/sdl2_gfx/ +F: package/sdl2_image/ +F: package/sdl2_ttf/ + +N: Petr Kulhavy +F: package/linuxptp/ + +N: Petr Vorel +F: package/ima-evm-utils/ +F: package/iproute2/ +F: package/iputils/ +F: package/libtirpc/ +F: package/linux-backports/ +F: package/ltp-testsuite/ +F: package/nfs-utils/ +F: support/kconfig/ + +N: Phil Eichinger +F: package/libqrencode/ +F: package/psplash/ +F: package/sispmctl/ +F: package/zsh/ + +N: Philipp Richter +F: package/libtorrent-rasterbar/ + +N: Philippe Proulx +F: package/lttng-babeltrace/ +F: package/lttng-libust/ +F: package/lttng-modules/ +F: package/lttng-tools/ +F: package/python-ipython/ +F: package/liburcu/ + +N: Pierre Crokaert +F: board/hardkernel/odroidxu4/ +F: configs/odroidxu4_defconfig + +N: Pierre Ducroquet +F: package/kf5/ + +N: Pierre Floury +F: package/trace-cmd/ + +N: Pierre-Jean Texier +F: package/fping/ +F: package/genimage/ +F: package/haveged/ +F: package/ipset/ +F: package/libarchive/ +F: package/libevent/ +F: package/libubootenv/ +F: package/libxml2/ +F: package/mongoose/ +F: package/mxml/ +F: package/numactl/ +F: package/python-periphery/ +F: package/raspi-gpio/ +F: package/sbc/ +F: package/stunnel/ +F: package/tree/ + +N: Pieter De Gendt +F: package/libvips/ + +N: Pieterjan Camerlynck +F: package/libdvbpsi/ +F: package/mraa/ +F: package/synergy/ + +N: Rafal Susz +F: board/avnet/s6lx9_microboard/ +F: configs/s6lx9_microboard_defconfig + +N: Rahul Bedarkar +F: package/cxxtest/ +F: package/gflags/ +F: package/glog/ +F: package/gssdp/ +F: package/gupnp/ +F: package/gupnp-av/ +F: package/let-me-create/ +F: package/nanomsg/ + +N: Rahul Jain +F: package/uhttpd/ +F: package/ustream-ssl/ + +N: Refik Tuzakli +F: package/freescale-imx/ +F: package/paho-mqtt-cpp/ + +N: Raphaël Mélotte +F: package/jbig2dec/ + +N: Rémi Rérolle +F: package/libfreeimage/ + +N: Renaud Aubin +F: package/libhttpparser/ + +N: Rhys Williams +F: package/lirc-tools/ + +N: Ricardo Martincoski +F: package/atop/ +F: package/thermald/ + +N: Ricardo Martincoski +F: support/testing/infra/ +F: support/testing/run-tests +F: support/testing/tests/core/test_file_capabilities.py +F: support/testing/tests/download/ +F: support/testing/tests/package/*_python*.py +F: support/testing/tests/package/test_atop.py +F: support/testing/tests/package/test_syslog_ng.py +F: support/testing/tests/package/test_tmux.py +F: support/testing/tests/utils/test_check_package.py +F: utils/check-package +F: utils/checkpackagelib/ + +N: Richard Braun +F: package/curlftpfs/ +F: package/tzdata/ + +N: RJ Ascani +F: package/azmq/ + +N: Robert Rose +F: package/grpc/ + +N: Rodrigo Rebello +F: package/chocolate-doom/ +F: package/irssi/ +F: package/vnstat/ + +N: Romain Naour +F: board/qemu/ +F: configs/qemu_* +F: package/alure/ +F: package/aubio/ +F: package/binutils/ +F: package/bullet/ +F: package/clang/ +F: package/clinfo/ +F: package/efl/ +F: package/enet/ +F: package/enlightenment/ +F: package/flare-engine/ +F: package/flare-game/ +F: package/gcc/ +F: package/glibc/ +F: package/irrlicht/ +F: package/liblinear/ +F: package/lensfun/ +F: package/libclc/ +F: package/libgta/ +F: package/libspatialindex/ +F: package/linux-syscall-support/ +F: package/llvm/ +F: package/lugaru/ +F: package/mcelog/ +F: package/mesa3d/ +F: package/minetest/ +F: package/minetest-game/ +F: package/ogre/ +F: package/openpowerlink/ +F: package/physfs/ +F: package/piglit/ +F: package/solarus/ +F: package/stress-ng/ +F: package/supertux/ +F: package/supertuxkart/ +F: package/terminology/ +F: package/tk/ +F: package/upower/ +F: package/waffle/ +F: package/xenomai/ +F: package/zziplib/ +F: support/testing/tests/package/test_glxinfo.py +F: toolchain/ + +N: Roman Gorbenkov +F: package/davfs2/ + +N: Ryan Barnett +F: package/atftp/ +F: package/miraclecast/ +F: package/python-pyasn/ +F: package/python-pycrypto/ +F: package/python-pysnmp/ +F: package/python-pysnmp-mibs/ +F: package/python-tornado/ +F: package/websocketpp/ + +N: Ryan Coe +F: package/inadyn/ +F: package/libite/ +F: package/mariadb/ + +N: Ryan Wilkins +F: package/biosdevname/ + +N: Sam Lancia +F: package/lrzip/ + +N: Samuel Martin +F: package/armadillo/ +F: package/canfestival/ +F: package/clapack/ +F: package/cwiid/ +F: package/flite/ +F: package/nginx/ +F: package/opencv/ +F: package/opencv3/ +F: package/openobex/ +F: package/pkg-cmake.mk +F: package/python-numpy/ +F: package/scrub/ +F: package/urg/ +F: package/ussp-push/ +F: support/misc/toolchainfile.cmake.in + +N: Sam Voss +F: package/ripgrep/ + +N: Santosh Multhalli +F: package/valijson/ + +N: Scott Fan +F: package/libssh/ +F: package/x11r7/xdriver_xf86-video-fbturbo/ + +N: Sébastien Szymanski +F: package/mmc-utils/ +F: package/python-flask-jsonrpc/ +F: package/python-flask-login/ +F: package/qt5/qt5charts/ + +N: Semyon Kolganov +F: package/fmt/ +F: package/libbson/ +F: package/lua-resty-http/ +F: package/mpir/ + +N: Sergey Matyukevich +F: boot/arm-trusted-firmware/ +F: boot/binaries-marvell/ +F: boot/mv-ddr-marvell/ +F: board/linksprite/pcduino +F: board/orangepi/orangepi-zero +F: board/orangepi/orangepi-one +F: board/orangepi/orangepi-pc-plus/ +F: board/solidrun/macchiatobin +F: configs/linksprite_pcduino_defconfig +F: configs/orangepi_one_defconfig +F: configs/orangepi_pc_plus_defconfig +F: configs/orangepi_zero_defconfig +F: configs/solidrun_macchiatobin_mainline_defconfig +F: configs/solidrun_macchiatobin_marvell_defconfig +F: package/armbian-firmware/ +F: package/hostapd/ +F: package/rtl8189fs/ +F: package/wpa_supplicant/ +F: package/xr819-xradio/ + +N: Sergio Prado +F: board/toradex/apalis-imx6/ +F: configs/toradex_apalis_imx6_defconfig +F: package/aoetools/ +F: package/curlpp/ +F: package/daq/ +F: package/libgdiplus/ +F: package/pimd/ +F: package/snort/ +F: package/stella/ +F: package/tio/ +F: package/traceroute/ +F: package/tunctl/ +F: package/ubus/ +F: package/wolfssl/ + +N: Simon Dawson +F: boot/at91bootstrap3/ +F: package/cppzmq/ +F: package/czmq/ +F: package/filemq/ +F: package/googlefontdirectory/ +F: package/jansson/ +F: package/jquery-ui/ +F: package/jquery-ui-themes/ +F: package/json-javascript/ +F: package/lcdapi/ +F: package/libfreefare/ +F: package/libjson/ +F: package/libnfc/ +F: package/libnfc/ +F: package/libserial/ +F: package/libsigsegv/ +F: package/macchanger/ +F: package/minicom/ +F: package/minidlna/ +F: package/msgpack/ +F: package/nanocom/ +F: package/neard/ +F: package/neardal/ +F: package/owl-linux/ +F: package/python-nfc/ +F: package/rapidjson/ +F: package/sconeserver/ +F: package/sound-theme-borealis/ +F: package/sound-theme-freedesktop/ +F: package/vlc/ +F: package/xscreensaver/ +F: package/zmqpp/ +F: package/zyre/ + +N: Spenser Gilliland +F: arch/Config.in.microblaze +F: package/a10disp/ +F: package/glmark2/ +F: package/libvpx/ +F: package/mesa3d-demos/ +F: package/ti-gfx/ + +N: Stefan Sørensen +F: package/cracklib/ +F: package/libpwquality/ +F: package/libscrypt/ + +N: Stephan Hoffmann +F: package/cache-calibrator/ +F: package/gtest/ +F: package/mtdev/ +F: package/mtdev2tuio/ + +N: Stephan Hoffmann +F: package/libhttpserver/ + +N: Steve Calfee +F: package/python-pymysql/ +F: package/python-pyratemp/ + +N: Steve James +F: package/leveldb/ +F: package/libcli/ + +N: Steve Kenton +F: package/dvdauthor/ +F: package/dvdrw-tools/ +F: package/memtest86/ +F: package/mjpegtools/ +F: package/tovid/ +F: package/udftools/ +F: package/xorriso/ + +N: Steven Noonan +F: package/hwloc/ +F: package/powertop/ + +N: Suniel Mahesh +F: board/firefly/ +F: configs/roc_pc_rk3399_defconfig +F: package/arm-gnu-a-toolchain/ + +N: Sven Haardiek +F: package/lcdproc/ +F: package/python-influxdb/ + +N: Sven Oliver Moll +F: package/most/ + +N: Theo Debrouwere +F: board/beagleboardx15/ +F: configs/beagleboardx15_defconfig +F: package/pugixml/ + +N: Thierry Bultel +F: package/mpd-mpc/ + +N: Thijs Vermeir +F: package/ranger/ +F: package/x265/ + +N: Thomas Claveirole +F: package/fcgiwrap/ +F: package/openlayers/ + +N: Thomas Davis +F: package/civetweb/ + +N: Thomas De Schampheleire +F: docs/manual/ +F: package/cereal/ +F: package/chartjs/ +F: package/libtelnet/ +F: package/opkg-utils/ +F: package/perl-convert-asn1/ +F: package/perl-crypt-blowfish/ +F: package/perl-crypt-cbc/ +F: package/perl-crypt-openssl-aes/ +F: package/perl-i18n/ +F: package/perl-locale-maketext-lexicon/ +F: package/perl-lwp-protocol-https/ +F: package/perl-math-prime-util/ +F: package/perl-mime-base64-urlsafe/ +F: package/perl-mojolicious-plugin-authentication/ +F: package/perl-mojolicious-plugin-authorization/ +F: package/perl-mojolicious-plugin-cspheader/ +F: package/perl-mojolicious-plugin-i18n/ +F: package/perl-mojolicious-plugin-securityheader/ +F: package/perl-mozilla-ca/ +F: package/perl-net-snmp/ +F: package/perl-net-ssh2/ +F: package/perl-net-telnet/ +F: package/perl-path-class/ +F: package/pigz/ +F: package/xenomai/ +F: support/scripts/size-stats +F: support/testing/tests/package/test_perl_lwp_protocol_https.py +F: utils/size-stats-compare +F: toolchain/ + +N: Thomas Huth +F: package/ascii-invaders/ + +N: Thomas Petazzoni +F: arch/Config.in.arm +F: board/stmicroelectronics/stm32mp157c-dk2/ +F: boot/boot-wrapper-aarch64/ +F: boot/grub2/ +F: boot/gummiboot/ +F: configs/stm32mp157c_dk2_defconfig +F: package/android-tools/ +F: package/b43-firmware/ +F: package/b43-fwcutter/ +F: package/c-periphery/ +F: package/cdrkit/ +F: package/cifs-utils/ +F: package/cloop/ +F: package/cmake/ +F: package/cramfs/ +F: package/dmidecode/ +F: package/flashrom/ +F: package/gcc/ +F: package/genext2fs/ +F: package/genromfs/ +F: package/getent/ +F: package/gnu-efi/ +F: package/heirloom-mailx/ +F: package/hiawatha/ +F: package/igh-ethercat/ +F: package/intltool/ +F: package/libcap/ +F: package/libffi/ +F: package/libsha1/ +F: package/libtirpc/ +F: package/libxkbcommon/ +F: package/libxml-parser-perl/ +F: package/localedef/ +F: package/log4cxx/ +F: package/monit/ +F: package/mpdecimal/ +F: package/msmtp/ +F: package/musl/ +F: package/musl-fts/ +F: package/ne10/ +F: package/pkg-python.mk +F: package/pkg-autotools.mk +F: package/pkg-generic.mk +F: package/python/ +F: package/python3/ +F: package/python-mad/ +F: package/python-serial/ +F: package/qextserialport/ +F: package/rpcbind/ +F: package/rt-tests/ +F: package/rtc-tools/ +F: package/sam-ba/ +F: package/scons/ +F: package/squashfs/ +F: package/wayland/ +F: package/weston/ +F: toolchain/ + +N: Timo Ketola +F: package/fbgrab/ + +N: Titouan Christophe +F: package/avro-c/ +F: package/mosquitto/ +F: package/python-avro/ +F: package/redis/ +F: package/waf/ +F: support/testing/tests/package/test_crudini.py + +N: Trent Piepho +F: package/libp11/ + +N: Tudor Holton +F: package/openjdk/ + +N: Tzu-Jung Lee +F: package/dropwatch/ +F: package/tstools/ + +N: Vadim Kochan +F: package/brcm-patchram-plus/ +F: package/gettext-tiny/ +F: package/tinyssh/ + +N: Valentin Korenblit +F: package/clang/ +F: package/clinfo/ +F: package/libclc/ +F: package/llvm/ + +N: Vanya Sergeev +F: package/lua-periphery/ + +N: Victor Huesca +F: support/testing/tests/core/test_root_password.py + +N: Vincent Prince +F: package/nss-myhostname/ +F: package/utp_com/ + +N: Vincent Stehlé +F: package/i7z/ +F: package/msr-tools/ +F: package/pixz/ + +N: Vinicius Tinti +F: package/python-thrift/ + +N: Vivien Didelot +F: board/technologic/ts5500/ +F: configs/ts5500_defconfig + +N: Volkov Viacheslav +F: package/v4l2grab/ +F: package/zbar/ + +N: Wade Berrier +F: package/ngrep/ + +N: Waldemar Brodkorb +F: package/uclibc/ +F: package/uclibc-ng-test/ + +N: Will Newton +F: package/enchant/ +F: package/erlang/ +F: package/libmicrohttpd/ +F: package/sysprof/ +F: package/time/ + +N: Will Wagner +F: package/yaffs2utils/ + +N: Wojciech M. Zabolotny +F: package/avrdude/ +F: package/jack2/ +F: package/python-msgpack/ +F: package/python-pyusb/ + +N: Wojciech Niziński +F: package/fwup/ + +N: Yann E. MORIN +F: board/friendlyarm/nanopi-neo/ +F: configs/nanopi_neo_defconfig +F: fs/squashfs/ +F: package/asterisk/ +F: package/cegui/ +F: package/dahdi-linux/ +F: package/dahdi-tools/ +F: package/dtc/ +F: package/dtv-scan-tables/ +F: package/dvb-apps/ +F: package/freerdp/ +F: package/keyutils/ +F: package/libbsd/ +F: package/libedit/ +F: package/libgsm/ +F: package/libiberty/ +F: package/libinput/ +F: package/libiscsi/ +F: package/libpri/ +F: package/libseccomp/ +F: package/libss7/ +F: package/linux-firmware/ +F: package/linux-tools/ +F: package/matchbox* +F: package/mesa3d-headers/ +F: package/nbd/ +F: package/nut/ +F: package/nvidia-driver/ +F: package/omxplayer/ +F: package/python-pyparsing/ +F: package/pkg-download.mk +F: package/pkg-waf.mk +F: package/slirp/ +F: package/snappy/ +F: package/spice/ +F: package/spice-protocol/ +F: package/systemd/ +F: package/systemd-bootchart/ +F: package/tmux/ +F: package/tvheadend/ +F: package/usbredir/ +F: package/vde2/ +F: package/w_scan/ +F: package/wayland/ +F: package/weston/ +F: package/zisofs-tools/ +F: support/download/ + +N: Yegor Yefremov +F: configs/beaglebone_defconfig +F: configs/beaglebone_qt5_defconfig +F: package/acl/ +F: package/attr/ +F: package/boost/ +F: package/bootstrap/ +F: package/cannelloni/ +F: package/can-utils/ +F: package/circus/ +F: package/dhcpcd/ +F: package/feh/ +F: package/giblib/ +F: package/imlib2/ +F: package/jquery-datetimepicker/ +F: package/jquery-sidebar/ +F: package/kmod/ +F: package/libftdi1/ +F: package/libical/ +F: package/libmbim/ +F: package/libndp/ +F: package/libnftnl/ +F: package/libsoc/ +F: package/libsocketcan/ +F: package/libubox/ +F: package/libuci/ +F: package/linux-firmware/ +F: package/linux-serial-test/ +F: package/modem-manager/ +F: package/nftables/ +F: package/nuttcp/ +F: package/parted/ +F: package/phytool/ +F: package/poco/ +F: package/python* +F: package/ser2net/ +F: package/socketcand/ +F: package/swig/ +F: package/qt5/qt5serialbus/ +F: package/sdparm/ +F: package/ti-utils/ +F: package/x11r7/xapp_xconsole/ +F: package/x11r7/xapp_xinput-calibrator/ +F: package/zlog/ +F: support/testing/tests/package/test_libftdi1.py +F: support/testing/tests/package/test_python_can.py +F: utils/scanpypi + +N: Zoltan Gyarmati +F: package/crudini/ +F: package/grantlee/ +F: package/libusb/ +F: package/libusb-compat/ +F: package/proj/ +F: package/python-configobj/ +F: package/python-iniparse/ +F: package/qjson/ +F: package/quazip/ +F: package/shapelib/ +F: package/tinc/ From f85c3df185b28c001307b11498594e80762661f0 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 00:26:05 +0100 Subject: [PATCH 10/12] Update pkg-generic.mk from upstream --- Makefile | 26 +- package/pkg-generic.mk | 551 ++++++++++++++++++++++++++++++++++------- package/pkg-utils.mk | 169 +++++++++++-- 3 files changed, 629 insertions(+), 117 deletions(-) diff --git a/Makefile b/Makefile index eaff91f36f..493fe61492 100644 --- a/Makefile +++ b/Makefile @@ -442,7 +442,6 @@ prepare: $(BUILD_DIR)/buildroot-config/auto.conf world: target-post-image .PHONY: all world toolchain dirs clean distclean source outputmakefile \ - legal-info legal-info-prepare legal-info-clean printvars \ target-finalize target-post-image \ $(PACKAGES) $(TARGETS_ROOTFS) \ $(PACKAGES_DIRCLEAN) $(PACKAGES_SOURCE) $(PACKAGES_LEGAL_INFO) \ @@ -650,31 +649,38 @@ source: $(PACKAGES_SOURCE) $(HOST_SOURCE) external-deps: @$(MAKE1) -Bs DL_MODE=SHOW_EXTERNAL_DEPS $(EXTRAMAKEARGS) source | sort -u +.PHONY: legal-info-clean legal-info-clean: @rm -fr $(LEGAL_INFO_DIR) +.PHONY: legal-info-prepare legal-info-prepare: $(LEGAL_INFO_DIR) - @$(call MESSAGE,"Collecting legal info") - @$(call legal-license-file,buildroot,COPYING,COPYING,HOST) - @$(call legal-manifest,PACKAGE,VERSION,LICENSE,LICENSE FILES,SOURCE ARCHIVE,SOURCE SITE,TARGET) - @$(call legal-manifest,PACKAGE,VERSION,LICENSE,LICENSE FILES,SOURCE ARCHIVE,SOURCE SITE,HOST) - @$(call legal-manifest,buildroot,$(BR2_VERSION_FULL),GPLv2+,COPYING,not saved,not saved,HOST) + @$(call MESSAGE,"Buildroot $(BR2_VERSION_FULL) Collecting legal info") + @$(call legal-license-file,buildroot,buildroot,support/legal-info/buildroot.hash,COPYING,COPYING,HOST) + @$(call legal-manifest,TARGET,PACKAGE,VERSION,LICENSE,LICENSE FILES,SOURCE ARCHIVE,SOURCE SITE,DEPENDENCIES WITH LICENSES) + @$(call legal-manifest,HOST,PACKAGE,VERSION,LICENSE,LICENSE FILES,SOURCE ARCHIVE,SOURCE SITE,DEPENDENCIES WITH LICENSES) + @$(call legal-manifest,HOST,buildroot,$(BR2_VERSION_FULL),GPL-2.0+,COPYING,not saved,not saved) @$(call legal-warning,the Buildroot source code has not been saved) - @$(call legal-warning,the toolchain has not been saved) @cp $(BR2_CONFIG) $(LEGAL_INFO_DIR)/buildroot.config -legal-info: dirs legal-info-clean legal-info-prepare $(PACKAGES_LEGAL_INFO) \ +.PHONY: legal-info +legal-info: legal-info-clean legal-info-prepare $(foreach p,$(PACKAGES),$(p)-all-legal-info) \ $(REDIST_SOURCES_DIR_TARGET) $(REDIST_SOURCES_DIR_HOST) @cat support/legal-info/README.header >>$(LEGAL_REPORT) @if [ -r $(LEGAL_WARNINGS) ]; then \ cat support/legal-info/README.warnings-header \ $(LEGAL_WARNINGS) >>$(LEGAL_REPORT); \ cat $(LEGAL_WARNINGS); fi - @echo "Legal info produced in $(LEGAL_INFO_DIR)" @rm -f $(LEGAL_WARNINGS) + @(cd $(LEGAL_INFO_DIR); \ + find * -type f -exec sha256sum {} + | LC_ALL=C sort -k2 \ + >.legal-info.sha256; \ + mv .legal-info.sha256 legal-info.sha256) + @echo "Legal info produced in $(LEGAL_INFO_DIR)" +.PHONY: show-targets show-targets: - @echo $(HOST_DEPS) $(TARGETS_HOST_DEPS) $(PACKAGES) $(TARGETS_ROOTFS) + @echo $(sort $(PACKAGES)) $(sort $(TARGETS_ROOTFS)) graph-build: $(O)/build/build-time.log @install -d $(O)/graphs diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index 4973efd75a..6ae9a08c29 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -50,20 +50,114 @@ endef # Time steps define step_time printf "%s:%-5.5s:%-20.20s: %s\n" \ - "$$(date +%s)" "$(1)" "$(2)" "$(3)" \ + "$$(date +%s.%N)" "$(1)" "$(2)" "$(3)" \ >>"$(BUILD_DIR)/build-time.log" endef GLOBAL_INSTRUMENTATION_HOOKS += step_time +# Hooks to collect statistics about installed files + +# $(1): package name +# $(2): base directory to search in +# $(3): suffix of file (optional) +define step_pkg_size_before + cd $(2); \ + LC_ALL=C find . \( -type f -o -type l \) -printf '%T@:%i:%#m:%y:%s,%p\n' \ + | LC_ALL=C sort > $($(PKG)_DIR)/.files-list$(3).before +endef + +# $(1): package name +# $(2): base directory to search in +# $(3): suffix of file (optional) +define step_pkg_size_after + cd $(2); \ + LC_ALL=C find . \( -type f -o -type l \) -printf '%T@:%i:%#m:%y:%s,%p\n' \ + | LC_ALL=C sort > $($(PKG)_DIR)/.files-list$(3).after + LC_ALL=C comm -13 \ + $($(PKG)_DIR)/.files-list$(3).before \ + $($(PKG)_DIR)/.files-list$(3).after \ + | sed -r -e 's/^[^,]+/$(1)/' \ + > $($(PKG)_DIR)/.files-list$(3).txt + rm -f $($(PKG)_DIR)/.files-list$(3).before + rm -f $($(PKG)_DIR)/.files-list$(3).after +endef + +define step_pkg_size + $(if $(filter start-install-target,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(TARGET_DIR))) + $(if $(filter start-install-staging,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(STAGING_DIR),-staging)) + $(if $(filter start-install-host,$(1)-$(2)),\ + $(call step_pkg_size_before,$(3),$(HOST_DIR),-host)) + + $(if $(filter end-install-target,$(1)-$(2)),\ + $(call step_pkg_size_after,$(3),$(TARGET_DIR))) + $(if $(filter end-install-staging,$(1)-$(2)),\ + $(call step_pkg_size_after,$(3),$(STAGING_DIR),-staging)) + $(if $(filter end-install-host,$(1)-$(2)),\ + $(call step_pkg_size_after,$(3),$(HOST_DIR),-host)) +endef +GLOBAL_INSTRUMENTATION_HOOKS += step_pkg_size + +# Relies on step_pkg_size, so must be after +define check_bin_arch + $(if $(filter end-install-target,$(1)-$(2)),\ + support/scripts/check-bin-arch -p $(3) \ + -l $($(PKG)_DIR)/.files-list.txt \ + $(foreach i,$($(PKG)_BIN_ARCH_EXCLUDE),-i "$(i)") \ + -r $(TARGET_READELF) \ + -a $(BR2_READELF_ARCH_NAME)) +endef + +GLOBAL_INSTRUMENTATION_HOOKS += check_bin_arch + +# This hook checks that host packages that need libraries that we build +# have a proper DT_RPATH or DT_RUNPATH tag +define check_host_rpath + $(if $(filter install-host,$(2)),\ + $(if $(filter end,$(1)),support/scripts/check-host-rpath $(3) $(HOST_DIR) $(PER_PACKAGE_DIR))) +endef +GLOBAL_INSTRUMENTATION_HOOKS += check_host_rpath + +define step_check_build_dir_one + if [ -d $(2) ]; then \ + printf "%s: installs files in %s\n" $(1) $(2) >&2; \ + exit 1; \ + fi +endef + +define step_check_build_dir + $(if $(filter install-staging,$(2)),\ + $(if $(filter end,$(1)),$(call step_check_build_dir_one,$(3),$(STAGING_DIR)/$(O)))) + $(if $(filter install-target,$(2)),\ + $(if $(filter end,$(1)),$(call step_check_build_dir_one,$(3),$(TARGET_DIR)/$(O)))) +endef +GLOBAL_INSTRUMENTATION_HOOKS += step_check_build_dir + # User-supplied script +ifneq ($(BR2_INSTRUMENTATION_SCRIPTS),) define step_user @$(foreach user_hook, $(BR2_INSTRUMENTATION_SCRIPTS), \ $(EXTRA_ENV) $(user_hook) "$(1)" "$(2)" "$(3)"$(sep)) endef -ifneq ($(BR2_INSTRUMENTATION_SCRIPTS),) GLOBAL_INSTRUMENTATION_HOOKS += step_user endif +####################################### +# Helper functions + +# Make sure .la files only reference the current per-package +# directory. + +# $1: package name (lower case) +# $2: staging directory of the package +ifeq ($(BR2_PER_PACKAGE_DIRECTORIES),y) +define fixup-libtool-files + $(Q)find $(2)/usr/lib* -name "*.la" | xargs --no-run-if-empty \ + $(SED) "s:$(PER_PACKAGE_DIR)/[^/]\+/:$(PER_PACKAGE_DIR)/$(1)/:g" +endef +endif + ################################################################################ # Implicit targets -- produce a stamp file for each step of a package build ################################################################################ @@ -86,40 +180,41 @@ $(BUILD_DIR)/%/.stamp_downloaded: @$(call step_end,download) $(Q)touch $@ +# Retrieve actual source archive, e.g. for prebuilt external toolchains +$(BUILD_DIR)/%/.stamp_actual_downloaded: + @$(call step_start,actual-download) + $(call DOWNLOAD,$($(PKG)_ACTUAL_SOURCE_SITE)/$($(PKG)_ACTUAL_SOURCE_TARBALL),$(PKG)) + $(Q)mkdir -p $(@D) + @$(call step_end,actual-download) + $(Q)touch $@ + # Unpack the archive $(BUILD_DIR)/%/.stamp_extracted: @$(call step_start,extract) @$(call MESSAGE,"Extracting") + $(call prepare-per-package-directory,$($(PKG)_FINAL_EXTRACT_DEPENDENCIES)) $(foreach hook,$($(PKG)_PRE_EXTRACT_HOOKS),$(call $(hook))$(sep)) $(Q)mkdir -p $(@D) $($(PKG)_EXTRACT_CMDS) # some packages have messed up permissions inside $(Q)chmod -R +rw $(@D) $(foreach hook,$($(PKG)_POST_EXTRACT_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,extract) + $(Q)touch $@ # Rsync the source directory if the _OVERRIDE_SRCDIR feature is # used. $(BUILD_DIR)/%/.stamp_rsynced: + @$(call step_start,rsync) @$(call MESSAGE,"Syncing from source dir $(SRCDIR)") - @test -d $(SRCDIR) || (echo "ERROR: $(SRCDIR) does not exist" ; exit 1) + @mkdir -p $(@D) $(foreach hook,$($(PKG)_PRE_RSYNC_HOOKS),$(call $(hook))$(sep)) - rsync -au $(RSYNC_VCS_EXCLUSIONS) $(SRCDIR)/ $(@D) + @test -d $(SRCDIR) || (echo "ERROR: $(SRCDIR) does not exist" ; exit 1) + rsync -au --chmod=u=rwX,go=rX $($(PKG)_OVERRIDE_SRCDIR_RSYNC_EXCLUSIONS) $(RSYNC_VCS_EXCLUSIONS) $(call qstrip,$(SRCDIR))/ $(@D) $(foreach hook,$($(PKG)_POST_RSYNC_HOOKS),$(call $(hook))$(sep)) + @$(call step_end,rsync) $(Q)touch $@ -# Handle the SOURCE_CHECK and SHOW_EXTERNAL_DEPS cases for rsynced -# packages -$(BUILD_DIR)/%/.stamp_rsync_sourced: -ifeq ($(DL_MODE),SOURCE_CHECK) - test -d $(SRCDIR) -else ifeq ($(DL_MODE),SHOW_EXTERNAL_DEPS) - echo "file://$(SRCDIR)" -else - @true # Nothing to do to source a local package -endif - # Patch # # The RAWNAME variable is the lowercased package name, which allows to @@ -127,38 +222,44 @@ endif # prefix of the patches # # For BR2_GLOBAL_PATCH_DIR, only generate if it is defined -$(BUILD_DIR)/%/.stamp_patched: NAMEVER = $(RAWNAME)-$($(PKG)_VERSION) $(BUILD_DIR)/%/.stamp_patched: PATCH_BASE_DIRS = $(PKGDIR) $(BUILD_DIR)/%/.stamp_patched: PATCH_BASE_DIRS += $(addsuffix /$(RAWNAME),$(call qstrip,$(BR2_GLOBAL_PATCH_DIR))) $(BUILD_DIR)/%/.stamp_patched: @$(call step_start,patch) @$(call MESSAGE,"Patching") $(foreach hook,$($(PKG)_PRE_PATCH_HOOKS),$(call $(hook))$(sep)) - $(foreach p,$($(PKG)_PATCH),support/scripts/apply-patches.sh $(@D) $(DL_DIR) $(notdir $(p))$(sep)) + $(foreach p,$($(PKG)_PATCH),$(APPLY_PATCHES) $(@D) $($(PKG)_DL_DIR) $(notdir $(p))$(sep)) $(Q)( \ for D in $(PATCH_BASE_DIRS); do \ if test -d $${D}; then \ if test -d $${D}/$($(PKG)_VERSION); then \ - support/scripts/apply-patches.sh $(@D) $${D}/$($(PKG)_VERSION) \*.patch \*.patch.$(ARCH) || exit 1; \ + $(APPLY_PATCHES) $(@D) $${D}/$($(PKG)_VERSION) \*.patch \*.patch.$(ARCH) || exit 1; \ else \ - support/scripts/apply-patches.sh $(@D) $${D} \*.patch \*.patch.$(ARCH) || exit 1; \ + $(APPLY_PATCHES) $(@D) $${D} \*.patch \*.patch.$(ARCH) || exit 1; \ fi; \ fi; \ done; \ ) $(foreach hook,$($(PKG)_POST_PATCH_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,patch) + $(Q)touch $@ + +# Check that all directories specified in BR2_GLOBAL_PATCH_DIR exist. +$(foreach dir,$(call qstrip,$(BR2_GLOBAL_PATCH_DIR)),\ + $(if $(wildcard $(dir)),,\ + $(error BR2_GLOBAL_PATCH_DIR contains nonexistent directory $(dir)))) # Configure $(BUILD_DIR)/%/.stamp_configured: @$(call step_start,configure) @$(call MESSAGE,"Configuring") + $(call prepare-per-package-directory,$($(PKG)_FINAL_DEPENDENCIES)) + $(call fixup-libtool-files,$(NAME),$(STAGING_DIR)) $(foreach hook,$($(PKG)_PRE_CONFIGURE_HOOKS),$(call $(hook))$(sep)) $($(PKG)_CONFIGURE_CMDS) $(foreach hook,$($(PKG)_POST_CONFIGURE_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,configure) + $(Q)touch $@ # Build $(BUILD_DIR)/%/.stamp_built:: @@ -167,21 +268,42 @@ $(BUILD_DIR)/%/.stamp_built:: $(foreach hook,$($(PKG)_PRE_BUILD_HOOKS),$(call $(hook))$(sep)) +$($(PKG)_BUILD_CMDS) $(foreach hook,$($(PKG)_POST_BUILD_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,build) + $(Q)touch $@ # Install to host dir $(BUILD_DIR)/%/.stamp_host_installed: + @mkdir -p $(HOST_DIR) @$(call step_start,install-host) @$(call MESSAGE,"Installing to host directory") $(foreach hook,$($(PKG)_PRE_INSTALL_HOOKS),$(call $(hook))$(sep)) +$($(PKG)_INSTALL_CMDS) $(foreach hook,$($(PKG)_POST_INSTALL_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,install-host) + $(Q)touch $@ # Install to staging dir +# +# Some packages install libtool .la files alongside any installed +# libraries. These .la files sometimes refer to paths relative to the +# sysroot, which libtool will interpret as absolute paths to host +# libraries instead of the target libraries. Since this is not what we +# want, these paths are fixed by prefixing them with $(STAGING_DIR). +# As we configure with --prefix=/usr, this fix needs to be applied to +# any path that starts with /usr. +# +# To protect against the case that the output or staging directories or +# the pre-installed external toolchain themselves are under /usr, we first +# substitute away any occurrences of these directories with @BASE_DIR@, +# @STAGING_DIR@ and @TOOLCHAIN_EXTERNAL_INSTALL_DIR@ respectively. +# +# Note that STAGING_DIR can be outside BASE_DIR when the user sets +# BR2_HOST_DIR to a custom value. Note that TOOLCHAIN_EXTERNAL_INSTALL_DIR +# can be under @BASE_DIR@ when it's a downloaded toolchain, and can be +# empty when we use an internal toolchain. +# $(BUILD_DIR)/%/.stamp_staging_installed: + @mkdir -p $(STAGING_DIR) @$(call step_start,install-staging) @$(call MESSAGE,"Installing to staging directory") $(foreach hook,$($(PKG)_PRE_INSTALL_STAGING_HOOKS),$(call $(hook))$(sep)) @@ -189,30 +311,52 @@ $(BUILD_DIR)/%/.stamp_staging_installed: $(foreach hook,$($(PKG)_POST_INSTALL_STAGING_HOOKS),$(call $(hook))$(sep)) $(Q)if test -n "$($(PKG)_CONFIG_SCRIPTS)" ; then \ $(call MESSAGE,"Fixing package configuration files") ;\ - $(SED) "s,$(BASE_DIR),@BASE_DIR@,g" \ - -e "s,$(STAGING_DIR),@STAGING_DIR@,g" \ + $(SED) "s,$(HOST_DIR),@HOST_DIR@,g" \ + -e "s,$(BASE_DIR),@BASE_DIR@,g" \ -e "s,^\(exec_\)\?prefix=.*,\1prefix=@STAGING_DIR@/usr,g" \ -e "s,-I/usr/,-I@STAGING_DIR@/usr/,g" \ -e "s,-L/usr/,-L@STAGING_DIR@/usr/,g" \ - -e "s,@STAGING_DIR@,$(STAGING_DIR),g" \ + -e 's,@STAGING_DIR@,$$(dirname $$(readlink -e $$0))/../..,g' \ + -e 's,@HOST_DIR@,$$(dirname $$(readlink -e $$0))/../../../..,g' \ -e "s,@BASE_DIR@,$(BASE_DIR),g" \ $(addprefix $(STAGING_DIR)/usr/bin/,$($(PKG)_CONFIG_SCRIPTS)) ;\ fi - $(Q)touch $@ + @$(call MESSAGE,"Fixing libtool files") + for la in $$(find $(STAGING_DIR)/usr/lib* -name "*.la"); do \ + cp -a "$${la}" "$${la}.fixed" && \ + $(SED) "s:$(BASE_DIR):@BASE_DIR@:g" \ + -e "s:$(STAGING_DIR):@STAGING_DIR@:g" \ + $(if $(TOOLCHAIN_EXTERNAL_INSTALL_DIR),\ + -e "s:$(TOOLCHAIN_EXTERNAL_INSTALL_DIR):@TOOLCHAIN_EXTERNAL_INSTALL_DIR@:g") \ + -e "s:\(['= ]\)/usr:\\1@STAGING_DIR@/usr:g" \ + $(if $(TOOLCHAIN_EXTERNAL_INSTALL_DIR),\ + -e "s:@TOOLCHAIN_EXTERNAL_INSTALL_DIR@:$(TOOLCHAIN_EXTERNAL_INSTALL_DIR):g") \ + -e "s:@STAGING_DIR@:$(STAGING_DIR):g" \ + -e "s:@BASE_DIR@:$(BASE_DIR):g" \ + "$${la}.fixed" && \ + if cmp -s "$${la}" "$${la}.fixed"; then \ + rm -f "$${la}.fixed"; \ + else \ + mv "$${la}.fixed" "$${la}"; \ + fi || exit 1; \ + done @$(call step_end,install-staging) + $(Q)touch $@ # Install to images dir $(BUILD_DIR)/%/.stamp_images_installed: + @mkdir -p $(BINARIES_DIR) @$(call step_start,install-image) - $(foreach hook,$($(PKG)_PRE_INSTALL_IMAGES_HOOKS),$(call $(hook))$(sep)) @$(call MESSAGE,"Installing to images directory") + $(foreach hook,$($(PKG)_PRE_INSTALL_IMAGES_HOOKS),$(call $(hook))$(sep)) +$($(PKG)_INSTALL_IMAGES_CMDS) $(foreach hook,$($(PKG)_POST_INSTALL_IMAGES_HOOKS),$(call $(hook))$(sep)) - $(Q)touch $@ @$(call step_end,install-image) + $(Q)touch $@ # Install to target dir $(BUILD_DIR)/%/.stamp_target_installed: + @mkdir -p $(TARGET_DIR) @$(call step_start,install-target) @$(call MESSAGE,"Installing to target") $(foreach hook,$($(PKG)_PRE_INSTALL_TARGET_HOOKS),$(call $(hook))$(sep)) @@ -221,15 +365,19 @@ $(BUILD_DIR)/%/.stamp_target_installed: $($(PKG)_INSTALL_INIT_SYSTEMD)) $(if $(BR2_INIT_SYSV)$(BR2_INIT_BUSYBOX),\ $($(PKG)_INSTALL_INIT_SYSV)) + $(if $(BR2_INIT_OPENRC), \ + $(or $($(PKG)_INSTALL_INIT_OPENRC), \ + $($(PKG)_INSTALL_INIT_SYSV))) $(foreach hook,$($(PKG)_POST_INSTALL_TARGET_HOOKS),$(call $(hook))$(sep)) $(Q)if test -n "$($(PKG)_CONFIG_SCRIPTS)" ; then \ $(RM) -f $(addprefix $(TARGET_DIR)/usr/bin/,$($(PKG)_CONFIG_SCRIPTS)) ; \ fi - $(Q)touch $@ @$(call step_end,install-target) + $(Q)touch $@ # Remove package sources $(BUILD_DIR)/%/.stamp_dircleaned: + $(if $(BR2_PER_PACKAGE_DIRECTORIES),rm -Rf $(PER_PACKAGE_DIR)/$(NAME)) rm -Rf $(@D) ################################################################################ @@ -251,6 +399,16 @@ be selected at a time. Please fix your configuration) endif endef +define pkg-graph-depends + @$$(INSTALL) -d $$(GRAPHS_DIR) + @cd "$$(CONFIG_DIR)"; \ + $$(TOPDIR)/support/scripts/graph-depends $$(BR2_GRAPH_DEPS_OPTS) \ + -p $(1) $(2) -o $$(GRAPHS_DIR)/$$(@).dot + dot $$(BR2_GRAPH_DOT_OPTS) -T$$(BR_GRAPH_OUT) \ + -o $$(GRAPHS_DIR)/$$(@).$$(BR_GRAPH_OUT) \ + $$(GRAPHS_DIR)/$$(@).dot +endef + ################################################################################ # inner-generic-package -- generates the make targets needed to build a # generic package @@ -288,6 +446,18 @@ endef define inner-generic-package +# When doing a package, we're definitely not doing a rootfs, but we +# may inherit it via the dependency chain, so we reset it. +$(1): ROOTFS= + +# Ensure the package is only declared once, i.e. do not accept that a +# package be re-defined by a br2-external tree +ifneq ($(call strip,$(filter $(1),$(PACKAGES_ALL))),) +$$(error Package '$(1)' defined a second time in '$(pkgdir)'; \ + previous definition was in '$$($(2)_PKGDIR)') +endif +PACKAGES_ALL += $(1) + # Define default values for various package-related variables, if not # already defined. For some variables (version, source, site and # subdir), if they are undefined, we try to see if a variable without @@ -305,6 +475,8 @@ $(2)_PKGDIR = $(pkgdir) # sanitize the package version that is used in paths, directory and file names. # Forward slashes may appear in the package's version when pointing to a # version control system branch or tag, for example remotes/origin/1_10_stable. +# Similar for spaces and colons (:) that may appear in date-based revisions for +# CVS. ifndef $(2)_VERSION ifdef $(3)_DL_VERSION $(2)_DL_VERSION := $$($(3)_DL_VERSION) @@ -316,6 +488,12 @@ else endif $(2)_VERSION := $$(call sanitize,$$($(2)_DL_VERSION)) +$(2)_HASH_FILE = \ + $$(strip \ + $$(if $$(wildcard $$($(2)_PKGDIR)/$$($(2)_VERSION)/$$($(2)_RAWNAME).hash),\ + $$($(2)_PKGDIR)/$$($(2)_VERSION)/$$($(2)_RAWNAME).hash,\ + $$($(2)_PKGDIR)/$$($(2)_RAWNAME).hash)) + ifdef $(3)_OVERRIDE_SRCDIR $(2)_OVERRIDE_SRCDIR ?= $$($(3)_OVERRIDE_SRCDIR) endif @@ -445,21 +623,89 @@ $(2)_REDIST_SOURCES_DIR = $$(REDIST_SOURCES_DIR_$$(call UPPERCASE,$(4)))/$$($(2) # When a target package is a toolchain dependency set this variable to # 'NO' so the 'toolchain' dependency is not added to prevent a circular -# dependency +# dependency. +# Similarly for the skeleton. $(2)_ADD_TOOLCHAIN_DEPENDENCY ?= YES +$(2)_ADD_SKELETON_DEPENDENCY ?= YES + -ifeq ($(4),host) -$(2)_DEPENDENCIES ?= $$(filter-out host-toolchain $(1),\ - $$(patsubst host-host-%,host-%,$$(addprefix host-,$$($(3)_DEPENDENCIES)))) -endif ifeq ($(4),target) +ifeq ($$($(2)_ADD_SKELETON_DEPENDENCY),YES) +# TODO(glebm): backport skeleton infra +# $(2)_DEPENDENCIES += skeleton +endif ifeq ($$($(2)_ADD_TOOLCHAIN_DEPENDENCY),YES) $(2)_DEPENDENCIES += toolchain endif endif +# TODO(glebm): backport skeleton infra +# ifneq ($(1),host-skeleton) +# $(2)_DEPENDENCIES += host-skeleton +# endif + +ifneq ($$(filter cvs git svn,$$($(2)_SITE_METHOD)),) +$(2)_DOWNLOAD_DEPENDENCIES += \ + $(BR2_GZIP_HOST_DEPENDENCY) \ + $(BR2_TAR_HOST_DEPENDENCY) +endif + +ifeq ($$(filter host-tar host-skeleton host-fakedate,$(1)),) +$(2)_EXTRACT_DEPENDENCIES += $$(BR2_TAR_HOST_DEPENDENCY) +endif + +ifeq ($$(filter host-tar host-skeleton host-xz host-lzip host-fakedate,$(1)),) +$(2)_EXTRACT_DEPENDENCIES += \ + $$(foreach dl,$$($(2)_ALL_DOWNLOADS),\ + $$(call extractor-pkg-dependency,$$(notdir $$(dl)))) +endif + +ifeq ($$(BR2_CCACHE),y) +ifeq ($$(filter host-tar host-skeleton host-xz host-lzip host-fakedate host-ccache,$(1)),) +$(2)_DEPENDENCIES += host-ccache +endif +endif + +ifeq ($$(BR2_REPRODUCIBLE),y) +ifeq ($$(filter host-skeleton host-fakedate,$(1)),) +$(2)_DEPENDENCIES += host-fakedate +endif +endif + # Eliminate duplicates in dependencies $(2)_FINAL_DEPENDENCIES = $$(sort $$($(2)_DEPENDENCIES)) +$(2)_FINAL_DOWNLOAD_DEPENDENCIES = $$(sort $$($(2)_DOWNLOAD_DEPENDENCIES)) +$(2)_FINAL_EXTRACT_DEPENDENCIES = $$(sort $$($(2)_EXTRACT_DEPENDENCIES)) +$(2)_FINAL_PATCH_DEPENDENCIES = $$(sort $$($(2)_PATCH_DEPENDENCIES)) +$(2)_FINAL_ALL_DEPENDENCIES = \ + $$(sort \ + $$($(2)_FINAL_DEPENDENCIES) \ + $$($(2)_FINAL_DOWNLOAD_DEPENDENCIES) \ + $$($(2)_FINAL_EXTRACT_DEPENDENCIES) \ + $$($(2)_FINAL_PATCH_DEPENDENCIES)) +$(2)_FINAL_RECURSIVE_DEPENDENCIES = $$(sort \ + $$(if $$(filter undefined,$$(origin $(2)_FINAL_RECURSIVE_DEPENDENCIES__X)), \ + $$(eval $(2)_FINAL_RECURSIVE_DEPENDENCIES__X := \ + $$(foreach p, \ + $$($(2)_FINAL_ALL_DEPENDENCIES), \ + $$(p) \ + $$($$(call UPPERCASE,$$(p))_FINAL_RECURSIVE_DEPENDENCIES) \ + ) \ + ) \ + ) \ + $$($(2)_FINAL_RECURSIVE_DEPENDENCIES__X)) + +$(2)_FINAL_RECURSIVE_RDEPENDENCIES = $$(sort \ + $$(if $$(filter undefined,$$(origin $(2)_FINAL_RECURSIVE_RDEPENDENCIES__X)), \ + $$(eval $(2)_FINAL_RECURSIVE_RDEPENDENCIES__X := \ + $$(foreach p, \ + $$($(2)_RDEPENDENCIES), \ + $$(p) \ + $$($$(call UPPERCASE,$$(p))_FINAL_RECURSIVE_RDEPENDENCIES) \ + ) \ + ) \ + ) \ + $$($(2)_FINAL_RECURSIVE_RDEPENDENCIES__X)) $(2)_INSTALL_STAGING ?= NO $(2)_INSTALL_IMAGES ?= NO @@ -469,14 +715,14 @@ $(2)_INSTALL_TARGET ?= YES $(2)_TARGET_INSTALL_TARGET = $$($(2)_DIR)/.stamp_target_installed $(2)_TARGET_INSTALL_STAGING = $$($(2)_DIR)/.stamp_staging_installed $(2)_TARGET_INSTALL_IMAGES = $$($(2)_DIR)/.stamp_images_installed -$(2)_TARGET_INSTALL_HOST = $$($(2)_DIR)/.stamp_host_installed +$(2)_TARGET_INSTALL_HOST = $$($(2)_DIR)/.stamp_host_installed $(2)_TARGET_BUILD = $$($(2)_DIR)/.stamp_built $(2)_TARGET_CONFIGURE = $$($(2)_DIR)/.stamp_configured -$(2)_TARGET_RSYNC = $$($(2)_DIR)/.stamp_rsynced -$(2)_TARGET_RSYNC_SOURCE = $$($(2)_DIR)/.stamp_rsync_sourced +$(2)_TARGET_RSYNC = $$($(2)_DIR)/.stamp_rsynced $(2)_TARGET_PATCH = $$($(2)_DIR)/.stamp_patched $(2)_TARGET_EXTRACT = $$($(2)_DIR)/.stamp_extracted $(2)_TARGET_SOURCE = $$($(2)_DIR)/.stamp_downloaded +$(2)_TARGET_ACTUAL_SOURCE = $$($(2)_DIR)/.stamp_actual_downloaded $(2)_TARGET_DIRCLEAN = $$($(2)_DIR)/.stamp_dircleaned # default extract command @@ -510,6 +756,14 @@ $(2)_PRE_INSTALL_IMAGES_HOOKS ?= $(2)_POST_INSTALL_IMAGES_HOOKS ?= $(2)_PRE_LEGAL_INFO_HOOKS ?= $(2)_POST_LEGAL_INFO_HOOKS ?= +$(2)_TARGET_FINALIZE_HOOKS ?= +$(2)_ROOTFS_PRE_CMD_HOOKS ?= + +ifeq ($$($(2)_TYPE),target) +ifneq ($$(HOST_$(2)_KCONFIG_VAR),) +$$(error "Package $(1) defines host variant before target variant!") +endif +endif # human-friendly targets and target sequencing $(1): $(1)-install @@ -543,7 +797,7 @@ else $(1)-install-images: endif -$(1)-install-host: $$($(2)_TARGET_INSTALL_HOST) +$(1)-install-host: $$($(2)_TARGET_INSTALL_HOST) $$($(2)_TARGET_INSTALL_HOST): $$($(2)_TARGET_BUILD) $(1)-build: $$($(2)_TARGET_BUILD) @@ -558,10 +812,8 @@ $$($(2)_TARGET_BUILD): $$($(2)_TARGET_CONFIGURE) $(1)-configure: $$($(2)_TARGET_CONFIGURE) $$($(2)_TARGET_CONFIGURE): | $$($(2)_FINAL_DEPENDENCIES) -$$($(2)_TARGET_SOURCE) $$($(2)_TARGET_RSYNC): | dirs prepare -ifeq ($$(filter $(1),$$(DEPENDENCIES_HOST_PREREQ)),) +$$($(2)_TARGET_SOURCE) $$($(2)_TARGET_RSYNC): | prepare $$($(2)_TARGET_SOURCE) $$($(2)_TARGET_RSYNC): | dependencies -endif ifeq ($$($(2)_OVERRIDE_SRCDIR),) # In the normal case (no package override), the sequence of steps is @@ -574,13 +826,33 @@ $$($(2)_TARGET_CONFIGURE): $$($(2)_TARGET_PATCH) $(1)-patch: $$($(2)_TARGET_PATCH) $$($(2)_TARGET_PATCH): $$($(2)_TARGET_EXTRACT) +# Order-only dependency +$$($(2)_TARGET_PATCH): | $$(patsubst %,%-patch,$$($(2)_FINAL_PATCH_DEPENDENCIES)) $(1)-extract: $$($(2)_TARGET_EXTRACT) $$($(2)_TARGET_EXTRACT): $$($(2)_TARGET_SOURCE) +$$($(2)_TARGET_EXTRACT): | $$($(2)_FINAL_EXTRACT_DEPENDENCIES) -$(1)-depends: $$($(2)_FINAL_DEPENDENCIES) +$(1)-depends: $$($(2)_FINAL_ALL_DEPENDENCIES) $(1)-source: $$($(2)_TARGET_SOURCE) +$$($(2)_TARGET_SOURCE): | $$($(2)_FINAL_DOWNLOAD_DEPENDENCIES) + +$(1)-all-source: $(1)-legal-source +$(1)-legal-info: $(1)-legal-source +$(1)-legal-source: $(1)-source + +# Only download the actual source if it differs from the 'main' archive +ifneq ($$($(2)_ACTUAL_SOURCE_TARBALL),) +ifneq ($$($(2)_ACTUAL_SOURCE_TARBALL),$$($(2)_SOURCE)) +$(1)-legal-source: $$($(2)_TARGET_ACTUAL_SOURCE) +endif # actual sources != sources +endif # actual sources != "" + +$(1)-external-deps: + @for p in $$($(2)_SOURCE) $$($(2)_PATCH) $$($(2)_EXTRA_DOWNLOADS) ; do \ + echo `basename $$$$p` ; \ + done else # In the package override case, the sequence of steps # source, by rsyncing @@ -598,31 +870,67 @@ $(1)-extract: $(1)-rsync $(1)-rsync: $$($(2)_TARGET_RSYNC) -$(1)-source: $$($(2)_TARGET_RSYNC_SOURCE) +$(1)-source: +$(1)-legal-source: + +$(1)-external-deps: + @echo "file://$$($(2)_OVERRIDE_SRCDIR)" endif +$(1)-show-version: + @echo $$($(2)_VERSION) + $(1)-show-depends: - @echo $$($(2)_FINAL_DEPENDENCIES) + @echo $$($(2)_FINAL_ALL_DEPENDENCIES) + +$(1)-show-recursive-depends: + @echo $$($(2)_FINAL_RECURSIVE_DEPENDENCIES) + +$(1)-show-rdepends: + @echo $$($(2)_RDEPENDENCIES) + +$(1)-show-recursive-rdepends: + @echo $$($(2)_FINAL_RECURSIVE_RDEPENDENCIES) + +$(1)-show-build-order: $$(patsubst %,%-show-build-order,$$($(2)_FINAL_ALL_DEPENDENCIES)) + @: + $$(info $(1)) + +$(1)-show-info: + @: + $$(info $$(call clean-json,{ $$(call json-info,$(2)) })) $(1)-graph-depends: graph-depends-requirements - @$$(INSTALL) -d $$(O)/graphs - @cd "$$(CONFIG_DIR)"; \ - $$(TOPDIR)/support/scripts/graph-depends -p $(1) $$(BR2_GRAPH_DEPS_OPTS) \ - |tee $$(O)/graphs/$$(@).dot \ - |dot $$(BR2_GRAPH_DOT_OPTS) -T$$(BR_GRAPH_OUT) -o $$(O)/graphs/$$(@).$$(BR_GRAPH_OUT) + $(call pkg-graph-depends,$(1),--direct) + +$(1)-graph-rdepends: graph-depends-requirements + $(call pkg-graph-depends,$(1),--reverse) + +$(1)-all-source: $(1)-source +$(1)-all-source: $$(foreach p,$$($(2)_FINAL_ALL_DEPENDENCIES),$$(p)-all-source) + +$(1)-all-external-deps: $(1)-external-deps +$(1)-all-external-deps: $$(foreach p,$$($(2)_FINAL_ALL_DEPENDENCIES),$$(p)-all-external-deps) + +$(1)-all-legal-info: $(1)-legal-info +$(1)-all-legal-info: $$(foreach p,$$($(2)_FINAL_ALL_DEPENDENCIES),$$(p)-all-legal-info) $(1)-dirclean: $$($(2)_TARGET_DIRCLEAN) -$(1)-clean-for-rebuild: +$(1)-clean-for-reinstall: ifneq ($$($(2)_OVERRIDE_SRCDIR),) rm -f $$($(2)_TARGET_RSYNC) endif - rm -f $$($(2)_TARGET_BUILD) rm -f $$($(2)_TARGET_INSTALL_STAGING) rm -f $$($(2)_TARGET_INSTALL_TARGET) rm -f $$($(2)_TARGET_INSTALL_IMAGES) rm -f $$($(2)_TARGET_INSTALL_HOST) +$(1)-reinstall: $(1)-clean-for-reinstall $(1) + +$(1)-clean-for-rebuild: $(1)-clean-for-reinstall + rm -f $$($(2)_TARGET_BUILD) + $(1)-rebuild: $(1)-clean-for-rebuild $(1) $(1)-clean-for-reconfigure: $(1)-clean-for-rebuild @@ -635,29 +943,31 @@ $(1)-reconfigure: $(1)-clean-for-reconfigure $(1) $$($(2)_TARGET_INSTALL_TARGET): PKG=$(2) $$($(2)_TARGET_INSTALL_STAGING): PKG=$(2) $$($(2)_TARGET_INSTALL_IMAGES): PKG=$(2) -$$($(2)_TARGET_INSTALL_HOST): PKG=$(2) +$$($(2)_TARGET_INSTALL_HOST): PKG=$(2) $$($(2)_TARGET_BUILD): PKG=$(2) $$($(2)_TARGET_CONFIGURE): PKG=$(2) -$$($(2)_TARGET_RSYNC): SRCDIR=$$($(2)_OVERRIDE_SRCDIR) -$$($(2)_TARGET_RSYNC): PKG=$(2) -$$($(2)_TARGET_RSYNC_SOURCE): SRCDIR=$$($(2)_OVERRIDE_SRCDIR) -$$($(2)_TARGET_RSYNC_SOURCE): PKG=$(2) +$$($(2)_TARGET_CONFIGURE): NAME=$(1) +$$($(2)_TARGET_RSYNC): SRCDIR=$$($(2)_OVERRIDE_SRCDIR) +$$($(2)_TARGET_RSYNC): PKG=$(2) $$($(2)_TARGET_PATCH): PKG=$(2) $$($(2)_TARGET_PATCH): RAWNAME=$$(patsubst host-%,%,$(1)) $$($(2)_TARGET_PATCH): PKGDIR=$(pkgdir) $$($(2)_TARGET_EXTRACT): PKG=$(2) $$($(2)_TARGET_SOURCE): PKG=$(2) $$($(2)_TARGET_SOURCE): PKGDIR=$(pkgdir) +$$($(2)_TARGET_ACTUAL_SOURCE): PKG=$(2) +$$($(2)_TARGET_ACTUAL_SOURCE): PKGDIR=$(pkgdir) $$($(2)_TARGET_DIRCLEAN): PKG=$(2) +$$($(2)_TARGET_DIRCLEAN): NAME=$(1) # Compute the name of the Kconfig option that correspond to the # package being enabled. We handle three cases: the special Linux # kernel case, the bootloaders case, and the normal packages case. ifeq ($(1),linux) $(2)_KCONFIG_VAR = BR2_LINUX_KERNEL -else ifneq ($$(filter boot/%,$(pkgdir)),) +else ifneq ($$(filter boot/% $$(foreach dir,$$(BR2_EXTERNAL_DIRS),$$(dir)/boot/%),$(pkgdir)),) $(2)_KCONFIG_VAR = BR2_TARGET_$(2) -else ifneq ($$(filter toolchain/%,$(pkgdir)),) +else ifneq ($$(filter toolchain/% $$(foreach dir,$$(BR2_EXTERNAL_DIRS),$$(dir)/toolchain/%),$(pkgdir)),) $(2)_KCONFIG_VAR = BR2_$(2) else $(2)_KCONFIG_VAR = BR2_PACKAGE_$(2) @@ -667,33 +977,25 @@ endif ifneq ($$($(2)_LICENSE_FILES),) $(2)_MANIFEST_LICENSE_FILES = $$($(2)_LICENSE_FILES) endif -$(2)_MANIFEST_LICENSE_FILES ?= not saved -# If the package declares _LICENSE_FILES, we need to extract it, -# for overriden, local or normal remote packages alike, whether -# we want to redistribute it or not. -ifneq ($$($(2)_LICENSE_FILES),) +# We need to extract and patch a package to be able to retrieve its +# license files (if any) and the list of patches applied to it (if +# any). $(1)-legal-info: $(1)-patch -endif # We only save the sources of packages we want to redistribute, that are -# non-local, and non-overriden. So only store, in the manifest, the tarball -# name of those packages. +# non-overriden (local or true override). ifeq ($$($(2)_REDISTRIBUTE),YES) -ifneq ($$($(2)_SITE_METHOD),local) -ifneq ($$($(2)_SITE_METHOD),override) +ifeq ($$($(2)_OVERRIDE_SRCDIR),) # Packages that have a tarball need it downloaded beforehand $(1)-legal-info: $(1)-source $$(REDIST_SOURCES_DIR_$$(call UPPERCASE,$(4))) -$(2)_MANIFEST_TARBALL = $$($(2)_SOURCE) -$(2)_MANIFEST_SITE = $$(call qstrip,$$($(2)_SITE)) -endif endif endif -$(2)_MANIFEST_TARBALL ?= not saved -$(2)_MANIFEST_SITE ?= not saved # legal-info: produce legally relevant info. +$(1)-legal-info: PKG=$(2) $(1)-legal-info: + @$$(call MESSAGE,"Collecting legal info") # Packages without a source are assumed to be part of Buildroot, skip them. $$(foreach hook,$$($(2)_PRE_LEGAL_INFO_HOOKS),$$(call $$(hook))$$(sep)) ifneq ($$(call qstrip,$$($(2)_SOURCE)),) @@ -705,10 +1007,9 @@ ifneq ($$(call qstrip,$$($(2)_SOURCE)),) # is that the license still applies to the files distributed as part # of the rootfs, even if the sources are not themselves redistributed. ifeq ($$(call qstrip,$$($(2)_LICENSE_FILES)),) - @$$(call legal-license-nofiles,$$($(2)_RAWNAME),$$(call UPPERCASE,$(4))) - @$$(call legal-warning-pkg,$$($(2)_RAWNAME),cannot save license ($(2)_LICENSE_FILES not defined)) + $(Q)$$(call legal-warning-pkg,$$($(2)_BASENAME_RAW),cannot save license ($(2)_LICENSE_FILES not defined)) else - @$$(foreach F,$$($(2)_LICENSE_FILES),$$(call legal-license-file,$$($(2)_RAWNAME),$$(F),$$($(2)_DIR)/$$(F),$$(call UPPERCASE,$(4)))$$(sep)) + $(Q)$$(foreach F,$$($(2)_LICENSE_FILES),$$(call legal-license-file,$$($(2)_RAWNAME),$$($(2)_BASENAME_RAW),$$($(2)_HASH_FILE),$$(F),$$($(2)_DIR)/$$(F),$$(call UPPERCASE,$(4)))$$(sep)) endif # license files ifeq ($$($(2)_SITE_METHOD),local) @@ -722,13 +1023,23 @@ else # Other packages ifeq ($$($(2)_REDISTRIBUTE),YES) -# Copy the source tarball (just hardlink if possible) - @cp -l $$(DL_DIR)/$$($(2)_SOURCE) $$(REDIST_SOURCES_DIR_$$(call UPPERCASE,$(4))) 2>/dev/null || \ - cp $$(DL_DIR)/$$($(2)_SOURCE) $$(REDIST_SOURCES_DIR_$$(call UPPERCASE,$(4))) +# Save the source tarball and any extra downloads, but not +# patches, as they are handled specially afterwards. + $$(foreach e,$$($(2)_ACTUAL_SOURCE_TARBALL) $$(notdir $$($(2)_EXTRA_DOWNLOADS)),\ + $$(Q)support/scripts/hardlink-or-copy \ + $$($(2)_DL_DIR)/$$(e) \ + $$($(2)_REDIST_SOURCES_DIR)$$(sep)) +# Save patches and generate the series file + $$(Q)while read f; do \ + support/scripts/hardlink-or-copy \ + $$$${f} \ + $$($(2)_REDIST_SOURCES_DIR) || exit 1; \ + printf "%s\n" "$$$${f##*/}" >>$$($(2)_REDIST_SOURCES_DIR)/series || exit 1; \ + done <$$($(2)_DIR)/.applied_patches_list endif # redistribute endif # other packages - @$$(call legal-manifest,$$($(2)_RAWNAME),$$($(2)_VERSION),$$($(2)_LICENSE),$$($(2)_MANIFEST_LICENSE_FILES),$$($(2)_MANIFEST_TARBALL),$$($(2)_MANIFEST_SITE),$$(call UPPERCASE,$(4))) + @$$(call legal-manifest,$$(call UPPERCASE,$(4)),$$($(2)_RAWNAME),$$($(2)_VERSION),$$(subst $$(space)$$(comma),$$(comma),$$($(2)_LICENSE)),$$($(2)_MANIFEST_LICENSE_FILES),$$($(2)_ACTUAL_SOURCE_TARBALL),$$($(2)_ACTUAL_SOURCE_SITE),$$(call legal-deps,$(1))) endif # ifneq ($$(call qstrip,$$($(2)_SOURCE)),) $$(foreach hook,$$($(2)_POST_LEGAL_INFO_HOOKS),$$(call $$(hook))$$(sep)) @@ -743,6 +1054,26 @@ $$(foreach pkg,$$($(2)_PROVIDES),\ $$(eval $$(call virt-provides-single,$$(pkg),$$(call UPPERCASE,$$(pkg)),$(1))$$(sep))) endif +# Register package as a reverse-dependencies of all its dependencies +$$(eval $$(foreach p,$$($(2)_FINAL_ALL_DEPENDENCIES),\ + $$(call UPPERCASE,$$(p))_RDEPENDENCIES += $(1)$$(sep))) + +# Ensure unified variable name conventions between all packages Some +# of the variables are used by more than one infrastructure; so, +# rather than duplicating the checks in each infrastructure, we check +# all variables here in pkg-generic, even though pkg-generic should +# have no knowledge of infra-specific variables. +$(eval $(call check-deprecated-variable,$(2)_MAKE_OPT,$(2)_MAKE_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_INSTALL_OPT,$(2)_INSTALL_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_INSTALL_TARGET_OPT,$(2)_INSTALL_TARGET_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_INSTALL_STAGING_OPT,$(2)_INSTALL_STAGING_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_INSTALL_HOST_OPT,$(2)_INSTALL_HOST_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_AUTORECONF_OPT,$(2)_AUTORECONF_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_CONF_OPT,$(2)_CONF_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_BUILD_OPT,$(2)_BUILD_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_GETTEXTIZE_OPT,$(2)_GETTEXTIZE_OPTS)) +$(eval $(call check-deprecated-variable,$(2)_KCONFIG_OPT,$(2)_KCONFIG_OPTS)) + PACKAGES += $(1) ifneq ($$($(2)_PERMISSIONS),) @@ -754,6 +1085,9 @@ endif ifneq ($$($(2)_USERS),) PACKAGES_USERS += $$($(2)_USERS)$$(sep) endif +TARGET_FINALIZE_HOOKS += $$($(2)_TARGET_FINALIZE_HOOKS) +ROOTFS_PRE_CMD_HOOKS += $$($(2)_ROOTFS_PRE_CMD_HOOKS) +KEEP_PYTHON_PY_FILES += $$($(2)_KEEP_PY_FILES) ifeq ($$($(2)_SITE_METHOD),svn) DL_TOOLS_DEPENDENCIES += svn @@ -769,12 +1103,53 @@ else ifeq ($$($(2)_SITE_METHOD),cvs) DL_TOOLS_DEPENDENCIES += cvs endif # SITE_METHOD -# $(firstword) is used here because the extractor can have arguments, like -# ZCAT="gzip -d -c", and to check for the dependency we only want 'gzip'. -# Do not add xzcat to the list of required dependencies, as it gets built -# automatically if it isn't found. -ifneq ($$(call suitable-extractor,$$($(2)_SOURCE)),$$(XZCAT)) -DL_TOOLS_DEPENDENCIES += $$(firstword $$(call suitable-extractor,$$($(2)_SOURCE))) +DL_TOOLS_DEPENDENCIES += $$(call extractor-system-dependency,$$($(2)_SOURCE)) + +# Ensure all virtual targets are PHONY. Listed alphabetically. +.PHONY: $(1) \ + $(1)-all-external-deps \ + $(1)-all-legal-info \ + $(1)-all-source \ + $(1)-build \ + $(1)-clean-for-rebuild \ + $(1)-clean-for-reconfigure \ + $(1)-clean-for-reinstall \ + $(1)-configure \ + $(1)-depends \ + $(1)-dirclean \ + $(1)-external-deps \ + $(1)-extract \ + $(1)-graph-depends \ + $(1)-graph-rdepends \ + $(1)-install \ + $(1)-install-host \ + $(1)-install-images \ + $(1)-install-staging \ + $(1)-install-target \ + $(1)-legal-info \ + $(1)-legal-source \ + $(1)-patch \ + $(1)-rebuild \ + $(1)-reconfigure \ + $(1)-reinstall \ + $(1)-rsync \ + $(1)-show-depends \ + $(1)-show-info \ + $(1)-show-version \ + $(1)-source + +ifneq ($$($(2)_SOURCE),) +ifeq ($$($(2)_SITE),) +$$(error $(2)_SITE cannot be empty when $(2)_SOURCE is not) +endif +endif + +ifeq ($$(patsubst %/,ERROR,$$($(2)_SITE)),ERROR) +$$(error $(2)_SITE ($$($(2)_SITE)) cannot have a trailing slash) +endif + +ifneq ($$($(2)_HELP_CMDS),) +HELP_PACKAGES += $(2) endif endif # $(2)_KCONFIG_VAR diff --git a/package/pkg-utils.mk b/package/pkg-utils.mk index e854656d05..d324934dba 100644 --- a/package/pkg-utils.mk +++ b/package/pkg-utils.mk @@ -28,15 +28,16 @@ endef # Helper functions to determine the name of a package and its # directory from its makefile directory, using the $(MAKEFILE_LIST) -# variable provided by make. This is used by the *TARGETS macros to +# variable provided by make. This is used by the *-package macros to # automagically find where the package is located. -pkgdir = $(dir $(lastword $(MAKEFILE_LIST))) -pkgname = $(lastword $(subst /, ,$(pkgdir))) +pkgdir = $(dir $(lastword $(MAKEFILE_LIST))) +pkgname = $(lastword $(subst /, ,$(pkgdir))) # Define extractors for different archive suffixes INFLATE.bz2 = $(BZCAT) INFLATE.gz = $(ZCAT) INFLATE.lz = $(LZCAT) +INFLATE.lzma = $(XZCAT) INFLATE.tbz = $(BZCAT) INFLATE.tbz2 = $(BZCAT) INFLATE.tgz = $(ZCAT) @@ -45,10 +46,137 @@ INFLATE.tar = cat # suitable-extractor(filename): returns extractor based on suffix suitable-extractor = $(INFLATE$(suffix $(1))) +EXTRACTOR_PKG_DEPENDENCY.lzma = $(BR2_XZCAT_HOST_DEPENDENCY) +EXTRACTOR_PKG_DEPENDENCY.xz = $(BR2_XZCAT_HOST_DEPENDENCY) +EXTRACTOR_PKG_DEPENDENCY.lz = $(BR2_LZIP_HOST_DEPENDENCY) + +# extractor-pkg-dependency(filename): returns a Buildroot package +# dependency needed to extract file based on suffix +extractor-pkg-dependency = $(EXTRACTOR_PKG_DEPENDENCY$(suffix $(1))) + +# extractor-system-dependency(filename): returns the name of the tool +# needed to extract 'filename', and is meant to be used with +# DL_TOOLS_DEPENDENCIES, in order to check that the necesary tool is +# provided by the system Buildroot runs on. +# +# $(firstword) is used here because the extractor can have arguments, +# like ZCAT="gzip -d -c", and to check for the dependency we only want +# 'gzip'. +extractor-system-dependency = $(if $(EXTRACTOR_PKG_DEPENDENCY$(suffix $(1))),,\ + $(firstword $(INFLATE$(suffix $(1))))) + +# check-deprecated-variable -- throw an error on deprecated variables +# example: +# $(eval $(call check-deprecated-variable,FOO_MAKE_OPT,FOO_MAKE_OPTS)) +define check-deprecated-variable # (deprecated var, new var) +ifneq ($$(origin $(1)),undefined) +$$(error Package error: use $(2) instead of $(1). Please fix your .mk file) +endif +endef + +# $(1): YES or NO +define yesno-to-bool + $(subst NO,false,$(subst YES,true,$(1))) +endef + +# json-info -- return package or filesystem metadata formatted as an entry +# of a JSON dictionnary +# $(1): upper-case package or filesystem name +define json-info + "$($(1)_NAME)": { + "type": "$($(1)_TYPE)", + $(if $(filter rootfs,$($(1)_TYPE)), \ + $(call _json-info-fs,$(1)), \ + $(call _json-info-pkg,$(1)), \ + ) + } +endef + +# _json-info-pkg, _json-info-pkg-details, _json-info-fs: private helpers +# for json-info, above +define _json-info-pkg + $(if $($(1)_IS_VIRTUAL), \ + "virtual": true$(comma), + "virtual": false$(comma) + $(call _json-info-pkg-details,$(1)) \ + ) + "dependencies": [ + $(call make-comma-list,$(sort $($(1)_FINAL_ALL_DEPENDENCIES))) + ], + "reverse_dependencies": [ + $(call make-comma-list,$(sort $($(1)_RDEPENDENCIES))) + ] +endef + +define _json-info-pkg-details + "version": "$($(1)_DL_VERSION)", + "licenses": "$($(1)_LICENSE)", + "dl_dir": "$($(1)_DL_SUBDIR)", + "install_target": $(call yesno-to-bool,$($(1)_INSTALL_TARGET)), + "install_staging": $(call yesno-to-bool,$($(1)_INSTALL_STAGING)), + "install_images": $(call yesno-to-bool,$($(1)_INSTALL_IMAGES)), + "downloads": [ + $(foreach dl,$(sort $($(1)_ALL_DOWNLOADS)), + { + "source": "$(notdir $(dl))", + "uris": [ + $(call make-comma-list, + $(subst \|,|, + $(call DOWNLOAD_URIS,$(dl),$(1)) + ) + ) + ] + }, + ) + ], +endef + +define _json-info-fs + "dependencies": [ + $(call make-comma-list,$(sort $($(1)_DEPENDENCIES))) + ] +endef + +# clean-json -- cleanup pseudo-json into clean json: +# - remove commas before closing ] and } +# - minify with $(strip) +clean-json = $(strip \ + $(subst $(comma)},}, $(subst $(comma)$(space)},$(space)}, \ + $(subst $(comma)],], $(subst $(comma)$(space)],$(space)], \ + $(strip $(1)) \ + )))) \ +) + +ifeq ($(BR2_PER_PACKAGE_DIRECTORIES),y) +# rsync the contents of per-package directories +# $1: space-separated list of packages to rsync from +# $2: 'host' or 'target' +# $3: destination directory +define per-package-rsync + mkdir -p $(3) + $(foreach pkg,$(1),\ + rsync -a --link-dest=$(PER_PACKAGE_DIR)/$(pkg)/$(2)/ \ + $(PER_PACKAGE_DIR)/$(pkg)/$(2)/ \ + $(3)$(sep)) +endef + +# prepares the per-package HOST_DIR and TARGET_DIR of the current +# package, by rsync the host and target directories of the +# dependencies of this package. The list of dependencies is passed as +# argument, so that this function can be used to prepare with +# different set of dependencies (download, extract, configure, etc.) +# +# $1: space-separated list of packages to rsync from +define prepare-per-package-directory + $(call per-package-rsync,$(1),host,$(HOST_DIR)) + $(call per-package-rsync,$(1),target,$(TARGET_DIR)) +endef +endif + # # legal-info helper functions # -LEGAL_INFO_SEPARATOR="::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" +LEGAL_INFO_SEPARATOR = "::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" define legal-warning # text echo "WARNING: $(1)" >>$(LEGAL_WARNINGS) @@ -62,23 +190,26 @@ define legal-warning-nosource # pkg, {local|override} $(call legal-warning-pkg,$(1),sources not saved ($(2) packages not handled)) endef -define legal-manifest # pkg, version, license, license-files, source, url, {HOST|TARGET} - echo '"$(1)","$(2)","$(3)","$(4)","$(5)","$(6)"' >>$(LEGAL_MANIFEST_CSV_$(7)) +define legal-manifest # {HOST|TARGET}, pkg, version, license, license-files, source, url, dependencies + echo '"$(2)","$(3)","$(4)","$(5)","$(6)","$(7)","$(8)"' >>$(LEGAL_MANIFEST_CSV_$(1)) endef -define legal-license-header # pkg, license-file, {HOST|TARGET} - printf "$(LEGAL_INFO_SEPARATOR)\n\t$(1):\ - $(2)\n$(LEGAL_INFO_SEPARATOR)\n\n\n" >>$(LEGAL_LICENSES_TXT_$(3)) +define legal-license-file # pkgname, pkgname-pkgver, pkg-hashfile, filename, file-fullpath, {HOST|TARGET} + mkdir -p $(LICENSE_FILES_DIR_$(6))/$(2)/$(dir $(4)) && \ + { \ + support/download/check-hash $(3) $(5) $(4); \ + case $${?} in (0|3) ;; (*) exit 1;; esac; \ + } && \ + cp $(5) $(LICENSE_FILES_DIR_$(6))/$(2)/$(4) endef -define legal-license-nofiles # pkg, {HOST|TARGET} - $(call legal-license-header,$(1),unknown license file(s),$(2)) -endef +non-virtual-deps = $(foreach p,$(1),$(if $($(call UPPERCASE,$(p))_IS_VIRTUAL),,$(p))) -define legal-license-file # pkg, filename, file-fullpath, {HOST|TARGET} - $(call legal-license-header,$(1),$(2) file,$(4)) && \ - cat $(3) >>$(LEGAL_LICENSES_TXT_$(4)) && \ - echo >>$(LEGAL_LICENSES_TXT_$(4)) && \ - mkdir -p $(LICENSE_FILES_DIR_$(4))/$(1)/$(dir $(2)) && \ - cp $(3) $(LICENSE_FILES_DIR_$(4))/$(1)/$(2) -endef +# Returns the list of recursive dependencies and their licensing terms +# for the package specified in parameter (in lowercase). If that +# package is a target package, remove host packages from the list. +legal-deps = \ + $(foreach p,\ + $(filter-out $(if $(1:host-%=),host-%),\ + $(call non-virtual-deps,\ + $($(call UPPERCASE,$(1))_FINAL_RECURSIVE_DEPENDENCIES))),$(p) [$($(call UPPERCASE,$(p))_LICENSE)]) From dd7dc4963d70896493a8acf002dd868e6a3e2908 Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 00:48:22 +0100 Subject: [PATCH 11/12] Backport NLS variables --- package/Makefile.in | 13 +++++++++---- package/pkg-autotools.mk | 2 +- package/ushare/ushare.mk | 26 ++++++++++++++++---------- toolchain/Config.in | 1 + toolchain/toolchain-common.in | 27 ++++++++++++++++++++++++++- 5 files changed, 53 insertions(+), 16 deletions(-) diff --git a/package/Makefile.in b/package/Makefile.in index 614fdeda35..1f5ab8b0c8 100644 --- a/package/Makefile.in +++ b/package/Makefile.in @@ -353,10 +353,15 @@ TARGET_CONFIGURE_ARGS = \ ################################################################################ -ifeq ($(BR2_ENABLE_LOCALE),y) -DISABLE_NLS := +ifeq ($(BR2_SYSTEM_ENABLE_NLS),y) +NLS_OPTS = --enable-nls +TARGET_NLS_DEPENDENCIES = host-gettext +ifeq ($(BR2_PACKAGE_GETTEXT_PROVIDES_LIBINTL),y) +TARGET_NLS_DEPENDENCIES += gettext +TARGET_NLS_LIBS += -lintl +endif else -DISABLE_NLS :=--disable-nls +NLS_OPTS = --disable-nls endif ifneq ($(BR2_LARGEFILE),y) @@ -397,4 +402,4 @@ include package/pkg-python.mk include package/pkg-virtual.mk include package/pkg-generic.mk include package/pkg-kconfig.mk -include package/pkg-meson.mk \ No newline at end of file +include package/pkg-meson.mk diff --git a/package/pkg-autotools.mk b/package/pkg-autotools.mk index 693ec3af4d..dcb11aa1fb 100644 --- a/package/pkg-autotools.mk +++ b/package/pkg-autotools.mk @@ -140,7 +140,7 @@ define $(2)_CONFIGURE_CMDS --disable-documentation \ --with-xmlto=no \ --with-fop=no \ - $$(DISABLE_NLS) \ + $$(NLS_OPTS) \ $$(DISABLE_LARGEFILE) \ $$(DISABLE_IPV6) \ $$(SHARED_STATIC_LIBS_OPTS) \ diff --git a/package/ushare/ushare.mk b/package/ushare/ushare.mk index 25f880c35f..bb194f6374 100644 --- a/package/ushare/ushare.mk +++ b/package/ushare/ushare.mk @@ -7,30 +7,36 @@ USHARE_VERSION = 1.1a USHARE_SOURCE = ushare-$(USHARE_VERSION).tar.bz2 USHARE_SITE = http://ushare.geexbox.org/releases -USHARE_DEPENDENCIES = host-pkgconf libupnp -USHARE_LICENSE = GPLv2+ +USHARE_DEPENDENCIES = host-pkgconf libupnp $(TARGET_NLS_DEPENDENCIES) +USHARE_LICENSE = GPL-2.0+ USHARE_LICENSE_FILES = COPYING +USHARE_LDFLAGS = $(TARGET_NLS_LIBS) -ifeq ($(BR2_NEEDS_GETTEXT_IF_LOCALE),y) -USHARE_DEPENDENCIES += gettext -USHARE_LDFLAGS += -lintl +USHARE_CONF_OPTS = \ + --prefix=/usr \ + --cross-compile \ + --cross-prefix="$(TARGET_CROSS)" \ + --sysconfdir=/etc \ + --disable-strip + +ifeq ($(BR2_SYSTEM_ENABLE_NLS),) +USHARE_CONF_OPTS += --disable-nls endif define USHARE_CONFIGURE_CMDS (cd $(@D); \ $(TARGET_CONFIGURE_OPTS) \ - ./configure --prefix=/usr $(DISABLE_NLS) --cross-compile \ - --cross-prefix="$(TARGET_CROSS)" --sysconfdir=/etc \ - --disable-strip \ + ./configure \ + $(USHARE_CONF_OPTS) \ ) endef define USHARE_BUILD_CMDS - $(MAKE) LDFLAGS="$(TARGET_LDFLAGS) $(USHARE_LDFLAGS)" -C $(@D) + $(TARGET_MAKE_ENV) $(MAKE) LDFLAGS="$(TARGET_LDFLAGS) $(USHARE_LDFLAGS)" -C $(@D) endef define USHARE_INSTALL_TARGET_CMDS - $(MAKE) -C $(@D) DESTDIR=$(TARGET_DIR) install + $(TARGET_MAKE_ENV) $(MAKE) -C $(@D) DESTDIR=$(TARGET_DIR) install rm -f $(TARGET_DIR)/etc/init.d/ushare endef diff --git a/toolchain/Config.in b/toolchain/Config.in index a851ce4666..7576680459 100644 --- a/toolchain/Config.in +++ b/toolchain/Config.in @@ -11,6 +11,7 @@ config BR2_TOOLCHAIN_USES_GLIBC bool select BR2_USE_WCHAR select BR2_ENABLE_LOCALE + select BR2_TOOLCHAIN_HAS_FULL_GETTEXT select BR2_TOOLCHAIN_HAS_THREADS select BR2_TOOLCHAIN_HAS_THREADS_DEBUG select BR2_TOOLCHAIN_HAS_THREADS_NPTL diff --git a/toolchain/toolchain-common.in b/toolchain/toolchain-common.in index e860d774c6..f54cfc127f 100644 --- a/toolchain/toolchain-common.in +++ b/toolchain/toolchain-common.in @@ -52,7 +52,7 @@ config BR2_ENABLE_LOCALE_PURGE config BR2_ENABLE_LOCALE_WHITELIST string "Locales to keep" - default "C en_US de fr" + default "C en_US" depends on BR2_ENABLE_LOCALE_PURGE help Whitespace seperated list of locales to allow on target. @@ -77,6 +77,25 @@ config BR2_GENERATE_LOCALE specified, UTF-8 is assumed. Examples of locales: en_US, fr_FR.UTF-8. +config BR2_SYSTEM_ENABLE_NLS + bool "Enable Native Language Support (NLS)" + depends on BR2_USE_WCHAR + # - glibc has built-in NLS support, but anyway doesn't + # support static linking + # - musl and uclibc support static linking, but they don't + # have built-in NLS support, which is provided by the + # libintl library from gettext. The fact that it is a + # separate library causes too many problems for static + # linking. + depends on !BR2_STATIC_LIBS + select BR2_PACKAGE_GETTEXT if !BR2_TOOLCHAIN_HAS_FULL_GETTEXT + help + This option will enable Native Language Support, which will + allow software packages to support translations. + +comment "NLS support needs a toolchain w/ wchar, dynamic library" + depends on !BR2_USE_WCHAR || BR2_STATIC_LIBS + config BR2_TOOLCHAIN_GLIBC_GCONV_LIBS_COPY bool "Copy gconv libraries" depends on BR2_TOOLCHAIN_USES_GLIBC @@ -113,6 +132,12 @@ config BR2_NEEDS_GETTEXT_IF_LOCALE bool default y if (BR2_NEEDS_GETTEXT && BR2_ENABLE_LOCALE) +# This boolean is true if the toolchain provides a built-in full +# featured gettext implementation (glibc), and false if only a stub +# gettext implementation is provided (uclibc, musl) +config BR2_TOOLCHAIN_HAS_FULL_GETTEXT + bool + config BR2_USE_MMU bool "Enable MMU support" if BR2_arm || BR2_armeb || BR2_sh || BR2_xtensa default y if !BR2_bfin From 22d87da5373b879dd030904d4b9136116302568a Mon Sep 17 00:00:00 2001 From: Gleb Mazovetskiy Date: Sun, 5 Apr 2020 01:18:52 +0100 Subject: [PATCH 12/12] Backport skeleton infra from upstream Now $(HOST_DIR)/bin and $(HOST_DIR)/usr/bin are the same thing One less difference to take care of when backporting --- Makefile | 33 ------ package/Config.in | 7 ++ package/pkg-generic.mk | 10 +- package/pkg-virtual.mk | 26 +++-- package/skeleton-custom/Config.in | 6 + package/skeleton-custom/skeleton-custom.mk | 61 ++++++++++ package/skeleton-init-common/Config.in | 2 + .../skeleton-init-common.mk | 105 ++++++++++++++++++ package/skeleton-init-none/Config.in | 7 ++ .../skeleton-init-none/skeleton-init-none.mk | 18 +++ package/skeleton-init-openrc/Config.in | 7 ++ .../skeleton-init-openrc.mk | 37 ++++++ package/skeleton-init-openrc/skeleton/dev/log | 1 + .../skeleton/etc/conf.d/.empty | 0 .../skeleton-init-openrc/skeleton/etc/fstab | 4 + .../skeleton/etc/runlevels/default/.empty | 0 .../skeleton/etc/runlevels/default/sysv-rcs | 1 + .../skeleton-init-openrc/skeleton/var/cache | 1 + .../skeleton/var/lib/misc | 1 + .../skeleton-init-openrc/skeleton/var/lock | 1 + package/skeleton-init-openrc/skeleton/var/log | 1 + package/skeleton-init-openrc/skeleton/var/run | 1 + .../skeleton-init-openrc/skeleton/var/spool | 1 + package/skeleton-init-openrc/skeleton/var/tmp | 1 + package/skeleton-init-systemd/Config.in | 7 ++ .../skeleton-init-systemd.mk | 68 ++++++++++++ package/skeleton-init-sysv/Config.in | 7 ++ .../skeleton-init-sysv/skeleton-init-sysv.mk | 33 ++++++ package/skeleton-init-sysv/skeleton/dev/log | 1 + .../skeleton/dev/pts/.empty | 0 .../skeleton/dev/shm/.empty | 0 package/skeleton-init-sysv/skeleton/etc/fstab | 8 ++ package/skeleton-init-sysv/skeleton/var/cache | 1 + .../skeleton-init-sysv/skeleton/var/lib/misc | 1 + package/skeleton-init-sysv/skeleton/var/lock | 1 + package/skeleton-init-sysv/skeleton/var/log | 1 + package/skeleton-init-sysv/skeleton/var/run | 1 + package/skeleton-init-sysv/skeleton/var/spool | 1 + package/skeleton-init-sysv/skeleton/var/tmp | 1 + package/skeleton/Config.in | 9 ++ package/skeleton/skeleton.mk | 27 +++++ 41 files changed, 450 insertions(+), 49 deletions(-) create mode 100644 package/skeleton-custom/Config.in create mode 100644 package/skeleton-custom/skeleton-custom.mk create mode 100644 package/skeleton-init-common/Config.in create mode 100644 package/skeleton-init-common/skeleton-init-common.mk create mode 100644 package/skeleton-init-none/Config.in create mode 100644 package/skeleton-init-none/skeleton-init-none.mk create mode 100644 package/skeleton-init-openrc/Config.in create mode 100644 package/skeleton-init-openrc/skeleton-init-openrc.mk create mode 120000 package/skeleton-init-openrc/skeleton/dev/log create mode 100644 package/skeleton-init-openrc/skeleton/etc/conf.d/.empty create mode 100644 package/skeleton-init-openrc/skeleton/etc/fstab create mode 100644 package/skeleton-init-openrc/skeleton/etc/runlevels/default/.empty create mode 120000 package/skeleton-init-openrc/skeleton/etc/runlevels/default/sysv-rcs create mode 120000 package/skeleton-init-openrc/skeleton/var/cache create mode 120000 package/skeleton-init-openrc/skeleton/var/lib/misc create mode 120000 package/skeleton-init-openrc/skeleton/var/lock create mode 120000 package/skeleton-init-openrc/skeleton/var/log create mode 120000 package/skeleton-init-openrc/skeleton/var/run create mode 120000 package/skeleton-init-openrc/skeleton/var/spool create mode 120000 package/skeleton-init-openrc/skeleton/var/tmp create mode 100644 package/skeleton-init-systemd/Config.in create mode 100644 package/skeleton-init-systemd/skeleton-init-systemd.mk create mode 100644 package/skeleton-init-sysv/Config.in create mode 100644 package/skeleton-init-sysv/skeleton-init-sysv.mk create mode 120000 package/skeleton-init-sysv/skeleton/dev/log create mode 100644 package/skeleton-init-sysv/skeleton/dev/pts/.empty create mode 100644 package/skeleton-init-sysv/skeleton/dev/shm/.empty create mode 100644 package/skeleton-init-sysv/skeleton/etc/fstab create mode 120000 package/skeleton-init-sysv/skeleton/var/cache create mode 120000 package/skeleton-init-sysv/skeleton/var/lib/misc create mode 120000 package/skeleton-init-sysv/skeleton/var/lock create mode 120000 package/skeleton-init-sysv/skeleton/var/log create mode 120000 package/skeleton-init-sysv/skeleton/var/run create mode 120000 package/skeleton-init-sysv/skeleton/var/spool create mode 120000 package/skeleton-init-sysv/skeleton/var/tmp create mode 100644 package/skeleton/Config.in create mode 100644 package/skeleton/skeleton.mk diff --git a/Makefile b/Makefile index 493fe61492..8b90d9d7ea 100644 --- a/Makefile +++ b/Makefile @@ -343,8 +343,6 @@ HOST_DIR := $(call qstrip,$(BR2_HOST_DIR)) # Quotes are needed for spaces and all in the original PATH content. BR_PATH = "$(HOST_DIR)/bin:$(HOST_DIR)/sbin:$(HOST_DIR)/usr/bin:$(HOST_DIR)/usr/sbin:$(PATH)" -TARGET_SKELETON = $(TOPDIR)/system/skeleton - # Location of a file giving a big fat warning that output/target # should not be used as the root filesystem. TARGET_DIR_WARNING_FILE = $(TARGET_DIR)/THIS_IS_NOT_YOUR_ROOT_FILESYSTEM @@ -465,37 +463,6 @@ else LIB_SYMLINK = lib32 endif -$(STAGING_DIR): - @mkdir -p $(STAGING_DIR)/bin - @mkdir -p $(STAGING_DIR)/lib - @ln -snf lib $(STAGING_DIR)/$(LIB_SYMLINK) - @mkdir -p $(STAGING_DIR)/usr/lib - @ln -snf lib $(STAGING_DIR)/usr/$(LIB_SYMLINK) - @mkdir -p $(STAGING_DIR)/usr/include - @mkdir -p $(STAGING_DIR)/usr/bin - @ln -snf $(STAGING_DIR) $(BASE_DIR)/staging - -ifeq ($(BR2_ROOTFS_SKELETON_CUSTOM),y) -TARGET_SKELETON = $(BR2_ROOTFS_SKELETON_CUSTOM_PATH) -endif - -RSYNC_VCS_EXCLUSIONS = \ - --exclude .svn --exclude .git --exclude .hg --exclude .bzr \ - --exclude CVS - -$(BUILD_DIR)/.root: - mkdir -p $(TARGET_DIR) - rsync -a --ignore-times $(RSYNC_VCS_EXCLUSIONS) \ - --chmod=Du+w --exclude .empty --exclude '*~' \ - $(TARGET_SKELETON)/ $(TARGET_DIR)/ - $(INSTALL) -m 0644 support/misc/target-dir-warning.txt $(TARGET_DIR_WARNING_FILE) - @ln -snf lib $(TARGET_DIR)/$(LIB_SYMLINK) - @mkdir -p $(TARGET_DIR)/usr - @ln -snf lib $(TARGET_DIR)/usr/$(LIB_SYMLINK) - touch $@ - -$(TARGET_DIR): $(BUILD_DIR)/.root - ifeq ($(BR2_PACKAGE_GDB),y) BR2_STRIP_EXCLUDE_FILES += libpthread*.so* endif diff --git a/package/Config.in b/package/Config.in index 6587779fc9..59a7895de2 100755 --- a/package/Config.in +++ b/package/Config.in @@ -1,6 +1,13 @@ menu "Target packages" source "package/busybox/Config.in" + source "package/skeleton/Config.in" + source "package/skeleton-custom/Config.in" + source "package/skeleton-init-common/Config.in" + source "package/skeleton-init-none/Config.in" + source "package/skeleton-init-openrc/Config.in" + source "package/skeleton-init-systemd/Config.in" + source "package/skeleton-init-sysv/Config.in" menu "Audio and video applications" source "package/alsa-utils/Config.in" diff --git a/package/pkg-generic.mk b/package/pkg-generic.mk index 6ae9a08c29..20b07a7fa9 100644 --- a/package/pkg-generic.mk +++ b/package/pkg-generic.mk @@ -631,18 +631,16 @@ $(2)_ADD_SKELETON_DEPENDENCY ?= YES ifeq ($(4),target) ifeq ($$($(2)_ADD_SKELETON_DEPENDENCY),YES) -# TODO(glebm): backport skeleton infra -# $(2)_DEPENDENCIES += skeleton +$(2)_DEPENDENCIES += skeleton endif ifeq ($$($(2)_ADD_TOOLCHAIN_DEPENDENCY),YES) $(2)_DEPENDENCIES += toolchain endif endif -# TODO(glebm): backport skeleton infra -# ifneq ($(1),host-skeleton) -# $(2)_DEPENDENCIES += host-skeleton -# endif +ifneq ($(1),host-skeleton) +$(2)_DEPENDENCIES += host-skeleton +endif ifneq ($$(filter cvs git svn,$$($(2)_SITE_METHOD)),) $(2)_DOWNLOAD_DEPENDENCIES += \ diff --git a/package/pkg-virtual.mk b/package/pkg-virtual.mk index 2889ef9bea..05bd63eb18 100644 --- a/package/pkg-virtual.mk +++ b/package/pkg-virtual.mk @@ -41,21 +41,27 @@ $$(error No implementation selected for virtual package $(1). Configuration erro endif endif -# A virtual package does not have any source associated +# explicitly set these so we do not get confused by environment +# variables with the same names. +$(2)_VERSION = $(2)_SOURCE = -# Fake a version string, so it looks nicer in the build log -$(2)_VERSION = virtual - -# This must be repeated from inner-generic-package, otherwise we get an empty -# _DEPENDENCIES -ifeq ($(4),host) -$(2)_DEPENDENCIES ?= $$(filter-out host-toolchain $(1),\ - $$(patsubst host-host-%,host-%,$$(addprefix host-,$$($(3)_DEPENDENCIES)))) -endif +$(2)_IS_VIRTUAL = YES # Add dependency against the provider +# For a host package, there is no corresponding BR2_PACKAGE_PROVIDES_HOST_FOO, +# so we need to compute it from the target variant. +ifeq ($(4),target) +$(2)_DEPENDENCIES += $$(call qstrip,$$(BR2_PACKAGE_PROVIDES_$(2))) +else +ifeq ($$(call qstrip,$$(BR2_PACKAGE_PROVIDES_$(2))),) +# Inherit from target package BR2_PACKAGE_PROVIDES_FOO +$(2)_DEPENDENCIES += host-$$(call qstrip,$$(BR2_PACKAGE_PROVIDES_$(3))) +else +# BR2_PACKAGE_PROVIDES_HOST_ is explicitly defined $(2)_DEPENDENCIES += $$(call qstrip,$$(BR2_PACKAGE_PROVIDES_$(2))) +endif +endif # Call the generic package infrastructure to generate the necessary # make targets diff --git a/package/skeleton-custom/Config.in b/package/skeleton-custom/Config.in new file mode 100644 index 0000000000..601c3b247e --- /dev/null +++ b/package/skeleton-custom/Config.in @@ -0,0 +1,6 @@ +config BR2_PACKAGE_SKELETON_CUSTOM + bool + select BR2_PACKAGE_HAS_SKELETON + +config BR2_PACKAGE_PROVIDES_SKELETON + default "skeleton-custom" if BR2_PACKAGE_SKELETON_CUSTOM diff --git a/package/skeleton-custom/skeleton-custom.mk b/package/skeleton-custom/skeleton-custom.mk new file mode 100644 index 0000000000..b05c834b94 --- /dev/null +++ b/package/skeleton-custom/skeleton-custom.mk @@ -0,0 +1,61 @@ +################################################################################ +# +# skeleton-custom +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_CUSTOM_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_CUSTOM_ADD_SKELETON_DEPENDENCY = NO + +SKELETON_CUSTOM_PROVIDES = skeleton + +SKELETON_CUSTOM_INSTALL_STAGING = YES + +SKELETON_CUSTOM_PATH = $(call qstrip,$(BR2_ROOTFS_SKELETON_CUSTOM_PATH)) + +ifeq ($(BR2_PACKAGE_SKELETON_CUSTOM)$(BR_BUILDING),yy) +ifeq ($(SKELETON_CUSTOM_PATH),) +$(error No path specified for the custom skeleton) +endif +endif + +# For a merged /usr, ensure that /lib, /bin and /sbin and their /usr +# counterparts are appropriately setup as symlinks ones to the others. +ifeq ($(BR2_ROOTFS_MERGED_USR),y) +SKELETON_CUSTOM_NOT_MERGED_USR_DIRS = \ + $(shell support/scripts/check-merged-usr.sh $(SKELETON_CUSTOM_PATH)) +endif # merged /usr + +ifeq ($(BR2_PACKAGE_SKELETON_CUSTOM)$(BR_BUILDING),yy) +ifneq ($(SKELETON_CUSTOM_NOT_MERGED_USR_DIRS),) +$(error The custom skeleton in $(SKELETON_CUSTOM_PATH) is not \ + using a merged /usr for the following directories: \ + $(SKELETON_CUSTOM_NOT_MERGED_USR_DIRS)) +endif +endif + +# The target-dir-warning file and the lib{32,64} symlinks are the only +# things we customise in the custom skeleton. +define SKELETON_CUSTOM_INSTALL_TARGET_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_CUSTOM_PATH),$(TARGET_DIR)) + $(call SYSTEM_USR_SYMLINKS_OR_DIRS,$(TARGET_DIR)) + $(call SYSTEM_LIB_SYMLINK,$(TARGET_DIR)) + $(INSTALL) -m 0644 support/misc/target-dir-warning.txt \ + $(TARGET_DIR_WARNING_FILE) +endef + +# For the staging dir, we don't really care what we install, but we +# need the /lib and /usr/lib appropriately setup. Since we ensure, +# above, that they are correct in the skeleton, we can simply copy the +# skeleton to staging. +define SKELETON_CUSTOM_INSTALL_STAGING_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_CUSTOM_PATH),$(STAGING_DIR)) + $(call SYSTEM_USR_SYMLINKS_OR_DIRS,$(STAGING_DIR)) + $(call SYSTEM_LIB_SYMLINK,$(STAGING_DIR)) +endef + +$(eval $(generic-package)) diff --git a/package/skeleton-init-common/Config.in b/package/skeleton-init-common/Config.in new file mode 100644 index 0000000000..0fd95de822 --- /dev/null +++ b/package/skeleton-init-common/Config.in @@ -0,0 +1,2 @@ +config BR2_PACKAGE_SKELETON_INIT_COMMON + bool diff --git a/package/skeleton-init-common/skeleton-init-common.mk b/package/skeleton-init-common/skeleton-init-common.mk new file mode 100644 index 0000000000..4a67f51c19 --- /dev/null +++ b/package/skeleton-init-common/skeleton-init-common.mk @@ -0,0 +1,105 @@ +################################################################################ +# +# skeleton-init-common +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_INIT_COMMON_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_INIT_COMMON_ADD_SKELETON_DEPENDENCY = NO + +# The skeleton also handles the merged /usr case in the sysroot +SKELETON_INIT_COMMON_INSTALL_STAGING = YES + +SKELETON_INIT_COMMON_PATH = system/skeleton + +define SKELETON_INIT_COMMON_INSTALL_TARGET_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_INIT_COMMON_PATH),$(TARGET_DIR)) + $(call SYSTEM_USR_SYMLINKS_OR_DIRS,$(TARGET_DIR)) + $(call SYSTEM_LIB_SYMLINK,$(TARGET_DIR)) + $(SED) 's,@PATH@,$(BR2_SYSTEM_DEFAULT_PATH),' $(TARGET_DIR)/etc/profile + $(INSTALL) -m 0644 support/misc/target-dir-warning.txt \ + $(TARGET_DIR_WARNING_FILE) +endef + +# We don't care much about what goes in staging, as long as it is +# correctly setup for merged/non-merged /usr. The simplest is to +# fill it in with the content of the skeleton. +define SKELETON_INIT_COMMON_INSTALL_STAGING_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_INIT_COMMON_PATH),$(STAGING_DIR)) + $(call SYSTEM_USR_SYMLINKS_OR_DIRS,$(STAGING_DIR)) + $(call SYSTEM_LIB_SYMLINK,$(STAGING_DIR)) + $(INSTALL) -d -m 0755 $(STAGING_DIR)/usr/include +endef + +SKELETON_INIT_COMMON_HOSTNAME = $(call qstrip,$(BR2_TARGET_GENERIC_HOSTNAME)) +SKELETON_INIT_COMMON_ISSUE = $(call qstrip,$(BR2_TARGET_GENERIC_ISSUE)) +SKELETON_INIT_COMMON_ROOT_PASSWD = $(call qstrip,$(BR2_TARGET_GENERIC_ROOT_PASSWD)) +SKELETON_INIT_COMMON_PASSWD_METHOD = $(call qstrip,$(BR2_TARGET_GENERIC_PASSWD_METHOD)) +SKELETON_INIT_COMMON_BIN_SH = $(call qstrip,$(BR2_SYSTEM_BIN_SH)) + +ifneq ($(SKELETON_INIT_COMMON_HOSTNAME),) +SKELETON_INIT_COMMON_HOSTS_LINE = $(SKELETON_INIT_COMMON_HOSTNAME) +SKELETON_INIT_COMMON_SHORT_HOSTNAME = $(firstword $(subst ., ,$(SKELETON_INIT_COMMON_HOSTNAME))) +ifneq ($(SKELETON_INIT_COMMON_HOSTNAME),$(SKELETON_INIT_COMMON_SHORT_HOSTNAME)) +SKELETON_INIT_COMMON_HOSTS_LINE += $(SKELETON_INIT_COMMON_SHORT_HOSTNAME) +endif +define SKELETON_INIT_COMMON_SET_HOSTNAME + mkdir -p $(TARGET_DIR)/etc + echo "$(SKELETON_INIT_COMMON_HOSTNAME)" > $(TARGET_DIR)/etc/hostname + $(SED) '$$a \127.0.1.1\t$(SKELETON_INIT_COMMON_HOSTS_LINE)' \ + -e '/^127.0.1.1/d' $(TARGET_DIR)/etc/hosts +endef +SKELETON_INIT_COMMON_TARGET_FINALIZE_HOOKS += SKELETON_INIT_COMMON_SET_HOSTNAME +endif + +ifneq ($(SKELETON_INIT_COMMON_ISSUE),) +define SKELETON_INIT_COMMON_SET_ISSUE + mkdir -p $(TARGET_DIR)/etc + echo "$(SKELETON_INIT_COMMON_ISSUE)" > $(TARGET_DIR)/etc/issue +endef +SKELETON_INIT_COMMON_TARGET_FINALIZE_HOOKS += SKELETON_INIT_COMMON_SET_ISSUE +endif + +ifeq ($(BR2_TARGET_ENABLE_ROOT_LOGIN),y) +ifneq ($(filter $$1$$% $$5$$% $$6$$%,$(SKELETON_INIT_COMMON_ROOT_PASSWD)),) +SKELETON_INIT_COMMON_ROOT_PASSWORD = '$(SKELETON_INIT_COMMON_ROOT_PASSWD)' +else ifneq ($(SKELETON_INIT_COMMON_ROOT_PASSWD),) +# This variable will only be evaluated in the finalize stage, so we can +# be sure that host-mkpasswd will have already been built by that time. +SKELETON_INIT_COMMON_ROOT_PASSWORD = "`$(MKPASSWD) -m "$(SKELETON_INIT_COMMON_PASSWD_METHOD)" "$(SKELETON_INIT_COMMON_ROOT_PASSWD)"`" +endif +else # !BR2_TARGET_ENABLE_ROOT_LOGIN +SKELETON_INIT_COMMON_ROOT_PASSWORD = "*" +endif +define SKELETON_INIT_COMMON_SET_ROOT_PASSWD + $(SED) s,^root:[^:]*:,root:$(SKELETON_INIT_COMMON_ROOT_PASSWORD):, $(TARGET_DIR)/etc/shadow +endef +SKELETON_INIT_COMMON_TARGET_FINALIZE_HOOKS += SKELETON_INIT_COMMON_SET_ROOT_PASSWD + +ifeq ($(BR2_SYSTEM_BIN_SH_NONE),y) +define SKELETON_INIT_COMMON_SET_BIN_SH + rm -f $(TARGET_DIR)/bin/sh +endef +else +# Add /bin/sh to /etc/shells otherwise some login tools like dropbear +# can reject the user connection. See man shells. +define SKELETON_INIT_COMMON_ADD_SH_TO_SHELLS + grep -qsE '^/bin/sh$$' $(TARGET_DIR)/etc/shells \ + || echo "/bin/sh" >> $(TARGET_DIR)/etc/shells +endef +SKELETON_INIT_COMMON_TARGET_FINALIZE_HOOKS += SKELETON_INIT_COMMON_ADD_SH_TO_SHELLS +ifneq ($(SKELETON_INIT_COMMON_BIN_SH),) +define SKELETON_INIT_COMMON_SET_BIN_SH + ln -sf $(SKELETON_INIT_COMMON_BIN_SH) $(TARGET_DIR)/bin/sh + $(SED) '/^root:/s,[^/]*$$,$(SKELETON_INIT_COMMON_BIN_SH),' \ + $(TARGET_DIR)/etc/passwd +endef +endif +endif +SKELETON_INIT_COMMON_TARGET_FINALIZE_HOOKS += SKELETON_INIT_COMMON_SET_BIN_SH + +$(eval $(generic-package)) diff --git a/package/skeleton-init-none/Config.in b/package/skeleton-init-none/Config.in new file mode 100644 index 0000000000..449bf66125 --- /dev/null +++ b/package/skeleton-init-none/Config.in @@ -0,0 +1,7 @@ +config BR2_PACKAGE_SKELETON_INIT_NONE + bool + select BR2_PACKAGE_HAS_SKELETON + select BR2_PACKAGE_SKELETON_INIT_COMMON + +config BR2_PACKAGE_PROVIDES_SKELETON + default "skeleton-init-none" if BR2_PACKAGE_SKELETON_INIT_NONE diff --git a/package/skeleton-init-none/skeleton-init-none.mk b/package/skeleton-init-none/skeleton-init-none.mk new file mode 100644 index 0000000000..8181aa0354 --- /dev/null +++ b/package/skeleton-init-none/skeleton-init-none.mk @@ -0,0 +1,18 @@ +################################################################################ +# +# skeleton-init-none +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_INIT_NONE_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_INIT_NONE_ADD_SKELETON_DEPENDENCY = NO + +SKELETON_INIT_NONE_DEPENDENCIES = skeleton-init-common + +SKELETON_INIT_NONE_PROVIDES = skeleton + +$(eval $(generic-package)) diff --git a/package/skeleton-init-openrc/Config.in b/package/skeleton-init-openrc/Config.in new file mode 100644 index 0000000000..8f4dd17ebd --- /dev/null +++ b/package/skeleton-init-openrc/Config.in @@ -0,0 +1,7 @@ +config BR2_PACKAGE_SKELETON_INIT_OPENRC + bool + select BR2_PACKAGE_HAS_SKELETON + select BR2_PACKAGE_SKELETON_INIT_COMMON + +config BR2_PACKAGE_PROVIDES_SKELETON + default "skeleton-init-openrc" if BR2_PACKAGE_SKELETON_INIT_OPENRC diff --git a/package/skeleton-init-openrc/skeleton-init-openrc.mk b/package/skeleton-init-openrc/skeleton-init-openrc.mk new file mode 100644 index 0000000000..b0538ad03f --- /dev/null +++ b/package/skeleton-init-openrc/skeleton-init-openrc.mk @@ -0,0 +1,37 @@ +################################################################################ +# +# skeleton-init-openrc +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_INIT_OPENRC_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_INIT_OPENRC_ADD_SKELETON_DEPENDENCY = NO + +SKELETON_INIT_OPENRC_DEPENDENCIES = skeleton-init-common + +SKELETON_INIT_OPENRC_PROVIDES = skeleton + +ifeq ($(BR2_TARGET_GENERIC_REMOUNT_ROOTFS_RW),y) +# Comment /dev/root entry in fstab. When openrc does not find fstab entry for +# "/", it will try to remount "/" as "rw". +define SKELETON_INIT_OPENRC_ROOT_RO_OR_RW + $(SED) '\:^/dev/root[[:blank:]]:s/^/# /' $(TARGET_DIR)/etc/fstab +endef +else +# Uncomment /dev/root entry in fstab which has "ro" option so openrc notices +# it and doesn't remount root to rw. +define SKELETON_INIT_OPENRC_ROOT_RO_OR_RW + $(SED) '\:^#[[:blank:]]*/dev/root[[:blank:]]:s/^# //' $(TARGET_DIR)/etc/fstab +endef +endif # BR2_TARGET_GENERIC_REMOUNT_ROOTFS_RW + +define SKELETON_INIT_OPENRC_INSTALL_TARGET_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_INIT_OPENRC_PKGDIR)/skeleton,$(TARGET_DIR)) + $(SKELETON_INIT_OPENRC_ROOT_RO_OR_RW) +endef + +$(eval $(generic-package)) diff --git a/package/skeleton-init-openrc/skeleton/dev/log b/package/skeleton-init-openrc/skeleton/dev/log new file mode 120000 index 0000000000..d96b3b1cb8 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/dev/log @@ -0,0 +1 @@ +../tmp/log \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/etc/conf.d/.empty b/package/skeleton-init-openrc/skeleton/etc/conf.d/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package/skeleton-init-openrc/skeleton/etc/fstab b/package/skeleton-init-openrc/skeleton/etc/fstab new file mode 100644 index 0000000000..21cf49ba5a --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/etc/fstab @@ -0,0 +1,4 @@ +# +/dev/root / ext2 ro,noauto 0 0 +tmpfs /tmp tmpfs mode=1777 0 0 +tmpfs /run tmpfs mode=0755,nosuid,nodev 0 0 diff --git a/package/skeleton-init-openrc/skeleton/etc/runlevels/default/.empty b/package/skeleton-init-openrc/skeleton/etc/runlevels/default/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package/skeleton-init-openrc/skeleton/etc/runlevels/default/sysv-rcs b/package/skeleton-init-openrc/skeleton/etc/runlevels/default/sysv-rcs new file mode 120000 index 0000000000..ef5e00823c --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/etc/runlevels/default/sysv-rcs @@ -0,0 +1 @@ +/etc/init.d/sysv-rcs \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/cache b/package/skeleton-init-openrc/skeleton/var/cache new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/cache @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/lib/misc b/package/skeleton-init-openrc/skeleton/var/lib/misc new file mode 120000 index 0000000000..f1fde8c107 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/lib/misc @@ -0,0 +1 @@ +../../tmp \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/lock b/package/skeleton-init-openrc/skeleton/var/lock new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/lock @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/log b/package/skeleton-init-openrc/skeleton/var/log new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/log @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/run b/package/skeleton-init-openrc/skeleton/var/run new file mode 120000 index 0000000000..84ba55b912 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/run @@ -0,0 +1 @@ +../run \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/spool b/package/skeleton-init-openrc/skeleton/var/spool new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/spool @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-openrc/skeleton/var/tmp b/package/skeleton-init-openrc/skeleton/var/tmp new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-openrc/skeleton/var/tmp @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-systemd/Config.in b/package/skeleton-init-systemd/Config.in new file mode 100644 index 0000000000..59b21d155e --- /dev/null +++ b/package/skeleton-init-systemd/Config.in @@ -0,0 +1,7 @@ +config BR2_PACKAGE_SKELETON_INIT_SYSTEMD + bool + select BR2_PACKAGE_HAS_SKELETON + select BR2_PACKAGE_SKELETON_INIT_COMMON + +config BR2_PACKAGE_PROVIDES_SKELETON + default "skeleton-init-systemd" if BR2_PACKAGE_SKELETON_INIT_SYSTEMD diff --git a/package/skeleton-init-systemd/skeleton-init-systemd.mk b/package/skeleton-init-systemd/skeleton-init-systemd.mk new file mode 100644 index 0000000000..5d6b716f7a --- /dev/null +++ b/package/skeleton-init-systemd/skeleton-init-systemd.mk @@ -0,0 +1,68 @@ +################################################################################ +# +# skeleton-init-systemd +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_INIT_SYSTEMD_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_INIT_SYSTEMD_ADD_SKELETON_DEPENDENCY = NO + +SKELETON_INIT_SYSTEMD_DEPENDENCIES = skeleton-init-common + +SKELETON_INIT_SYSTEMD_PROVIDES = skeleton + +ifeq ($(BR2_TARGET_GENERIC_REMOUNT_ROOTFS_RW),y) + +define SKELETON_INIT_SYSTEMD_ROOT_RO_OR_RW + echo "/dev/root / auto rw 0 1" >$(TARGET_DIR)/etc/fstab +endef + +else + +# On a R/O rootfs, /var is a tmpfs filesystem. So, at build time, we +# redirect /var to the "factory settings" location. Just before the +# filesystem gets created, the /var symlink will be replaced with +# a real (but empty) directory, and the "factory files" will be copied +# back there by the tmpfiles.d mechanism. +define SKELETON_INIT_SYSTEMD_ROOT_RO_OR_RW + mkdir -p $(TARGET_DIR)/etc/systemd/tmpfiles.d + echo "/dev/root / auto ro 0 1" >$(TARGET_DIR)/etc/fstab + echo "tmpfs /var tmpfs mode=1777 0 0" >>$(TARGET_DIR)/etc/fstab +endef + +define SKELETON_INIT_SYSTEMD_PRE_ROOTFS_VAR + rm -rf $(TARGET_DIR)/usr/share/factory/var + mv $(TARGET_DIR)/var $(TARGET_DIR)/usr/share/factory/var + mkdir -p $(TARGET_DIR)/var + for i in $(TARGET_DIR)/usr/share/factory/var/* \ + $(TARGET_DIR)/usr/share/factory/var/lib/* \ + $(TARGET_DIR)/usr/share/factory/var/lib/systemd/*; do \ + [ -e "$${i}" ] || continue; \ + j="$${i#$(TARGET_DIR)/usr/share/factory}"; \ + if [ -L "$${i}" ]; then \ + printf "L+! %s - - - - %s\n" \ + "$${j}" "../usr/share/factory/$${j}" \ + || exit 1; \ + else \ + printf "C! %s - - - -\n" "$${j}" \ + || exit 1; \ + fi; \ + done >$(TARGET_DIR)/etc/tmpfiles.d/var-factory.conf +endef +SKELETON_INIT_SYSTEMD_ROOTFS_PRE_CMD_HOOKS += SKELETON_INIT_SYSTEMD_PRE_ROOTFS_VAR + +endif + +define SKELETON_INIT_SYSTEMD_INSTALL_TARGET_CMDS + mkdir -p $(TARGET_DIR)/home + mkdir -p $(TARGET_DIR)/srv + mkdir -p $(TARGET_DIR)/var + ln -s ../run $(TARGET_DIR)/var/run + $(SKELETON_INIT_SYSTEMD_ROOT_RO_OR_RW) +endef + +$(eval $(generic-package)) diff --git a/package/skeleton-init-sysv/Config.in b/package/skeleton-init-sysv/Config.in new file mode 100644 index 0000000000..6f645f9e25 --- /dev/null +++ b/package/skeleton-init-sysv/Config.in @@ -0,0 +1,7 @@ +config BR2_PACKAGE_SKELETON_INIT_SYSV + bool + select BR2_PACKAGE_HAS_SKELETON + select BR2_PACKAGE_SKELETON_INIT_COMMON + +config BR2_PACKAGE_PROVIDES_SKELETON + default "skeleton-init-sysv" if BR2_PACKAGE_SKELETON_INIT_SYSV diff --git a/package/skeleton-init-sysv/skeleton-init-sysv.mk b/package/skeleton-init-sysv/skeleton-init-sysv.mk new file mode 100644 index 0000000000..009dc100ea --- /dev/null +++ b/package/skeleton-init-sysv/skeleton-init-sysv.mk @@ -0,0 +1,33 @@ +################################################################################ +# +# skeleton-init-sysv +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_INIT_SYSV_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_INIT_SYSV_ADD_SKELETON_DEPENDENCY = NO + +SKELETON_INIT_SYSV_DEPENDENCIES = skeleton-init-common + +SKELETON_INIT_SYSV_PROVIDES = skeleton + +define SKELETON_INIT_SYSV_INSTALL_TARGET_CMDS + $(call SYSTEM_RSYNC,$(SKELETON_INIT_SYSV_PKGDIR)/skeleton,$(TARGET_DIR)) +endef + +# enable/disable swapon/off calls depending on availability of the commands +define SKELETON_INIT_SYSV_SWAPON_SWAPOFF_INITTAB + if [ -x $(TARGET_DIR)/sbin/swapon -a -x $(TARGET_DIR)/sbin/swapoff ]; then \ + $(SED) '/^#.*\/sbin\/swap/s/^#\+[[:blank:]]*//' $(TARGET_DIR)/etc/inittab; \ + else \ + $(SED) '/^[^#].*\/sbin\/swap/s/^/#/' $(TARGET_DIR)/etc/inittab; \ + fi +endef + +SKELETON_INIT_SYSV_TARGET_FINALIZE_HOOKS += SKELETON_INIT_SYSV_SWAPON_SWAPOFF_INITTAB + +$(eval $(generic-package)) diff --git a/package/skeleton-init-sysv/skeleton/dev/log b/package/skeleton-init-sysv/skeleton/dev/log new file mode 120000 index 0000000000..d96b3b1cb8 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/dev/log @@ -0,0 +1 @@ +../tmp/log \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/dev/pts/.empty b/package/skeleton-init-sysv/skeleton/dev/pts/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package/skeleton-init-sysv/skeleton/dev/shm/.empty b/package/skeleton-init-sysv/skeleton/dev/shm/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package/skeleton-init-sysv/skeleton/etc/fstab b/package/skeleton-init-sysv/skeleton/etc/fstab new file mode 100644 index 0000000000..169054b74f --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/etc/fstab @@ -0,0 +1,8 @@ +# +/dev/root / ext2 rw,noauto 0 1 +proc /proc proc defaults 0 0 +devpts /dev/pts devpts defaults,gid=5,mode=620,ptmxmode=0666 0 0 +tmpfs /dev/shm tmpfs mode=0777 0 0 +tmpfs /tmp tmpfs mode=1777 0 0 +tmpfs /run tmpfs mode=0755,nosuid,nodev 0 0 +sysfs /sys sysfs defaults 0 0 diff --git a/package/skeleton-init-sysv/skeleton/var/cache b/package/skeleton-init-sysv/skeleton/var/cache new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/cache @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/lib/misc b/package/skeleton-init-sysv/skeleton/var/lib/misc new file mode 120000 index 0000000000..f1fde8c107 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/lib/misc @@ -0,0 +1 @@ +../../tmp \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/lock b/package/skeleton-init-sysv/skeleton/var/lock new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/lock @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/log b/package/skeleton-init-sysv/skeleton/var/log new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/log @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/run b/package/skeleton-init-sysv/skeleton/var/run new file mode 120000 index 0000000000..84ba55b912 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/run @@ -0,0 +1 @@ +../run \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/spool b/package/skeleton-init-sysv/skeleton/var/spool new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/spool @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton-init-sysv/skeleton/var/tmp b/package/skeleton-init-sysv/skeleton/var/tmp new file mode 120000 index 0000000000..1431b0e432 --- /dev/null +++ b/package/skeleton-init-sysv/skeleton/var/tmp @@ -0,0 +1 @@ +../tmp \ No newline at end of file diff --git a/package/skeleton/Config.in b/package/skeleton/Config.in new file mode 100644 index 0000000000..efaa1e135f --- /dev/null +++ b/package/skeleton/Config.in @@ -0,0 +1,9 @@ +config BR2_PACKAGE_SKELETON + bool + default y + +config BR2_PACKAGE_HAS_SKELETON + bool + +config BR2_PACKAGE_PROVIDES_SKELETON + string diff --git a/package/skeleton/skeleton.mk b/package/skeleton/skeleton.mk new file mode 100644 index 0000000000..9d97f02f08 --- /dev/null +++ b/package/skeleton/skeleton.mk @@ -0,0 +1,27 @@ +################################################################################ +# +# skeleton +# +################################################################################ + +# The skeleton can't depend on the toolchain, since all packages depends on the +# skeleton and the toolchain is a target package, as is skeleton. +# Hence, skeleton would depends on the toolchain and the toolchain would depend +# on skeleton. +SKELETON_ADD_TOOLCHAIN_DEPENDENCY = NO +SKELETON_ADD_SKELETON_DEPENDENCY = NO + +# We create a compatibility symlink in case a post-build script still +# uses $(HOST_DIR)/usr +define HOST_SKELETON_INSTALL_CMDS + $(Q)ln -snf . $(HOST_DIR)/usr + $(Q)mkdir -p $(HOST_DIR)/lib + $(Q)mkdir -p $(HOST_DIR)/include + $(Q)case $(HOSTARCH) in \ + (*64) ln -snf lib $(HOST_DIR)/lib64;; \ + (*) ln -snf lib $(HOST_DIR)/lib32;; \ + esac +endef + +$(eval $(virtual-package)) +$(eval $(host-generic-package))