Merge pull request #2318 from deitch/goodbye-packages-txt

kaniini · web-flow · commit 9dfd4f34b795 · 2023-07-06T20:03:16.000Z
replace packages.txt with calculated graph
diff --git a/.github/pull-request-template.md b/.github/pull-request-template.md
@@ -32,7 +32,6 @@ addressed, and remove any items that are not relevant to this PR.
   - [ ] Alternatively, the PR is marked as related to a pre-existing package request bug, such as a dependency
 - [ ] REQUIRED - The package is available under an OSI-approved or FSF-approved license
 - [ ] REQUIRED - The version of the package is still receiving security updates
-- [ ] REQUIRED - The package is added to `packages.txt`
 
 #### For security-related PRs
 <!-- remove if unrelated -->
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -7,6 +7,8 @@ on:
     branches:
       - gh-readonly-queue/main/**
 
+env:
+  WOLFICTL: "docker run -it --rm --entrypoint=wolfictl ghcr.io/wolfi-dev/sdk:latest@sha256:fd8c71214f6455c75ec44ae99eb9f7ffc85f260ccce69d4367eb2e5d568facd9 wolfictl"
 jobs:
   changes:
     name: Determine packages to test building
@@ -28,17 +30,13 @@ jobs:
       - name: Build package list
         id: package-list
         run: |
-          # Ensure there is a newline at the end of packages.txt to avoid `read` from ignoring
-          # the last entry.
-          echo >> $GITHUB_WORKSPACE/packages.txt
-
           printf "packages=" >> $GITHUB_OUTPUT
 
-          while read pkg; do
+          make list | while read pkg; do
             for file in ${{ steps.changes.outputs.all_changed_files }}; do
               [ "${file%.yaml}" = "$pkg" ] && printf "%s " ${file%.yaml} >> $GITHUB_OUTPUT
             done
-          done < $GITHUB_WORKSPACE/packages.txt
+          done
 
           printf "\n" >> $GITHUB_OUTPUT
 
@@ -72,7 +70,7 @@ jobs:
       - name: 'Build Wolfi'
         run: |
           for package in ${{needs.changes.outputs.packages}}; do
-            make MELANGE="melange" MELANGE_EXTRA_OPTS="--create-build-log" REPO="$GITHUB_WORKSPACE/packages" BUILDWORLD=no packages/$package -j1
+            make MELANGE="melange" MELANGE_EXTRA_OPTS="--create-build-log" REPO="$GITHUB_WORKSPACE/packages" BUILDWORLD=no package/$package -j1
           done
 
       - name: Check for file
diff --git a/.github/workflows/wolfictl-lint.yaml b/.github/workflows/wolfictl-lint.yaml
@@ -22,7 +22,7 @@ jobs:
       uses: docker://ghcr.io/wolfi-dev/wolfictl:latest@sha256:5876ee08b32ee3e9ff0d4c83867d168bbefe6b1e6cc5bee1f07442e5493ca6e4
       with:
         entrypoint: wolfictl
-        args: lint
+        args: lint --skip-rule no-makefile-entry-for-package
     - name: Enforce YAML formatting
       id: lint-yaml
       uses: docker://ghcr.io/wolfi-dev/wolfictl:latest@sha256:5876ee08b32ee3e9ff0d4c83867d168bbefe6b1e6cc5bee1f07442e5493ca6e4
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -26,12 +26,6 @@ Wolfi packages are built using melange. If you want to learn how packages are bu
 
 Start by cloning this repository and create a YAML file named `<your-package-name>.yaml` in its root directory. If you have any patches, create a folder with the same name and put them there.
 
-Add a new entry for your package near the bottom of [`packages.txt`](packages.txt) like this:
-
-```
-your-package-name
-```
-
 Once you're done writing the new package configuration file, you can test it by triggering a build with `make packages/<your-package-name> BUILDWORLD=no USE_CACHE=no`.
 
 note: `USE_CACHE=no` disables using a GCP cloud bucket for caching pipeline sources.  This is mainly used for production setups and not needed for local development.
diff --git a/Makefile b/Makefile
@@ -6,6 +6,7 @@ endif
 TARGETDIR = packages/${ARCH}
 
 MELANGE ?= $(shell which melange)
+WOLFICTL ?= $(shell which wolfictl)
 KEY ?= local-melange.rsa
 REPO ?= $(shell pwd)/packages
 CACHE_DIR ?= gs://wolfi-sources/
@@ -32,11 +33,8 @@ MELANGE_OPTS += -r ${WOLFI_PROD}
 endif
 
 # The list of packages to be built. The order matters.
-# At some point, when ready, this should be replaced with `wolfictl text -t name .`
-# non-standard source directories are provided by adding them separated by a comma,
-# e.g.
-# postgres-11,postgres
-PKGLIST ?= $(shell cat packages.txt | grep -v '^\#' )
+# wolfictl determines the list and order
+PKGLIST ?= $(shell $(WOLFICTL) text --dir . --type name)
 
 all: ${KEY} .build-packages
 
@@ -48,31 +46,24 @@ clean:
 
 .PHONY: list list-yaml
 list:
-	$(info $(PKGNAMELIST))
+	$(info $(PKGLIST))
 	@printf ''
 
 list-yaml:
-	$(info $(addsuffix .yaml,$(PKGNAMELIST)))
+	$(info $(addsuffix .yaml,$(PKGLIST)))
 	@printf ''
 
-.packagerules: Makefile .git/HEAD packages.txt
-	@echo "Solving build order, please wait..."
-	@grep -v '^\#' packages.txt | while read pkg; do		\
-		pkgname=`echo $$pkg | cut -d, -f1`;			\
-		pkgdir=`echo $$pkg | cut -d, -f2`;			\
-		[ -z "$$pkgdir" ] && pkgdir=$$pkgname;			\
-		pkgver=`${MELANGE} package-version $${pkgname}.yaml`;	\
-		pkgtarget="${TARGETDIR}/$${pkgver}.apk";		\
-		echo "PKGNAMELIST += $$pkgname";			\
-		echo ".build-packages: $$pkgtarget";			\
-		echo "packages/$$pkgname: $$pkgtarget";			\
-		echo "$$pkgtarget: $${pkgname}.yaml \$${KEY}";		\
-		printf "\t%s\n" "@mkdir -p ./$${pkgdir}/";		\
-		printf "\t%s" "SDE=\$${SOURCE_DATE_EPOCH}; [ -z \"\$$\$$SDE\" ] && SDE=\`git log -1 --pretty=%ct --follow $${pkgname}.yaml\`;"; \
-		printf "\t%s\n\n" "SOURCE_DATE_EPOCH=\$$\$$SDE \$${MELANGE} build $${pkgname}.yaml \$${MELANGE_OPTS} --source-dir ./$${pkgdir}/ --log-policy builtin:stderr,\$${TARGETDIR}/buildlogs/$${pkgver}.log"; \
-	done > .packagerules
+.build-packages: $(addprefix package/,${PKGLIST})
 
--include .packagerules
+package/%:
+	$(eval yamlfile := $*.yaml)
+	$(eval pkgver := $(shell $(MELANGE) package-version $(yamlfile)))
+	$(MAKE) yamlfile=$(yamlfile) pkgname=$* packages/$(ARCH)/$(pkgver).apk
+
+packages/$(ARCH)/%.apk: $(KEY)
+	@mkdir -p ./$(pkgname)/
+	$(eval SOURCE_DATE_EPOCH ?= $(shell git log -1 --pretty=%ct --follow $(yamlfile)))
+	@SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) $(MELANGE) build $(yamlfile) $(MELANGE_OPTS) --source-dir ./$(pkgname)/ --log-policy builtin:stderr,$(TARGETDIR)/buildlogs/$*.log
 
 dev-container:
-	docker run --privileged --rm -it -v "${PWD}:${PWD}" -w "${PWD}" ghcr.io/wolfi-dev/sdk:latest@sha256:3ef78225a85ab45f46faac66603c9da2877489deb643174ba1e42d8cbf0e0644
+	docker run --privileged --rm -it -v "${PWD}:${PWD}" -w "${PWD}" ghcr.io/wolfi-dev/sdk:latest@sha256:3ba6e392eff7f09493c62b8a6bff4b9378ecccc27e5dc4ba0fa9f2a0e95c666f
diff --git a/bazel-5 b/bazel-5
@@ -0,0 +1 @@
+bazel
diff --git a/bazel-6 b/bazel-6
@@ -0,0 +1 @@
+bazel
diff --git a/lint.sh b/lint.sh
@@ -6,12 +6,6 @@ makepkgs=$(make list-yaml)
 for f in *.yaml; do
   echo "---" $f
 
-  # Check that every package is listed in packages.txt
-  if ! echo "$makepkgs" | grep -wq "$f"; then
-    echo "missing $f in packages.txt"
-    exit 1
-  fi
-
   # Don't specify packages.wolfi.dev/os as a repository, and remove it from the keyring.
   # Packages from the bootstrap repo should be allowed, but otherwise packages
   # should be fetched locally and the local repository should be appended at
diff --git a/mariadb-10.11 b/mariadb-10.11
@@ -0,0 +1 @@
+mariadb
diff --git a/mariadb-10.6 b/mariadb-10.6
@@ -0,0 +1 @@
+mariadb
diff --git a/postgresql-11 b/postgresql-11
@@ -0,0 +1 @@
+postgresql
diff --git a/postgresql-12 b/postgresql-12
@@ -0,0 +1 @@
+postgresql
diff --git a/postgresql-13 b/postgresql-13
@@ -0,0 +1 @@
+postgresql
diff --git a/postgresql-14 b/postgresql-14
@@ -0,0 +1 @@
+postgresql
diff --git a/postgresql-15 b/postgresql-15
@@ -0,0 +1 @@
+postgresql
