rebased and remerged

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: nigoroll

README.rst

@@ -1,12 +1,48 @@
-..
-	Copyright (c) 2016-2020 Varnish Software AS
-	SPDX-License-Identifier: BSD-2-Clause
-	See LICENSE file for full text of license
-
 Varnish Cache
 =============
 
-This is Varnish Cache, the high-performance HTTP accelerator.
+This is Varnish Cache, the high-performance HTTP accelerator with
+additional features and fixes.
+
+How this Repository is Organized
+--------------------------------
+
+The ``unmerged_code`` branch is created by merging feature/bug fix
+branches onto `Varnish Cache master`_. These branches are usually for
+`Varnish Cache Pull Requests`_.
+
+.. _Varnish Cache master: https://github.com/varnishcache/varnish-cache/tree/master
+.. _Varnish Cache Pull Requests: https://github.com/varnishcache/varnish-cache/pulls
+*NOTE* The ``unmerged_code`` branch gets force-pushed to
+github. Individual releases of this repository are published as
+branches named ``unmerged_code_``\ *<YYYY><mm><dd>*\ ``_``\
+*<HH><MM><SS>*.
+
+* Users wishing to use the lastest code should run::
+
+  $origin=${remote}
+  git pull
+  git reset --hard ${remote}/unmerged_code
+
+* Alternatively, pull the respective release branch.
+
+VMOD Compatibility
+------------------
+
+Changes merged in this branch might break existing interfaces and thus
+require changes to VMODs. We make sure that this branch works with a
+set of VMODs and create branches where necessary.
+
+The list of VMODs which we support and the respective repositories and
+branches is contained in the file ``VMODS.json``.
+
+When we create a release branch, we add ``VMODS.commits.json`` to
+contain a list of VMOD commit ids which have been successfully built
+with the release branch.
+
+
+General Information
+-------------------
 
 Documentation and additional information about Varnish is available on
 https://www.varnish-cache.org/
@@ -15,15 +51,3 @@ Technical questions about Varnish and this release should be addressed
 to <varnish-misc@varnish-cache.org>.
 
 Please see CONTRIBUTING for how to contribute patches and report bugs.
-
-For questions about commercial support and services related to Varnish
-see the `Varnish HTTP Cache Business page
-<https://varnish-cache.org/business/index.html>`_ .
-
-.. |ccibadge| image:: https://circleci.com/gh/varnishcache/varnish-cache/tree/master.svg?style=svg
-    :target: https://circleci.com/gh/varnishcache/varnish-cache/tree/master
-.. _vtest: https://varnish-cache.org/vtest/
-
-CircleCI tests: |ccibadge|
-
-More platforms are tested via vtest_

VMODS.json

@@ -0,0 +1,122 @@
+[
+  {
+    "vmod": "libvdfp-pipe",
+    "git": "https://code.uplex.de/uplex-varnish/libvdfp-pipe.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvdp-pesi",
+    "git": "https://code.uplex.de/uplex-varnish/libvdp-pesi.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvfp-brotli",
+    "git": "https://code.uplex.de/uplex-varnish/libvfp-brotli.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-all_healthy",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-all_healthy.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-blobdigest",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-blobdigest.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-blobsynth",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-blobsynth.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-cluster",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-cluster.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-crypto",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-crypto.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-dispatch",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-dispatch.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-dns",
+    "git": "https://github.com/nigoroll/libvmod-dns.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-dynamic",
+    "git": "https://github.com/nigoroll/libvmod-dynamic.git",
+    "branch": "proxy_via_6"
+  },
+  {
+    "vmod": "libvmod-frozen",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-frozen.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-gcrypt",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-gcrypt.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-geoip2",
+    "git": "https://github.com/nigoroll/libvmod-geoip2.git",
+    "branch": "lookup_fields"
+  },
+  {
+    "vmod": "libvmod-hoailona",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-hoailona.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-querystring",
+    "git": "https://github.com/nigoroll/libvmod-querystring",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-re",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-re.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-re2",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-re2.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-selector",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-selector.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-tus",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-tus.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-weightadjust",
+    "git": "https://code.uplex.de/uplex-varnish/libvmod-weightadjust.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "libvmod-xcounter",
+    "git": "https://github.com/xcir/libvmod-xcounter.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "varnish-modules",
+    "git": "https://github.com/nigoroll/varnish-modules.git",
+    "branch": "master"
+  },
+  {
+    "vmod": "varnish-objvar",
+    "git": "https://code.uplex.de/uplex-varnish/varnish-objvar.git",
+    "branch": "master"
+  }
+]

build.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3.9
+
+# - build and test varnish-cache and a number of vmods from a JSON
+#   config file
+# - generate a JSON output file with the git commit hash of each
+#   of the used repositories
+# - leave no trace: The build happens entirely out-of-tree in
+#   temporary directories
+#
+# ARGS: [check]
+
+import json
+import os
+import sys
+
+from contextlib import contextmanager
+from multiprocessing import cpu_count
+from subprocess import run
+from tempfile import TemporaryDirectory
+
+
+def r(*args, **kwargs):
+    r = run(*args, **kwargs)
+    if r.returncode:
+        print("FAILED " + " ".join(*args))
+        sys.exit(r.returncode)
+    return r
+
+
+@contextmanager
+def pushd(new_dir):
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def bootstrap_autogen_configure(srcdir, prefix, **kwargs):
+    """Run a dumb check if bootstrap looks like it would call configure
+    Dridi-style.
+    * If bootstrap calls configure, return the call to be made from the
+      builddir.
+    * Otherwise, call bootstrap or autogen and return the configure
+      call to be made from builddir
+    """
+
+    p = "--prefix=" + prefix
+    a = os.path.join(srcdir, "bootstrap")
+
+    if os.path.isfile(a):
+        with open(a, "r") as file:
+            if "/configure" in file.read():
+                return [a, p]
+    else:
+        a = os.path.join(srcdir, "autogen.sh")
+
+    with pushd(srcdir):
+        r([a], **kwargs)
+
+    a = os.path.join(srcdir, "configure")
+    return [a, p]
+
+
+def make(configure, **kwargs):
+    r(configure, **kwargs)
+    r(["make", "-j", str(cpu_count() + 2)], **kwargs)
+    if len(sys.argv) >= 2 and sys.argv[1] == "check":
+        r(["make", "-j", str(cpu_count() * 15), "check"], **kwargs)
+    r(["make", "install"], **kwargs)
+
+
+def build(srcdir, prefix, **kwargs):
+    configure = bootstrap_autogen_configure(srcdir, prefix, **kwargs)
+    run(["make", "maintainer-clean"])
+    run(["make", "distclean"])
+    with TemporaryDirectory() as builddir:
+        with pushd(str(builddir)):
+            make(configure, **kwargs)
+
+
+def build_vmod(name, prefix):
+    env = os.environ
+    # only required by vmod_dispatch
+    env["VARNISHSRC"] = varnishsrc
+    env["PKG_CONFIG_PATH"] = os.path.join(prefix, "lib", "pkgconfig")
+    env["ACLOCAL_PATH"] = os.path.join(prefix, "share", "aclocal")
+    build(os.getcwd(), prefix, env=env)
+
+
+# XXX how can we avoid special casing?
+# XXX does not support out-of-tree build
+def build_maxmind(prefix):
+    with pushd("libmaxminddb"):
+        configure = bootstrap_autogen_configure(os.getcwd(), prefix)
+        make(configure)
+
+
+def clone_build_vmod(vmod, prefix):
+    with TemporaryDirectory() as gitdir:
+        with pushd(str(gitdir)):
+            r(["git", "clone", "--recursive", vmod["git"]])
+            with pushd(vmod["vmod"]):
+                r(["git", "checkout", vmod["branch"]])
+                sub = run(["git", "describe"], capture_output=True, text=True)
+                if sub.returncode:
+                    sub = r(["git", "rev-parse", "--short", "HEAD"],
+                            capture_output=True, text=True)
+                vmod["rev"] = sub.stdout.rstrip()
+                del vmod["branch"]
+                if "geoip2" in vmod["vmod"]:
+                    build_maxmind(prefix)
+                build_vmod(vmod["vmod"], prefix)
+
+
+def build_vmods(vmods, prefix):
+    for vmod in vmods:
+        clone_build_vmod(vmod, prefix)
+
+
+varnishsrc = os.getcwd()
+vmods = None
+with open(os.path.join(varnishsrc, "VMODS.json"), "rb") as file:
+    vmods = json.loads(file.read())
+
+with TemporaryDirectory() as prefix:
+    build(varnishsrc, str(prefix))
+    build_vmods(vmods, str(prefix))
+
+with open(os.path.join(varnishsrc, "VMODS_BUILT.json"), "w") as file:
+    json.dump(vmods, file, indent=4, sort_keys=True)